1234 files changed, 219136 insertions, 120390 deletions

diff --git a/.gitignore b/.gitignore
index c5371b264..ff253c1da 100644
--- a/.gitignore
+++ b/.gitignore
@@ -8,26 +8,41 @@ install-sh
 ltmain.sh
 Makefile.in
 missing
+py-compile
 *.sw?
 *~
-*lo
-*la
-*o
+*.lo
+*.la
+*.o
+*.tar.gz
+*.rpm
 .libs
+.deps
 Makefile
 stamp-h1
 
 # Generated files
-extras/init.d/glusterfs-server.plist
-extras/init.d/glusterfsd-Debian
-extras/init.d/glusterfsd-Redhat
-extras/init.d/glusterfsd-SuSE
-glusterfs.spec
-libtool
-xlators/mount/fuse/utils/mount.glusterfs
-xlators/mount/fuse/utils/mount_glusterfs
+api/examples/__init__.py*
+api/examples/setup.py
 argp-standalone/libargp.a
+contrib/uuid/uuid_types.h
+extras/init.d/glusterd-Debian
+extras/init.d/glusterd-Redhat
+extras/init.d/glusterd-SuSE
+extras/init.d/glusterd.plist
+extras/ocf/glusterd
+extras/ocf/volume
+extras/who-wrote-glusterfs/gitdm
+glusterfs-api.pc
+glusterfs.spec
 glusterfsd/src/glusterfsd
+libgfchangelog.pc
 libglusterfs/src/spec.lex.c
 libglusterfs/src/y.tab.c
 libglusterfs/src/y.tab.h
+libtool
+run-tests.sh
+ufo/.tox
+ufo/test/unit/.coverage
+xlators/mount/fuse/utils/mount.glusterfs
+xlators/mount/fuse/utils/mount_glusterfs
diff --git a/.mailmap b/.mailmap
new file mode 100644
index 000000000..6bcd95dea
--- /dev/null
+++ b/.mailmap
@@ -0,0 +1,31 @@
+# .mailmap, see 'git short-log --help' for details
+#
+# Listing of contributors that filed patches with different email addresses.
+# Format: <name> <main-email> <alias> [<alias> ...]
+#
+
+Amar Tumballi <amarts@redhat.com> <amar@gluster.com> <amar@del.gluster.com>
+Anand Avati <avati@redhat.com> <avati@gluster.com> <avati@dev.gluster.com> <avati@amp.gluster.com> <avati@blackhole.gluster.com>
+Anush Shetty <ashetty@redhat.com> <anush@gluster.com>
+Csaba Henk <csaba@redhat.com> <csaba@gluster.com> <csaba@lowlife.hu> <csaba@zresearch.com>
+Harshavardhana <fharshav@redhat.com> <harsha@gluster.com> <harsha@zresearch.com> <harsha@dev.gluster.com> <harsha@harshavardhana.net>
+Kaleb S. KEITHLEY <kkeithle@redhat.com> <kkeithle@f16node1.kkeithle.usersys.redhat.com>
+Kaushal M <kaushal@redhat.com> <kaushal@gluster.com>
+Kaushik BV <kbudiger@redhat.com> <kaushikbv@gluster.com>
+Krishna Srinivas <ksriniva@redhat.com> <krishna@gluster.com> <krishna@zresearch.com> <krishna@guest-laptop>
+Krishnan Parthasarathi <kparthas@redhat.com> <kp@gluster.com>
+Louis Zuckerman <louiszuckerman@gmail.com> <me@louiszuckerman.com>
+M S Vishwanath Bhat <vbhat@redhat.com> <msvbhat@gmail.com> <vishwanath@gluster.com>
+Pavan Sondur <pavan@gluster.com> <pavan@dev.gluster.com>
+Pete Zaitcev <zaitcev@kotori.zaitcev.us> <zaitcev@yahoo.com>
+Pranith Kumar K <pkarampu@redhat.com> <pranithk@gluster.com>
+Raghavendra Bhat <raghavendra@redhat.com> <raghavendrabhat@gluster.com>
+Raghavendra G <rgowdapp@redhat.com> <raghavendra@gluster.com> <raghavendra@zresearch.com>
+Rahul C S <rahulcs@redhat.com> <rahulcssjce@gmail.com>
+Rajesh Amaravathi <rajesh@redhat.com> <rajesh@gluster.com> <rajesh.amaravathi@gmail.com>
+Shehjar Tikoo <shehjart@gluster.com> <shehjart@zresearch.com>
+Venky Shankar <vshankar@redhat.com> <venky@gluster.com>
+Vijay Bellur <vbellur@redhat.com> <vijay@gluster.com> <vijay@dev.gluster.com>
+Vijaykumar Koppad <vkoppad@redhat.com> <vijaykumar.koppad@gmail.com>
+Vikas Gorur <vikas@gluster.com> <vikas@zresearch.com>
+shishir gowda <sgowda@redhat.com> <shishirng@gluster.com>
diff --git a/CONTRIBUTING b/CONTRIBUTING
index 8b3baa7e5..7bccd88d7 100644
--- a/CONTRIBUTING
+++ b/CONTRIBUTING
@@ -22,4 +22,4 @@
             are public and that a record of the contribution (including all
             personal information I submit with it, including my sign-off) is
             maintained indefinitely and may be redistributed consistent with
-            this project or the open source license(s) involved.
\ No newline at end of file
+            this project or the open source license(s) involved.
diff --git a/COPYING b/COPYING
deleted file mode 100644
index 5737cfa27..000000000
--- a/COPYING
+++ /dev/null
@@ -1,674 +0,0 @@
-                    GNU GENERAL PUBLIC LICENSE
-                       Version 3, 29 June 2007
-
- Copyright (C) 2007 Free Software Foundation, Inc. <http://fsf.org/>
- Everyone is permitted to copy and distribute verbatim copies
- of this license document, but changing it is not allowed.
-
-                            Preamble
-
-  The GNU General Public License is a free, copyleft license for
-software and other kinds of works.
-
-  The licenses for most software and other practical works are designed
-to take away your freedom to share and change the works.  By contrast,
-the GNU General Public License is intended to guarantee your freedom to
-share and change all versions of a program--to make sure it remains free
-software for all its users.  We, the Free Software Foundation, use the
-GNU General Public License for most of our software; it applies also to
-any other work released this way by its authors.  You can apply it to
-your programs, too.
-
-  When we speak of free software, we are referring to freedom, not
-price.  Our General Public Licenses are designed to make sure that you
-have the freedom to distribute copies of free software (and charge for
-them if you wish), that you receive source code or can get it if you
-want it, that you can change the software or use pieces of it in new
-free programs, and that you know you can do these things.
-
-  To protect your rights, we need to prevent others from denying you
-these rights or asking you to surrender the rights.  Therefore, you have
-certain responsibilities if you distribute copies of the software, or if
-you modify it: responsibilities to respect the freedom of others.
-
-  For example, if you distribute copies of such a program, whether
-gratis or for a fee, you must pass on to the recipients the same
-freedoms that you received.  You must make sure that they, too, receive
-or can get the source code.  And you must show them these terms so they
-know their rights.
-
-  Developers that use the GNU GPL protect your rights with two steps:
-(1) assert copyright on the software, and (2) offer you this License
-giving you legal permission to copy, distribute and/or modify it.
-
-  For the developers' and authors' protection, the GPL clearly explains
-that there is no warranty for this free software.  For both users' and
-authors' sake, the GPL requires that modified versions be marked as
-changed, so that their problems will not be attributed erroneously to
-authors of previous versions.
-
-  Some devices are designed to deny users access to install or run
-modified versions of the software inside them, although the manufacturer
-can do so.  This is fundamentally incompatible with the aim of
-protecting users' freedom to change the software.  The systematic
-pattern of such abuse occurs in the area of products for individuals to
-use, which is precisely where it is most unacceptable.  Therefore, we
-have designed this version of the GPL to prohibit the practice for those
-products.  If such problems arise substantially in other domains, we
-stand ready to extend this provision to those domains in future versions
-of the GPL, as needed to protect the freedom of users.
-
-  Finally, every program is threatened constantly by software patents.
-States should not allow patents to restrict development and use of
-software on general-purpose computers, but in those that do, we wish to
-avoid the special danger that patents applied to a free program could
-make it effectively proprietary.  To prevent this, the GPL assures that
-patents cannot be used to render the program non-free.
-
-  The precise terms and conditions for copying, distribution and
-modification follow.
-
-                       TERMS AND CONDITIONS
-
-  0. Definitions.
-
-  "This License" refers to version 3 of the GNU General Public License.
-
-  "Copyright" also means copyright-like laws that apply to other kinds of
-works, such as semiconductor masks.
-
-  "The Program" refers to any copyrightable work licensed under this
-License.  Each licensee is addressed as "you".  "Licensees" and
-"recipients" may be individuals or organizations.
-
-  To "modify" a work means to copy from or adapt all or part of the work
-in a fashion requiring copyright permission, other than the making of an
-exact copy.  The resulting work is called a "modified version" of the
-earlier work or a work "based on" the earlier work.
-
-  A "covered work" means either the unmodified Program or a work based
-on the Program.
-
-  To "propagate" a work means to do anything with it that, without
-permission, would make you directly or secondarily liable for
-infringement under applicable copyright law, except executing it on a
-computer or modifying a private copy.  Propagation includes copying,
-distribution (with or without modification), making available to the
-public, and in some countries other activities as well.
-
-  To "convey" a work means any kind of propagation that enables other
-parties to make or receive copies.  Mere interaction with a user through
-a computer network, with no transfer of a copy, is not conveying.
-
-  An interactive user interface displays "Appropriate Legal Notices"
-to the extent that it includes a convenient and prominently visible
-feature that (1) displays an appropriate copyright notice, and (2)
-tells the user that there is no warranty for the work (except to the
-extent that warranties are provided), that licensees may convey the
-work under this License, and how to view a copy of this License.  If
-the interface presents a list of user commands or options, such as a
-menu, a prominent item in the list meets this criterion.
-
-  1. Source Code.
-
-  The "source code" for a work means the preferred form of the work
-for making modifications to it.  "Object code" means any non-source
-form of a work.
-
-  A "Standard Interface" means an interface that either is an official
-standard defined by a recognized standards body, or, in the case of
-interfaces specified for a particular programming language, one that
-is widely used among developers working in that language.
-
-  The "System Libraries" of an executable work include anything, other
-than the work as a whole, that (a) is included in the normal form of
-packaging a Major Component, but which is not part of that Major
-Component, and (b) serves only to enable use of the work with that
-Major Component, or to implement a Standard Interface for which an
-implementation is available to the public in source code form.  A
-"Major Component", in this context, means a major essential component
-(kernel, window system, and so on) of the specific operating system
-(if any) on which the executable work runs, or a compiler used to
-produce the work, or an object code interpreter used to run it.
-
-  The "Corresponding Source" for a work in object code form means all
-the source code needed to generate, install, and (for an executable
-work) run the object code and to modify the work, including scripts to
-control those activities.  However, it does not include the work's
-System Libraries, or general-purpose tools or generally available free
-programs which are used unmodified in performing those activities but
-which are not part of the work.  For example, Corresponding Source
-includes interface definition files associated with source files for
-the work, and the source code for shared libraries and dynamically
-linked subprograms that the work is specifically designed to require,
-such as by intimate data communication or control flow between those
-subprograms and other parts of the work.
-
-  The Corresponding Source need not include anything that users
-can regenerate automatically from other parts of the Corresponding
-Source.
-
-  The Corresponding Source for a work in source code form is that
-same work.
-
-  2. Basic Permissions.
-
-  All rights granted under this License are granted for the term of
-copyright on the Program, and are irrevocable provided the stated
-conditions are met.  This License explicitly affirms your unlimited
-permission to run the unmodified Program.  The output from running a
-covered work is covered by this License only if the output, given its
-content, constitutes a covered work.  This License acknowledges your
-rights of fair use or other equivalent, as provided by copyright law.
-
-  You may make, run and propagate covered works that you do not
-convey, without conditions so long as your license otherwise remains
-in force.  You may convey covered works to others for the sole purpose
-of having them make modifications exclusively for you, or provide you
-with facilities for running those works, provided that you comply with
-the terms of this License in conveying all material for which you do
-not control copyright.  Those thus making or running the covered works
-for you must do so exclusively on your behalf, under your direction
-and control, on terms that prohibit them from making any copies of
-your copyrighted material outside their relationship with you.
-
-  Conveying under any other circumstances is permitted solely under
-the conditions stated below.  Sublicensing is not allowed; section 10
-makes it unnecessary.
-
-  3. Protecting Users' Legal Rights From Anti-Circumvention Law.
-
-  No covered work shall be deemed part of an effective technological
-measure under any applicable law fulfilling obligations under article
-11 of the WIPO copyright treaty adopted on 20 December 1996, or
-similar laws prohibiting or restricting circumvention of such
-measures.
-
-  When you convey a covered work, you waive any legal power to forbid
-circumvention of technological measures to the extent such circumvention
-is effected by exercising rights under this License with respect to
-the covered work, and you disclaim any intention to limit operation or
-modification of the work as a means of enforcing, against the work's
-users, your or third parties' legal rights to forbid circumvention of
-technological measures.
-
-  4. Conveying Verbatim Copies.
-
-  You may convey verbatim copies of the Program's source code as you
-receive it, in any medium, provided that you conspicuously and
-appropriately publish on each copy an appropriate copyright notice;
-keep intact all notices stating that this License and any
-non-permissive terms added in accord with section 7 apply to the code;
-keep intact all notices of the absence of any warranty; and give all
-recipients a copy of this License along with the Program.
-
-  You may charge any price or no price for each copy that you convey,
-and you may offer support or warranty protection for a fee.
-
-  5. Conveying Modified Source Versions.
-
-  You may convey a work based on the Program, or the modifications to
-produce it from the Program, in the form of source code under the
-terms of section 4, provided that you also meet all of these conditions:
-
-    a) The work must carry prominent notices stating that you modified
-    it, and giving a relevant date.
-
-    b) The work must carry prominent notices stating that it is
-    released under this License and any conditions added under section
-    7.  This requirement modifies the requirement in section 4 to
-    "keep intact all notices".
-
-    c) You must license the entire work, as a whole, under this
-    License to anyone who comes into possession of a copy.  This
-    License will therefore apply, along with any applicable section 7
-    additional terms, to the whole of the work, and all its parts,
-    regardless of how they are packaged.  This License gives no
-    permission to license the work in any other way, but it does not
-    invalidate such permission if you have separately received it.
-
-    d) If the work has interactive user interfaces, each must display
-    Appropriate Legal Notices; however, if the Program has interactive
-    interfaces that do not display Appropriate Legal Notices, your
-    work need not make them do so.
-
-  A compilation of a covered work with other separate and independent
-works, which are not by their nature extensions of the covered work,
-and which are not combined with it such as to form a larger program,
-in or on a volume of a storage or distribution medium, is called an
-"aggregate" if the compilation and its resulting copyright are not
-used to limit the access or legal rights of the compilation's users
-beyond what the individual works permit.  Inclusion of a covered work
-in an aggregate does not cause this License to apply to the other
-parts of the aggregate.
-
-  6. Conveying Non-Source Forms.
-
-  You may convey a covered work in object code form under the terms
-of sections 4 and 5, provided that you also convey the
-machine-readable Corresponding Source under the terms of this License,
-in one of these ways:
-
-    a) Convey the object code in, or embodied in, a physical product
-    (including a physical distribution medium), accompanied by the
-    Corresponding Source fixed on a durable physical medium
-    customarily used for software interchange.
-
-    b) Convey the object code in, or embodied in, a physical product
-    (including a physical distribution medium), accompanied by a
-    written offer, valid for at least three years and valid for as
-    long as you offer spare parts or customer support for that product
-    model, to give anyone who possesses the object code either (1) a
-    copy of the Corresponding Source for all the software in the
-    product that is covered by this License, on a durable physical
-    medium customarily used for software interchange, for a price no
-    more than your reasonable cost of physically performing this
-    conveying of source, or (2) access to copy the
-    Corresponding Source from a network server at no charge.
-
-    c) Convey individual copies of the object code with a copy of the
-    written offer to provide the Corresponding Source.  This
-    alternative is allowed only occasionally and noncommercially, and
-    only if you received the object code with such an offer, in accord
-    with subsection 6b.
-
-    d) Convey the object code by offering access from a designated
-    place (gratis or for a charge), and offer equivalent access to the
-    Corresponding Source in the same way through the same place at no
-    further charge.  You need not require recipients to copy the
-    Corresponding Source along with the object code.  If the place to
-    copy the object code is a network server, the Corresponding Source
-    may be on a different server (operated by you or a third party)
-    that supports equivalent copying facilities, provided you maintain
-    clear directions next to the object code saying where to find the
-    Corresponding Source.  Regardless of what server hosts the
-    Corresponding Source, you remain obligated to ensure that it is
-    available for as long as needed to satisfy these requirements.
-
-    e) Convey the object code using peer-to-peer transmission, provided
-    you inform other peers where the object code and Corresponding
-    Source of the work are being offered to the general public at no
-    charge under subsection 6d.
-
-  A separable portion of the object code, whose source code is excluded
-from the Corresponding Source as a System Library, need not be
-included in conveying the object code work.
-
-  A "User Product" is either (1) a "consumer product", which means any
-tangible personal property which is normally used for personal, family,
-or household purposes, or (2) anything designed or sold for incorporation
-into a dwelling.  In determining whether a product is a consumer product,
-doubtful cases shall be resolved in favor of coverage.  For a particular
-product received by a particular user, "normally used" refers to a
-typical or common use of that class of product, regardless of the status
-of the particular user or of the way in which the particular user
-actually uses, or expects or is expected to use, the product.  A product
-is a consumer product regardless of whether the product has substantial
-commercial, industrial or non-consumer uses, unless such uses represent
-the only significant mode of use of the product.
-
-  "Installation Information" for a User Product means any methods,
-procedures, authorization keys, or other information required to install
-and execute modified versions of a covered work in that User Product from
-a modified version of its Corresponding Source.  The information must
-suffice to ensure that the continued functioning of the modified object
-code is in no case prevented or interfered with solely because
-modification has been made.
-
-  If you convey an object code work under this section in, or with, or
-specifically for use in, a User Product, and the conveying occurs as
-part of a transaction in which the right of possession and use of the
-User Product is transferred to the recipient in perpetuity or for a
-fixed term (regardless of how the transaction is characterized), the
-Corresponding Source conveyed under this section must be accompanied
-by the Installation Information.  But this requirement does not apply
-if neither you nor any third party retains the ability to install
-modified object code on the User Product (for example, the work has
-been installed in ROM).
-
-  The requirement to provide Installation Information does not include a
-requirement to continue to provide support service, warranty, or updates
-for a work that has been modified or installed by the recipient, or for
-the User Product in which it has been modified or installed.  Access to a
-network may be denied when the modification itself materially and
-adversely affects the operation of the network or violates the rules and
-protocols for communication across the network.
-
-  Corresponding Source conveyed, and Installation Information provided,
-in accord with this section must be in a format that is publicly
-documented (and with an implementation available to the public in
-source code form), and must require no special password or key for
-unpacking, reading or copying.
-
-  7. Additional Terms.
-
-  "Additional permissions" are terms that supplement the terms of this
-License by making exceptions from one or more of its conditions.
-Additional permissions that are applicable to the entire Program shall
-be treated as though they were included in this License, to the extent
-that they are valid under applicable law.  If additional permissions
-apply only to part of the Program, that part may be used separately
-under those permissions, but the entire Program remains governed by
-this License without regard to the additional permissions.
-
-  When you convey a copy of a covered work, you may at your option
-remove any additional permissions from that copy, or from any part of
-it.  (Additional permissions may be written to require their own
-removal in certain cases when you modify the work.)  You may place
-additional permissions on material, added by you to a covered work,
-for which you have or can give appropriate copyright permission.
-
-  Notwithstanding any other provision of this License, for material you
-add to a covered work, you may (if authorized by the copyright holders of
-that material) supplement the terms of this License with terms:
-
-    a) Disclaiming warranty or limiting liability differently from the
-    terms of sections 15 and 16 of this License; or
-
-    b) Requiring preservation of specified reasonable legal notices or
-    author attributions in that material or in the Appropriate Legal
-    Notices displayed by works containing it; or
-
-    c) Prohibiting misrepresentation of the origin of that material, or
-    requiring that modified versions of such material be marked in
-    reasonable ways as different from the original version; or
-
-    d) Limiting the use for publicity purposes of names of licensors or
-    authors of the material; or
-
-    e) Declining to grant rights under trademark law for use of some
-    trade names, trademarks, or service marks; or
-
-    f) Requiring indemnification of licensors and authors of that
-    material by anyone who conveys the material (or modified versions of
-    it) with contractual assumptions of liability to the recipient, for
-    any liability that these contractual assumptions directly impose on
-    those licensors and authors.
-
-  All other non-permissive additional terms are considered "further
-restrictions" within the meaning of section 10.  If the Program as you
-received it, or any part of it, contains a notice stating that it is
-governed by this License along with a term that is a further
-restriction, you may remove that term.  If a license document contains
-a further restriction but permits relicensing or conveying under this
-License, you may add to a covered work material governed by the terms
-of that license document, provided that the further restriction does
-not survive such relicensing or conveying.
-
-  If you add terms to a covered work in accord with this section, you
-must place, in the relevant source files, a statement of the
-additional terms that apply to those files, or a notice indicating
-where to find the applicable terms.
-
-  Additional terms, permissive or non-permissive, may be stated in the
-form of a separately written license, or stated as exceptions;
-the above requirements apply either way.
-
-  8. Termination.
-
-  You may not propagate or modify a covered work except as expressly
-provided under this License.  Any attempt otherwise to propagate or
-modify it is void, and will automatically terminate your rights under
-this License (including any patent licenses granted under the third
-paragraph of section 11).
-
-  However, if you cease all violation of this License, then your
-license from a particular copyright holder is reinstated (a)
-provisionally, unless and until the copyright holder explicitly and
-finally terminates your license, and (b) permanently, if the copyright
-holder fails to notify you of the violation by some reasonable means
-prior to 60 days after the cessation.
-
-  Moreover, your license from a particular copyright holder is
-reinstated permanently if the copyright holder notifies you of the
-violation by some reasonable means, this is the first time you have
-received notice of violation of this License (for any work) from that
-copyright holder, and you cure the violation prior to 30 days after
-your receipt of the notice.
-
-  Termination of your rights under this section does not terminate the
-licenses of parties who have received copies or rights from you under
-this License.  If your rights have been terminated and not permanently
-reinstated, you do not qualify to receive new licenses for the same
-material under section 10.
-
-  9. Acceptance Not Required for Having Copies.
-
-  You are not required to accept this License in order to receive or
-run a copy of the Program.  Ancillary propagation of a covered work
-occurring solely as a consequence of using peer-to-peer transmission
-to receive a copy likewise does not require acceptance.  However,
-nothing other than this License grants you permission to propagate or
-modify any covered work.  These actions infringe copyright if you do
-not accept this License.  Therefore, by modifying or propagating a
-covered work, you indicate your acceptance of this License to do so.
-
-  10. Automatic Licensing of Downstream Recipients.
-
-  Each time you convey a covered work, the recipient automatically
-receives a license from the original licensors, to run, modify and
-propagate that work, subject to this License.  You are not responsible
-for enforcing compliance by third parties with this License.
-
-  An "entity transaction" is a transaction transferring control of an
-organization, or substantially all assets of one, or subdividing an
-organization, or merging organizations.  If propagation of a covered
-work results from an entity transaction, each party to that
-transaction who receives a copy of the work also receives whatever
-licenses to the work the party's predecessor in interest had or could
-give under the previous paragraph, plus a right to possession of the
-Corresponding Source of the work from the predecessor in interest, if
-the predecessor has it or can get it with reasonable efforts.
-
-  You may not impose any further restrictions on the exercise of the
-rights granted or affirmed under this License.  For example, you may
-not impose a license fee, royalty, or other charge for exercise of
-rights granted under this License, and you may not initiate litigation
-(including a cross-claim or counterclaim in a lawsuit) alleging that
-any patent claim is infringed by making, using, selling, offering for
-sale, or importing the Program or any portion of it.
-
-  11. Patents.
-
-  A "contributor" is a copyright holder who authorizes use under this
-License of the Program or a work on which the Program is based.  The
-work thus licensed is called the contributor's "contributor version".
-
-  A contributor's "essential patent claims" are all patent claims
-owned or controlled by the contributor, whether already acquired or
-hereafter acquired, that would be infringed by some manner, permitted
-by this License, of making, using, or selling its contributor version,
-but do not include claims that would be infringed only as a
-consequence of further modification of the contributor version.  For
-purposes of this definition, "control" includes the right to grant
-patent sublicenses in a manner consistent with the requirements of
-this License.
-
-  Each contributor grants you a non-exclusive, worldwide, royalty-free
-patent license under the contributor's essential patent claims, to
-make, use, sell, offer for sale, import and otherwise run, modify and
-propagate the contents of its contributor version.
-
-  In the following three paragraphs, a "patent license" is any express
-agreement or commitment, however denominated, not to enforce a patent
-(such as an express permission to practice a patent or covenant not to
-sue for patent infringement).  To "grant" such a patent license to a
-party means to make such an agreement or commitment not to enforce a
-patent against the party.
-
-  If you convey a covered work, knowingly relying on a patent license,
-and the Corresponding Source of the work is not available for anyone
-to copy, free of charge and under the terms of this License, through a
-publicly available network server or other readily accessible means,
-then you must either (1) cause the Corresponding Source to be so
-available, or (2) arrange to deprive yourself of the benefit of the
-patent license for this particular work, or (3) arrange, in a manner
-consistent with the requirements of this License, to extend the patent
-license to downstream recipients.  "Knowingly relying" means you have
-actual knowledge that, but for the patent license, your conveying the
-covered work in a country, or your recipient's use of the covered work
-in a country, would infringe one or more identifiable patents in that
-country that you have reason to believe are valid.
-
-  If, pursuant to or in connection with a single transaction or
-arrangement, you convey, or propagate by procuring conveyance of, a
-covered work, and grant a patent license to some of the parties
-receiving the covered work authorizing them to use, propagate, modify
-or convey a specific copy of the covered work, then the patent license
-you grant is automatically extended to all recipients of the covered
-work and works based on it.
-
-  A patent license is "discriminatory" if it does not include within
-the scope of its coverage, prohibits the exercise of, or is
-conditioned on the non-exercise of one or more of the rights that are
-specifically granted under this License.  You may not convey a covered
-work if you are a party to an arrangement with a third party that is
-in the business of distributing software, under which you make payment
-to the third party based on the extent of your activity of conveying
-the work, and under which the third party grants, to any of the
-parties who would receive the covered work from you, a discriminatory
-patent license (a) in connection with copies of the covered work
-conveyed by you (or copies made from those copies), or (b) primarily
-for and in connection with specific products or compilations that
-contain the covered work, unless you entered into that arrangement,
-or that patent license was granted, prior to 28 March 2007.
-
-  Nothing in this License shall be construed as excluding or limiting
-any implied license or other defenses to infringement that may
-otherwise be available to you under applicable patent law.
-
-  12. No Surrender of Others' Freedom.
-
-  If conditions are imposed on you (whether by court order, agreement or
-otherwise) that contradict the conditions of this License, they do not
-excuse you from the conditions of this License.  If you cannot convey a
-covered work so as to satisfy simultaneously your obligations under this
-License and any other pertinent obligations, then as a consequence you may
-not convey it at all.  For example, if you agree to terms that obligate you
-to collect a royalty for further conveying from those to whom you convey
-the Program, the only way you could satisfy both those terms and this
-License would be to refrain entirely from conveying the Program.
-
-  13. Use with the GNU General Public License.
-
-  Notwithstanding any other provision of this License, you have
-permission to link or combine any covered work with a work licensed
-under version 3 of the GNU General Public License into a single
-combined work, and to convey the resulting work.  The terms of this
-License will continue to apply to the part which is the covered work,
-but the special requirements of the GNU General Public License,
-section 13, concerning interaction through a network will apply to the
-combination as such.
-
-  14. Revised Versions of this License.
-
-  The Free Software Foundation may publish revised and/or new versions of
-the GNU General Public License from time to time.  Such new versions will
-be similar in spirit to the present version, but may differ in detail to
-address new problems or concerns.
-
-  Each version is given a distinguishing version number.  If the
-Program specifies that a certain numbered version of the GNU General
-Public License "or any later version" applies to it, you have the
-option of following the terms and conditions either of that numbered
-version or of any later version published by the Free Software
-Foundation.  If the Program does not specify a version number of the
-GNU General Public License, you may choose any version ever published
-by the Free Software Foundation.
-
-  If the Program specifies that a proxy can decide which future
-versions of the GNU General Public License can be used, that proxy's
-public statement of acceptance of a version permanently authorizes you
-to choose that version for the Program.
-
-  Later license versions may give you additional or different
-permissions.  However, no additional obligations are imposed on any
-author or copyright holder as a result of your choosing to follow a
-later version.
-
-  15. Disclaimer of Warranty.
-
-  THERE IS NO WARRANTY FOR THE PROGRAM, TO THE EXTENT PERMITTED BY
-APPLICABLE LAW.  EXCEPT WHEN OTHERWISE STATED IN WRITING THE COPYRIGHT
-HOLDERS AND/OR OTHER PARTIES PROVIDE THE PROGRAM "AS IS" WITHOUT WARRANTY
-OF ANY KIND, EITHER EXPRESSED OR IMPLIED, INCLUDING, BUT NOT LIMITED TO,
-THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
-PURPOSE.  THE ENTIRE RISK AS TO THE QUALITY AND PERFORMANCE OF THE PROGRAM
-IS WITH YOU.  SHOULD THE PROGRAM PROVE DEFECTIVE, YOU ASSUME THE COST OF
-ALL NECESSARY SERVICING, REPAIR OR CORRECTION.
-
-  16. Limitation of Liability.
-
-  IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN WRITING
-WILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MODIFIES AND/OR CONVEYS
-THE PROGRAM AS PERMITTED ABOVE, BE LIABLE TO YOU FOR DAMAGES, INCLUDING ANY
-GENERAL, SPECIAL, INCIDENTAL OR CONSEQUENTIAL DAMAGES ARISING OUT OF THE
-USE OR INABILITY TO USE THE PROGRAM (INCLUDING BUT NOT LIMITED TO LOSS OF
-DATA OR DATA BEING RENDERED INACCURATE OR LOSSES SUSTAINED BY YOU OR THIRD
-PARTIES OR A FAILURE OF THE PROGRAM TO OPERATE WITH ANY OTHER PROGRAMS),
-EVEN IF SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE POSSIBILITY OF
-SUCH DAMAGES.
-
-  17. Interpretation of Sections 15 and 16.
-
-  If the disclaimer of warranty and limitation of liability provided
-above cannot be given local legal effect according to their terms,
-reviewing courts shall apply local law that most closely approximates
-an absolute waiver of all civil liability in connection with the
-Program, unless a warranty or assumption of liability accompanies a
-copy of the Program in return for a fee.
-
-                     END OF TERMS AND CONDITIONS
-
-            How to Apply These Terms to Your New Programs
-
-  If you develop a new program, and you want it to be of the greatest
-possible use to the public, the best way to achieve this is to make it
-free software which everyone can redistribute and change under these terms.
-
-  To do so, attach the following notices to the program.  It is safest
-to attach them to the start of each source file to most effectively
-state the exclusion of warranty; and each file should have at least
-the "copyright" line and a pointer to where the full notice is found.
-
-    <one line to give the program's name and a brief idea of what it does.>
-    Copyright (C) <year>  <name of author>
-
-    This program is free software: you can redistribute it and/or modify
-    it under the terms of the GNU General Public License as published by
-    the Free Software Foundation, either version 3 of the License, or
-    (at your option) any later version.
-
-    This program is distributed in the hope that it will be useful,
-    but WITHOUT ANY WARRANTY; without even the implied warranty of
-    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-    GNU General Public License for more details.
-
-    You should have received a copy of the GNU General Public License
-    along with this program.  If not, see <http://www.gnu.org/licenses/>.
-
-Also add information on how to contact you by electronic and paper mail.
-
-  If the program does terminal interaction, make it output a short
-notice like this when it starts in an interactive mode:
-
-    <program>  Copyright (C) <year>  <name of author>
-    This program comes with ABSOLUTELY NO WARRANTY; for details type `show w'.
-    This is free software, and you are welcome to redistribute it
-    under certain conditions; type `show c' for details.
-
-The hypothetical commands `show w' and `show c' should show the appropriate
-parts of the General Public License.  Of course, your program's commands
-might be different; for a GUI interface, you would use an "about box".
-
-  You should also get your employer (if you work as a programmer) or school,
-if any, to sign a "copyright disclaimer" for the program, if necessary.
-For more information on this, and how to apply and follow the GNU GPL, see
-<http://www.gnu.org/licenses/>.
-
-  The GNU General Public License does not permit incorporating your program
-into proprietary programs.  If your program is a subroutine library, you
-may consider it more useful to permit linking proprietary applications with
-the library.  If this is what you want to do, use the GNU Lesser General
-Public License instead of this License.  But first, please read
-<http://www.gnu.org/philosophy/why-not-lgpl.html>.
diff --git a/COPYING-GPLV2 b/COPYING-GPLV2
new file mode 100644
index 000000000..d159169d1
--- /dev/null
+++ b/COPYING-GPLV2
@@ -0,0 +1,339 @@
+                    GNU GENERAL PUBLIC LICENSE
+                       Version 2, June 1991
+
+ Copyright (C) 1989, 1991 Free Software Foundation, Inc.,
+ 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
+ Everyone is permitted to copy and distribute verbatim copies
+ of this license document, but changing it is not allowed.
+
+                            Preamble
+
+  The licenses for most software are designed to take away your
+freedom to share and change it.  By contrast, the GNU General Public
+License is intended to guarantee your freedom to share and change free
+software--to make sure the software is free for all its users.  This
+General Public License applies to most of the Free Software
+Foundation's software and to any other program whose authors commit to
+using it.  (Some other Free Software Foundation software is covered by
+the GNU Lesser General Public License instead.)  You can apply it to
+your programs, too.
+
+  When we speak of free software, we are referring to freedom, not
+price.  Our General Public Licenses are designed to make sure that you
+have the freedom to distribute copies of free software (and charge for
+this service if you wish), that you receive source code or can get it
+if you want it, that you can change the software or use pieces of it
+in new free programs; and that you know you can do these things.
+
+  To protect your rights, we need to make restrictions that forbid
+anyone to deny you these rights or to ask you to surrender the rights.
+These restrictions translate to certain responsibilities for you if you
+distribute copies of the software, or if you modify it.
+
+  For example, if you distribute copies of such a program, whether
+gratis or for a fee, you must give the recipients all the rights that
+you have.  You must make sure that they, too, receive or can get the
+source code.  And you must show them these terms so they know their
+rights.
+
+  We protect your rights with two steps: (1) copyright the software, and
+(2) offer you this license which gives you legal permission to copy,
+distribute and/or modify the software.
+
+  Also, for each author's protection and ours, we want to make certain
+that everyone understands that there is no warranty for this free
+software.  If the software is modified by someone else and passed on, we
+want its recipients to know that what they have is not the original, so
+that any problems introduced by others will not reflect on the original
+authors' reputations.
+
+  Finally, any free program is threatened constantly by software
+patents.  We wish to avoid the danger that redistributors of a free
+program will individually obtain patent licenses, in effect making the
+program proprietary.  To prevent this, we have made it clear that any
+patent must be licensed for everyone's free use or not licensed at all.
+
+  The precise terms and conditions for copying, distribution and
+modification follow.
+
+                    GNU GENERAL PUBLIC LICENSE
+   TERMS AND CONDITIONS FOR COPYING, DISTRIBUTION AND MODIFICATION
+
+  0. This License applies to any program or other work which contains
+a notice placed by the copyright holder saying it may be distributed
+under the terms of this General Public License.  The "Program", below,
+refers to any such program or work, and a "work based on the Program"
+means either the Program or any derivative work under copyright law:
+that is to say, a work containing the Program or a portion of it,
+either verbatim or with modifications and/or translated into another
+language.  (Hereinafter, translation is included without limitation in
+the term "modification".)  Each licensee is addressed as "you".
+
+Activities other than copying, distribution and modification are not
+covered by this License; they are outside its scope.  The act of
+running the Program is not restricted, and the output from the Program
+is covered only if its contents constitute a work based on the
+Program (independent of having been made by running the Program).
+Whether that is true depends on what the Program does.
+
+  1. You may copy and distribute verbatim copies of the Program's
+source code as you receive it, in any medium, provided that you
+conspicuously and appropriately publish on each copy an appropriate
+copyright notice and disclaimer of warranty; keep intact all the
+notices that refer to this License and to the absence of any warranty;
+and give any other recipients of the Program a copy of this License
+along with the Program.
+
+You may charge a fee for the physical act of transferring a copy, and
+you may at your option offer warranty protection in exchange for a fee.
+
+  2. You may modify your copy or copies of the Program or any portion
+of it, thus forming a work based on the Program, and copy and
+distribute such modifications or work under the terms of Section 1
+above, provided that you also meet all of these conditions:
+
+    a) You must cause the modified files to carry prominent notices
+    stating that you changed the files and the date of any change.
+
+    b) You must cause any work that you distribute or publish, that in
+    whole or in part contains or is derived from the Program or any
+    part thereof, to be licensed as a whole at no charge to all third
+    parties under the terms of this License.
+
+    c) If the modified program normally reads commands interactively
+    when run, you must cause it, when started running for such
+    interactive use in the most ordinary way, to print or display an
+    announcement including an appropriate copyright notice and a
+    notice that there is no warranty (or else, saying that you provide
+    a warranty) and that users may redistribute the program under
+    these conditions, and telling the user how to view a copy of this
+    License.  (Exception: if the Program itself is interactive but
+    does not normally print such an announcement, your work based on
+    the Program is not required to print an announcement.)
+
+These requirements apply to the modified work as a whole.  If
+identifiable sections of that work are not derived from the Program,
+and can be reasonably considered independent and separate works in
+themselves, then this License, and its terms, do not apply to those
+sections when you distribute them as separate works.  But when you
+distribute the same sections as part of a whole which is a work based
+on the Program, the distribution of the whole must be on the terms of
+this License, whose permissions for other licensees extend to the
+entire whole, and thus to each and every part regardless of who wrote it.
+
+Thus, it is not the intent of this section to claim rights or contest
+your rights to work written entirely by you; rather, the intent is to
+exercise the right to control the distribution of derivative or
+collective works based on the Program.
+
+In addition, mere aggregation of another work not based on the Program
+with the Program (or with a work based on the Program) on a volume of
+a storage or distribution medium does not bring the other work under
+the scope of this License.
+
+  3. You may copy and distribute the Program (or a work based on it,
+under Section 2) in object code or executable form under the terms of
+Sections 1 and 2 above provided that you also do one of the following:
+
+    a) Accompany it with the complete corresponding machine-readable
+    source code, which must be distributed under the terms of Sections
+    1 and 2 above on a medium customarily used for software interchange; or,
+
+    b) Accompany it with a written offer, valid for at least three
+    years, to give any third party, for a charge no more than your
+    cost of physically performing source distribution, a complete
+    machine-readable copy of the corresponding source code, to be
+    distributed under the terms of Sections 1 and 2 above on a medium
+    customarily used for software interchange; or,
+
+    c) Accompany it with the information you received as to the offer
+    to distribute corresponding source code.  (This alternative is
+    allowed only for noncommercial distribution and only if you
+    received the program in object code or executable form with such
+    an offer, in accord with Subsection b above.)
+
+The source code for a work means the preferred form of the work for
+making modifications to it.  For an executable work, complete source
+code means all the source code for all modules it contains, plus any
+associated interface definition files, plus the scripts used to
+control compilation and installation of the executable.  However, as a
+special exception, the source code distributed need not include
+anything that is normally distributed (in either source or binary
+form) with the major components (compiler, kernel, and so on) of the
+operating system on which the executable runs, unless that component
+itself accompanies the executable.
+
+If distribution of executable or object code is made by offering
+access to copy from a designated place, then offering equivalent
+access to copy the source code from the same place counts as
+distribution of the source code, even though third parties are not
+compelled to copy the source along with the object code.
+
+  4. You may not copy, modify, sublicense, or distribute the Program
+except as expressly provided under this License.  Any attempt
+otherwise to copy, modify, sublicense or distribute the Program is
+void, and will automatically terminate your rights under this License.
+However, parties who have received copies, or rights, from you under
+this License will not have their licenses terminated so long as such
+parties remain in full compliance.
+
+  5. You are not required to accept this License, since you have not
+signed it.  However, nothing else grants you permission to modify or
+distribute the Program or its derivative works.  These actions are
+prohibited by law if you do not accept this License.  Therefore, by
+modifying or distributing the Program (or any work based on the
+Program), you indicate your acceptance of this License to do so, and
+all its terms and conditions for copying, distributing or modifying
+the Program or works based on it.
+
+  6. Each time you redistribute the Program (or any work based on the
+Program), the recipient automatically receives a license from the
+original licensor to copy, distribute or modify the Program subject to
+these terms and conditions.  You may not impose any further
+restrictions on the recipients' exercise of the rights granted herein.
+You are not responsible for enforcing compliance by third parties to
+this License.
+
+  7. If, as a consequence of a court judgment or allegation of patent
+infringement or for any other reason (not limited to patent issues),
+conditions are imposed on you (whether by court order, agreement or
+otherwise) that contradict the conditions of this License, they do not
+excuse you from the conditions of this License.  If you cannot
+distribute so as to satisfy simultaneously your obligations under this
+License and any other pertinent obligations, then as a consequence you
+may not distribute the Program at all.  For example, if a patent
+license would not permit royalty-free redistribution of the Program by
+all those who receive copies directly or indirectly through you, then
+the only way you could satisfy both it and this License would be to
+refrain entirely from distribution of the Program.
+
+If any portion of this section is held invalid or unenforceable under
+any particular circumstance, the balance of the section is intended to
+apply and the section as a whole is intended to apply in other
+circumstances.
+
+It is not the purpose of this section to induce you to infringe any
+patents or other property right claims or to contest validity of any
+such claims; this section has the sole purpose of protecting the
+integrity of the free software distribution system, which is
+implemented by public license practices.  Many people have made
+generous contributions to the wide range of software distributed
+through that system in reliance on consistent application of that
+system; it is up to the author/donor to decide if he or she is willing
+to distribute software through any other system and a licensee cannot
+impose that choice.
+
+This section is intended to make thoroughly clear what is believed to
+be a consequence of the rest of this License.
+
+  8. If the distribution and/or use of the Program is restricted in
+certain countries either by patents or by copyrighted interfaces, the
+original copyright holder who places the Program under this License
+may add an explicit geographical distribution limitation excluding
+those countries, so that distribution is permitted only in or among
+countries not thus excluded.  In such case, this License incorporates
+the limitation as if written in the body of this License.
+
+  9. The Free Software Foundation may publish revised and/or new versions
+of the General Public License from time to time.  Such new versions will
+be similar in spirit to the present version, but may differ in detail to
+address new problems or concerns.
+
+Each version is given a distinguishing version number.  If the Program
+specifies a version number of this License which applies to it and "any
+later version", you have the option of following the terms and conditions
+either of that version or of any later version published by the Free
+Software Foundation.  If the Program does not specify a version number of
+this License, you may choose any version ever published by the Free Software
+Foundation.
+
+  10. If you wish to incorporate parts of the Program into other free
+programs whose distribution conditions are different, write to the author
+to ask for permission.  For software which is copyrighted by the Free
+Software Foundation, write to the Free Software Foundation; we sometimes
+make exceptions for this.  Our decision will be guided by the two goals
+of preserving the free status of all derivatives of our free software and
+of promoting the sharing and reuse of software generally.
+
+                            NO WARRANTY
+
+  11. BECAUSE THE PROGRAM IS LICENSED FREE OF CHARGE, THERE IS NO WARRANTY
+FOR THE PROGRAM, TO THE EXTENT PERMITTED BY APPLICABLE LAW.  EXCEPT WHEN
+OTHERWISE STATED IN WRITING THE COPYRIGHT HOLDERS AND/OR OTHER PARTIES
+PROVIDE THE PROGRAM "AS IS" WITHOUT WARRANTY OF ANY KIND, EITHER EXPRESSED
+OR IMPLIED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF
+MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE.  THE ENTIRE RISK AS
+TO THE QUALITY AND PERFORMANCE OF THE PROGRAM IS WITH YOU.  SHOULD THE
+PROGRAM PROVE DEFECTIVE, YOU ASSUME THE COST OF ALL NECESSARY SERVICING,
+REPAIR OR CORRECTION.
+
+  12. IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN WRITING
+WILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MAY MODIFY AND/OR
+REDISTRIBUTE THE PROGRAM AS PERMITTED ABOVE, BE LIABLE TO YOU FOR DAMAGES,
+INCLUDING ANY GENERAL, SPECIAL, INCIDENTAL OR CONSEQUENTIAL DAMAGES ARISING
+OUT OF THE USE OR INABILITY TO USE THE PROGRAM (INCLUDING BUT NOT LIMITED
+TO LOSS OF DATA OR DATA BEING RENDERED INACCURATE OR LOSSES SUSTAINED BY
+YOU OR THIRD PARTIES OR A FAILURE OF THE PROGRAM TO OPERATE WITH ANY OTHER
+PROGRAMS), EVEN IF SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE
+POSSIBILITY OF SUCH DAMAGES.
+
+                     END OF TERMS AND CONDITIONS
+
+            How to Apply These Terms to Your New Programs
+
+  If you develop a new program, and you want it to be of the greatest
+possible use to the public, the best way to achieve this is to make it
+free software which everyone can redistribute and change under these terms.
+
+  To do so, attach the following notices to the program.  It is safest
+to attach them to the start of each source file to most effectively
+convey the exclusion of warranty; and each file should have at least
+the "copyright" line and a pointer to where the full notice is found.
+
+    <one line to give the program's name and a brief idea of what it does.>
+    Copyright (C) <year>  <name of author>
+
+    This program is free software; you can redistribute it and/or modify
+    it under the terms of the GNU General Public License as published by
+    the Free Software Foundation; either version 2 of the License, or
+    (at your option) any later version.
+
+    This program is distributed in the hope that it will be useful,
+    but WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+    GNU General Public License for more details.
+
+    You should have received a copy of the GNU General Public License along
+    with this program; if not, write to the Free Software Foundation, Inc.,
+    51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+
+Also add information on how to contact you by electronic and paper mail.
+
+If the program is interactive, make it output a short notice like this
+when it starts in an interactive mode:
+
+    Gnomovision version 69, Copyright (C) year name of author
+    Gnomovision comes with ABSOLUTELY NO WARRANTY; for details type `show w'.
+    This is free software, and you are welcome to redistribute it
+    under certain conditions; type `show c' for details.
+
+The hypothetical commands `show w' and `show c' should show the appropriate
+parts of the General Public License.  Of course, the commands you use may
+be called something other than `show w' and `show c'; they could even be
+mouse-clicks or menu items--whatever suits your program.
+
+You should also get your employer (if you work as a programmer) or your
+school, if any, to sign a "copyright disclaimer" for the program, if
+necessary.  Here is a sample; alter the names:
+
+  Yoyodyne, Inc., hereby disclaims all copyright interest in the program
+  `Gnomovision' (which makes passes at compilers) written by James Hacker.
+
+  <signature of Ty Coon>, 1 April 1989
+  Ty Coon, President of Vice
+
+This General Public License does not permit incorporating your program into
+proprietary programs.  If your program is a subroutine library, you may
+consider it more useful to permit linking proprietary applications with the
+library.  If this is what you want to do, use the GNU Lesser General
+Public License instead of this License.
diff --git a/COPYING-LGPLV3 b/COPYING-LGPLV3
new file mode 100644
index 000000000..65c5ca88a
--- /dev/null
+++ b/COPYING-LGPLV3
@@ -0,0 +1,165 @@
+                   GNU LESSER GENERAL PUBLIC LICENSE
+                       Version 3, 29 June 2007
+
+ Copyright (C) 2007 Free Software Foundation, Inc. <http://fsf.org/>
+ Everyone is permitted to copy and distribute verbatim copies
+ of this license document, but changing it is not allowed.
+
+
+  This version of the GNU Lesser General Public License incorporates
+the terms and conditions of version 3 of the GNU General Public
+License, supplemented by the additional permissions listed below.
+
+  0. Additional Definitions.
+
+  As used herein, "this License" refers to version 3 of the GNU Lesser
+General Public License, and the "GNU GPL" refers to version 3 of the GNU
+General Public License.
+
+  "The Library" refers to a covered work governed by this License,
+other than an Application or a Combined Work as defined below.
+
+  An "Application" is any work that makes use of an interface provided
+by the Library, but which is not otherwise based on the Library.
+Defining a subclass of a class defined by the Library is deemed a mode
+of using an interface provided by the Library.
+
+  A "Combined Work" is a work produced by combining or linking an
+Application with the Library.  The particular version of the Library
+with which the Combined Work was made is also called the "Linked
+Version".
+
+  The "Minimal Corresponding Source" for a Combined Work means the
+Corresponding Source for the Combined Work, excluding any source code
+for portions of the Combined Work that, considered in isolation, are
+based on the Application, and not on the Linked Version.
+
+  The "Corresponding Application Code" for a Combined Work means the
+object code and/or source code for the Application, including any data
+and utility programs needed for reproducing the Combined Work from the
+Application, but excluding the System Libraries of the Combined Work.
+
+  1. Exception to Section 3 of the GNU GPL.
+
+  You may convey a covered work under sections 3 and 4 of this License
+without being bound by section 3 of the GNU GPL.
+
+  2. Conveying Modified Versions.
+
+  If you modify a copy of the Library, and, in your modifications, a
+facility refers to a function or data to be supplied by an Application
+that uses the facility (other than as an argument passed when the
+facility is invoked), then you may convey a copy of the modified
+version:
+
+   a) under this License, provided that you make a good faith effort to
+   ensure that, in the event an Application does not supply the
+   function or data, the facility still operates, and performs
+   whatever part of its purpose remains meaningful, or
+
+   b) under the GNU GPL, with none of the additional permissions of
+   this License applicable to that copy.
+
+  3. Object Code Incorporating Material from Library Header Files.
+
+  The object code form of an Application may incorporate material from
+a header file that is part of the Library.  You may convey such object
+code under terms of your choice, provided that, if the incorporated
+material is not limited to numerical parameters, data structure
+layouts and accessors, or small macros, inline functions and templates
+(ten or fewer lines in length), you do both of the following:
+
+   a) Give prominent notice with each copy of the object code that the
+   Library is used in it and that the Library and its use are
+   covered by this License.
+
+   b) Accompany the object code with a copy of the GNU GPL and this license
+   document.
+
+  4. Combined Works.
+
+  You may convey a Combined Work under terms of your choice that,
+taken together, effectively do not restrict modification of the
+portions of the Library contained in the Combined Work and reverse
+engineering for debugging such modifications, if you also do each of
+the following:
+
+   a) Give prominent notice with each copy of the Combined Work that
+   the Library is used in it and that the Library and its use are
+   covered by this License.
+
+   b) Accompany the Combined Work with a copy of the GNU GPL and this license
+   document.
+
+   c) For a Combined Work that displays copyright notices during
+   execution, include the copyright notice for the Library among
+   these notices, as well as a reference directing the user to the
+   copies of the GNU GPL and this license document.
+
+   d) Do one of the following:
+
+       0) Convey the Minimal Corresponding Source under the terms of this
+       License, and the Corresponding Application Code in a form
+       suitable for, and under terms that permit, the user to
+       recombine or relink the Application with a modified version of
+       the Linked Version to produce a modified Combined Work, in the
+       manner specified by section 6 of the GNU GPL for conveying
+       Corresponding Source.
+
+       1) Use a suitable shared library mechanism for linking with the
+       Library.  A suitable mechanism is one that (a) uses at run time
+       a copy of the Library already present on the user's computer
+       system, and (b) will operate properly with a modified version
+       of the Library that is interface-compatible with the Linked
+       Version.
+
+   e) Provide Installation Information, but only if you would otherwise
+   be required to provide such information under section 6 of the
+   GNU GPL, and only to the extent that such information is
+   necessary to install and execute a modified version of the
+   Combined Work produced by recombining or relinking the
+   Application with a modified version of the Linked Version. (If
+   you use option 4d0, the Installation Information must accompany
+   the Minimal Corresponding Source and Corresponding Application
+   Code. If you use option 4d1, you must provide the Installation
+   Information in the manner specified by section 6 of the GNU GPL
+   for conveying Corresponding Source.)
+
+  5. Combined Libraries.
+
+  You may place library facilities that are a work based on the
+Library side by side in a single library together with other library
+facilities that are not Applications and are not covered by this
+License, and convey such a combined library under terms of your
+choice, if you do both of the following:
+
+   a) Accompany the combined library with a copy of the same work based
+   on the Library, uncombined with any other library facilities,
+   conveyed under the terms of this License.
+
+   b) Give prominent notice with the combined library that part of it
+   is a work based on the Library, and explaining where to find the
+   accompanying uncombined form of the same work.
+
+  6. Revised Versions of the GNU Lesser General Public License.
+
+  The Free Software Foundation may publish revised and/or new versions
+of the GNU Lesser General Public License from time to time. Such new
+versions will be similar in spirit to the present version, but may
+differ in detail to address new problems or concerns.
+
+  Each version is given a distinguishing version number. If the
+Library as you received it specifies that a certain numbered version
+of the GNU Lesser General Public License "or any later version"
+applies to it, you have the option of following the terms and
+conditions either of that published version or of any later version
+published by the Free Software Foundation. If the Library as you
+received it does not specify a version number of the GNU Lesser
+General Public License, you may choose any version of the GNU Lesser
+General Public License ever published by the Free Software Foundation.
+
+  If the Library as you received it specifies that a proxy can decide
+whether future versions of the GNU Lesser General Public License shall
+apply, that proxy's public statement of acceptance of any version is
+permanent authorization for you to choose that version for the
+Library.
diff --git a/Makefile.am b/Makefile.am
index 85e0a13f1..598ebb410 100644
--- a/Makefile.am
+++ b/Makefile.am
@@ -1,8 +1,18 @@
-EXTRA_DIST = autogen.sh COPYING INSTALL README AUTHORS THANKS NEWS glusterfs.spec
+EXTRA_DIST = autogen.sh \
+	COPYING-GPLV2 COPYING-LGPLV3 \
+	INSTALL README AUTHORS THANKS NEWS \
+	glusterfs.spec glusterfs-api.pc.in libgfchangelog.pc.in \
+	error-codes.json gf-error-codes.h.template \
+	gen-headers.py run-tests.sh \
+	$(shell find $(top_srcdir)/tests -type f -print)
 
-SUBDIRS = argp-standalone libglusterfs rpc xlators glusterfsd $(FUSERMOUNT_SUBDIR) doc extras cli
+SUBDIRS = argp-standalone libglusterfs rpc api xlators glusterfsd \
+	$(FUSERMOUNT_SUBDIR) doc extras cli @SYNCDAEMON_SUBDIR@
 
-CLEANFILES = 
+pkgconfigdir = @pkgconfigdir@
+pkgconfig_DATA = glusterfs-api.pc libgfchangelog.pc
+
+CLEANFILES =
 
 gitclean: distclean
 	find . -name Makefile.in -exec rm -f {} \;
@@ -11,4 +21,9 @@ gitclean: distclean
 	rm -fr autom4te.cache
 	rm -f missing aclocal.m4 config.h.in config.guess config.sub ltmain.sh install-sh configure depcomp
 	rm -fr argp-standalone/autom4te.cache
-	rm -f argp-standalone/aclocal.m4 argp-standalone/config.h.in argp-standalone/configure argp-standalone/depcomp argp-standalone/install-sh argp-standalone/missing
+	rm -f argp-standalone/aclocal.m4 argp-standalone/config.h.in
+	rm -f argp-standalone/configure argp-standalone/depcomp
+	rm -f argp-standalone/install-sh argp-standalone/missing
+
+dist-hook:
+	(cd $(srcdir) && git diff && echo ===== git log ==== && git log) > $(distdir)/ChangeLog
diff --git a/api/Makefile.am b/api/Makefile.am
new file mode 100644
index 000000000..f0ad1ee97
--- /dev/null
+++ b/api/Makefile.am
@@ -0,0 +1 @@
+SUBDIRS = src examples
diff --git a/api/examples/Makefile.am b/api/examples/Makefile.am
new file mode 100644
index 000000000..05f40ff53
--- /dev/null
+++ b/api/examples/Makefile.am
@@ -0,0 +1,6 @@
+EXTRA_PROGRAMS = glfsxmp
+glfsxmp_SOURCES = glfsxmp.c
+glfsxmp_CFLAGS = $(GLFS_CFLAGS) -Wall
+glfsxmp_LDADD = $(GLFS_LIBS) -lrt
+
+EXTRA_DIST = gfapi.py
diff --git a/api/examples/README b/api/examples/README
new file mode 100644
index 000000000..4d2b521f7
--- /dev/null
+++ b/api/examples/README
@@ -0,0 +1,36 @@
+This is an example application which uses libgfapi. It is
+a complete autotools based build system which demonstrates the
+required changes in configure.ac, Makefile.am etc to successfuly
+detect for and build an application against libgfapi.
+
+There are two approaches to building a libgfapi based application:
+
+1. In the presence of pkg-config in your build system.
+This is the recommended approach which is also used in this example.
+For this approach to work, you need to build glusterfs by passing
+--pkgconfigdir=/usr/lib64/pkgconfig (or the appropriate directory)
+in your distro. This already happens if you build RPMs with the
+glusterfs.spec provided in glusterfs.git. You will also need to
+install glusterfs-api RPM.
+
+2. In the absence of pkg-config in your build system.
+Make sure your LDFLAGS includes -L/path/to/lib where libgfapi.so is
+installed and -I/path/to/include/glusterfs where the 'api' directory
+containing the headers are available.
+
+glfsxmp.c
+=========
+
+glfsxmp.c is an example application which uses libgfapi
+
+Compilation Steps For glfsxmp.c
+===============================
+
+1. $./autogen.sh
+2. $./configure
+
+Note: Before running ./configure , as mentioned above, you need to
+      take care of #1 or #2 i.e. pkg-config path or LDFLAGS and
+      -I/<path> with correct values.
+
+3. $make glfsxmp
diff --git a/api/examples/autogen.sh b/api/examples/autogen.sh
new file mode 100755
index 000000000..1fee6be11
--- /dev/null
+++ b/api/examples/autogen.sh
@@ -0,0 +1,5 @@
+#!/bin/sh
+
+aclocal
+autoconf
+automake --foreign
diff --git a/api/examples/configure.ac b/api/examples/configure.ac
new file mode 100644
index 000000000..b80177a4e
--- /dev/null
+++ b/api/examples/configure.ac
@@ -0,0 +1,12 @@
+
+AC_INIT([glfs-test],[0.1],[gluster-devel@nongu.org])
+
+AM_INIT_AUTOMAKE
+
+AC_CONFIG_FILES([Makefile])
+
+AC_PROG_CC
+
+PKG_CHECK_MODULES([GLFS], [glusterfs-api >= 3])
+
+AC_OUTPUT
diff --git a/api/examples/gfapi.py b/api/examples/gfapi.py
new file mode 100755
index 000000000..3ac67f4d5
--- /dev/null
+++ b/api/examples/gfapi.py
@@ -0,0 +1,422 @@
+#!/usr/bin/python
+
+from ctypes import *
+from ctypes.util import find_library
+import os
+import sys
+import time
+import types
+
+# Looks like ctypes is having trouble with dependencies, so just force them to
+# load with RTLD_GLOBAL until I figure that out.
+glfs = CDLL(find_library("glusterfs"),RTLD_GLOBAL)
+xdr = CDLL(find_library("gfxdr"),RTLD_GLOBAL)
+api = CDLL(find_library("gfapi"),RTLD_GLOBAL)
+
+# Wow, the Linux kernel folks really play nasty games with this structure.  If
+# you look at the man page for stat(2) and then at this definition you'll note
+# two discrepancies.  First, we seem to have st_nlink and st_mode reversed.  In
+# fact that's exactly how they're defined *for 64-bit systems*; for 32-bit
+# they're in the man-page order.  Even uglier, the man page makes no mention of
+# the *nsec fields, but they are very much present and if they're not included
+# then we get memory corruption because libgfapi has a structure definition
+# that's longer than ours and they overwrite some random bit of memory after
+# the space we allocated.  Yes, that's all very disgusting, and I'm still not
+# sure this will really work on 32-bit because all of the field types are so
+# obfuscated behind macros and feature checks.
+class Stat (Structure):
+        _fields_ = [
+                ("st_dev",              c_ulong),
+                ("st_ino",              c_ulong),
+                ("st_nlink",            c_ulong),
+                ("st_mode",             c_uint),
+                ("st_uid",              c_uint),
+                ("st_gid",              c_uint),
+                ("st_rdev",             c_ulong),
+                ("st_size",             c_ulong),
+                ("st_blksize",          c_ulong),
+                ("st_blocks",           c_ulong),
+                ("st_atime",            c_ulong),
+                ("st_atimensec",        c_ulong),
+                ("st_mtime",            c_ulong),
+                ("st_mtimensec",        c_ulong),
+                ("st_ctime",            c_ulong),
+                ("st_ctimensec",        c_ulong),
+        ]
+api.glfs_creat.restype = c_void_p
+api.glfs_open.restype = c_void_p
+api.glfs_lstat.restype = c_int
+api.glfs_lstat.argtypes = [c_void_p, c_char_p, POINTER(Stat)]
+
+class Dirent (Structure):
+        _fields_ = [
+                ("d_ino",       c_ulong),
+                ("d_off",       c_ulong),
+                ("d_reclen",    c_ushort),
+                ("d_type",      c_char),
+                ("d_name",      c_char * 256),
+        ]
+api.glfs_opendir.restype = c_void_p
+api.glfs_readdir_r.restype = c_int
+api.glfs_readdir_r.argtypes = [c_void_p, POINTER(Dirent),
+                               POINTER(POINTER(Dirent))]
+
+# There's a bit of ctypes glitchiness around __del__ functions and module-level
+# variables.  If we unload the module while we still have references to File or
+# Volume objects, the module-level variables might have disappeared by the time
+# __del__ gets called.  Therefore the objects hold references which they
+# release when __del__ is done.  We only actually use the object-local values
+# in __del__; for clarity, we just use the simpler module-level form elsewhere.
+
+class File(object):
+
+        def __init__ (self, fd):
+                # Add a reference so the module-level variable "api" doesn't
+                # get yanked out from under us (see comment above File def'n).
+                self._api = api
+                self.fd = fd
+
+        def __del__ (self):
+                self._api.glfs_close(self.fd)
+                self._api = None
+
+        # File operations, in alphabetical order.
+
+        def fsync (self):
+                return api.glfs_fsync(self.fd)
+
+        def read (self, buflen, flags=0):
+                rbuf = create_string_buffer(buflen)
+                rc = api.glfs_read(self.fd,rbuf,buflen,flags)
+                if rc > 0:
+                        return rbuf.value[:rc]
+                else:
+                        return rc
+
+        def read_buffer (self, buf, flags=0):
+                return api.glfs_read(self.fd,buf,len(buf),flags)
+
+        def write (self, data, flags=0):
+                return api.glfs_write(self.fd,data,len(data),flags)
+
+        def fallocate (self, mode, offset, len):
+            return api.glfs_fallocate(self.fd, mode, offset, len)
+
+        def discard (self, offset, len):
+            return api.glfs_discard(self.fd, offset, len)
+
+
+class Dir(object):
+
+        def __init__ (self, fd):
+                # Add a reference so the module-level variable "api" doesn't
+                # get yanked out from under us (see comment above File def'n).
+                self._api = api
+                self.fd = fd
+                self.cursor = POINTER(Dirent)()
+
+        def __del__ (self):
+                self._api.glfs_closedir(self.fd)
+                self._api = None
+
+        def next (self):
+                entry = Dirent()
+                entry.d_reclen = 256
+                rc = api.glfs_readdir_r(self.fd,byref(entry),byref(self.cursor))
+                if (rc < 0) or (not self.cursor) or (not self.cursor.contents):
+                        return rc
+                return entry
+
+class Volume(object):
+
+        # Housekeeping functions.
+
+        def __init__ (self, host, volid, proto="tcp", port=24007):
+                # Add a reference so the module-level variable "api" doesn't
+                # get yanked out from under us (see comment above File def'n).
+                self._api = api
+                self.fs = api.glfs_new(volid)
+                api.glfs_set_volfile_server(self.fs,proto,host,port)
+
+        def __del__ (self):
+                self._api.glfs_fini(self.fs)
+                self._api = None
+
+        def set_logging (self, path, level):
+                api.glfs_set_logging(self.fs,path,level)
+
+        def mount (self):
+                api.glfs_init(self.fs)
+
+        # File operations, in alphabetical order.
+
+        def creat (self, path, flags, mode):
+                fd = api.glfs_creat(self.fs,path,flags,mode)
+                if not fd:
+                        return fd
+                return File(fd)
+
+        def getxattr (self, path, key, maxlen):
+                buf = create_string_buffer(maxlen)
+                rc = api.glfs_getxattr(self.fs,path,key,buf,maxlen)
+                if rc < 0:
+                        return rc
+                return buf.value[:rc]
+
+        def listxattr (self, path):
+                buf = create_string_buffer(512)
+                rc = api.glfs_listxattr(self.fs,path,buf,512)
+                if rc < 0:
+                        return rc
+                xattrs = []
+                # Parsing character by character is ugly, but it seems like the
+                # easiest way to deal with the "strings separated by NUL in one
+                # buffer" format.
+                i = 0
+                while i < rc:
+                        new_xa = buf.raw[i]
+                        i += 1
+                        while i < rc:
+                                next_char = buf.raw[i]
+                                i += 1
+                                if next_char == '\0':
+                                        xattrs.append(new_xa)
+                                        break
+                                new_xa += next_char
+                xattrs.sort()
+                return xattrs
+
+        def lstat (self, path):
+                x = Stat()
+                rc = api.glfs_lstat(self.fs,path,byref(x))
+                if rc >= 0:
+                        return x
+                else:
+                        return rc
+
+        def mkdir (self, path):
+                return api.glfs_mkdir(self.fs,path)
+
+        def open (self, path, flags):
+                fd = api.glfs_open(self.fs,path,flags)
+                if not fd:
+                        return fd
+                return File(fd)
+
+        def opendir (self, path):
+                fd = api.glfs_opendir(self.fs,path)
+                if not fd:
+                        return fd
+                return Dir(fd)
+
+        def rename (self, opath, npath):
+                return api.glfs_rename(self.fs,opath,npath)
+
+        def rmdir (self, path):
+                return api.glfs_rmdir(self.fs,path)
+
+        def setxattr (self, path, key, value, vlen):
+                return api.glfs_setxattr(self.fs,path,key,value,vlen,0)
+
+        def unlink (self, path):
+                return api.glfs_unlink(self.fs,path)
+
+if __name__ == "__main__":
+        def test_create_write (vol, path, data):
+                mypath = path + ".io"
+                fd = vol.creat(mypath,os.O_WRONLY|os.O_EXCL,0644)
+                if not fd:
+                        return False, "creat error"
+                rc = fd.write(data)
+                if rc != len(data):
+                        return False, "wrote %d/%d bytes" % (rc, len(data))
+                return True, "wrote %d bytes" % rc
+
+        # TBD: this test fails if we do create, open, write, read
+        def test_open_read (vol, path, data):
+                mypath = path + ".io"
+                fd = vol.open(mypath,os.O_RDONLY)
+                if not fd:
+                        return False, "open error"
+                dlen = len(data) * 2
+                buf = fd.read(dlen)
+                if type(buf) == types.IntType:
+                        return False, "read error %d" % buf
+                if len(buf) != len(data):
+                        return False, "read %d/%d bytes" % (len(buf), len(data))
+                return True, "read '%s'" % buf
+
+        def test_lstat (vol, path, data):
+                mypath = path + ".io"
+                sb = vol.lstat(mypath)
+                if type(sb) == types.IntType:
+                        return False, "lstat error %d" % sb
+                if sb.st_size != len(data):
+                        return False, "lstat size is %d, expected %d" % (
+                                sb.st_size, len(data))
+                return True, "lstat got correct size %d" % sb.st_size
+
+        def test_rename (vol, path, data):
+                opath = path + ".io"
+                npath = path + ".tmp"
+                rc = vol.rename(opath,npath)
+                if rc < 0:
+                        return False, "rename error %d" % rc
+                ofd = vol.open(opath,os.O_RDWR)
+                if isinstance(ofd,File):
+                        return False, "old path working after rename"
+                nfd = vol.open(npath,os.O_RDWR)
+                if isinstance(nfd,File):
+                        return False, "new path not working after rename"
+                return True, "rename worked"
+
+        def test_unlink (vol, path, data):
+                mypath = path + ".tmp"
+                rc = vol.unlink(mypath)
+                if rc < 0:
+                        return False, "unlink error %d" % fd
+                fd = vol.open(mypath,os.O_RDWR)
+                if isinstance(fd,File):
+                        return False, "path still usable after unlink"
+                return True, "unlink worked"
+
+        def test_mkdir (vol, path, data):
+                mypath = path + ".dir"
+                rc = vol.mkdir(mypath)
+                if rc < 0:
+                        return False, "mkdir error %d" % rc
+                return True, "mkdir worked"
+
+        def test_create_in_dir (vol, path, data):
+                mypath = path + ".dir/probe"
+                fd = vol.creat(mypath,os.O_RDWR,0644)
+                if not isinstance(fd,File):
+                        return False, "create (in dir) error"
+                return True, "create (in dir) worked"
+
+        def test_dir_listing (vol, path, data):
+                mypath = path + ".dir"
+                fd = vol.opendir(mypath)
+                if not isinstance(fd,Dir):
+                        return False, "opendir error %d" % fd
+                files = []
+                while True:
+                        ent = fd.next()
+                        if not isinstance(ent,Dirent):
+                                break
+                        name = ent.d_name[:ent.d_reclen]
+                        files.append(name)
+                if files != [".", "..", "probe"]:
+                        return False, "wrong directory contents"
+                return True, "directory listing worked"
+
+        def test_unlink_in_dir (vol, path, data):
+                mypath = path + ".dir/probe"
+                rc = vol.unlink(mypath)
+                if rc < 0:
+                        return False, "unlink (in dir) error %d" % rc
+                return True, "unlink (in dir) worked"
+
+        def test_rmdir (vol, path, data):
+                mypath = path + ".dir"
+                rc = vol.rmdir(mypath)
+                if rc < 0:
+                        return False, "rmdir error %d" % rc
+                sb = vol.lstat(mypath)
+                if not isinstance(sb,Stat):
+                        return False, "dir still there after rmdir"
+                return True, "rmdir worked"
+
+        def test_setxattr (vol, path, data):
+                mypath = path + ".xa"
+                fd = vol.creat(mypath,os.O_RDWR|os.O_EXCL,0644)
+                if not fd:
+                        return False, "creat (xattr test) error"
+                key1, key2 = "hello", "goodbye"
+                if vol.setxattr(mypath,"trusted.key1",key1,len(key1)) < 0:
+                        return False, "setxattr (key1) error"
+                if vol.setxattr(mypath,"trusted.key2",key2,len(key2)) < 0:
+                        return False, "setxattr (key2) error"
+                return True, "setxattr worked"
+
+        def test_getxattr (vol, path, data):
+                mypath = path + ".xa"
+                buf = vol.getxattr(mypath,"trusted.key1",32)
+                if type(buf) == types.IntType:
+                        return False, "getxattr error"
+                if buf != "hello":
+                        return False, "wrong getxattr value %s" % buf
+                return True, "getxattr worked"
+
+        def test_listxattr (vol, path, data):
+                mypath = path + ".xa"
+                xattrs = vol.listxattr(mypath)
+                if type(xattrs) == types.IntType:
+                        return False, "listxattr error"
+                if xattrs != ["trusted.key1","trusted.key2"]:
+                        return False, "wrong listxattr value %s" % repr(xattrs)
+                return True, "listxattr worked"
+
+        def test_fallocate (vol, path, data):
+                mypath = path + ".io"
+                fd = vol.creat(mypath,os.O_WRONLY|os.O_EXCL,0644)
+                if not fd:
+                        return False, "creat error"
+		rc = fd.fallocate(0, 0, 1024*1024)
+                if rc != 0:
+                        return False, "fallocate error"
+		rc = fd.discard(4096, 4096)
+		if rc != 0:
+			return False, "discard error"
+                return True, "fallocate/discard worked"
+
+        test_list = (
+                test_create_write,
+                test_open_read,
+                test_lstat,
+                test_rename,
+                test_unlink,
+                test_mkdir,
+                test_create_in_dir,
+                test_dir_listing,
+                test_unlink_in_dir,
+                test_rmdir,
+                test_setxattr,
+                test_getxattr,
+                test_listxattr,
+                test_fallocate,
+        )
+
+        ok_to_fail = (
+                # TBD: this fails opening the new file, even though the file
+                # did get renamed.  Looks like a gfapi bug, not ours.
+                (test_rename, "new path not working after rename"),
+                # TBD: similar, call returns error even though it worked
+                (test_rmdir, "dir still there after rmdir"),
+        )
+
+        volid, path = sys.argv[1:3]
+        data = "fubar"
+        vol = Volume("localhost",volid)
+        vol.set_logging("/dev/null",7)
+        #vol.set_logging("/dev/stderr",7)
+        vol.mount()
+
+        failures = 0
+        expected = 0
+        for t in test_list:
+                rc, msg = t(vol,path,data)
+                if rc:
+                        print "PASS: %s" % msg
+                else:
+                        print "FAIL: %s" % msg
+                        failures += 1
+                        for otf in ok_to_fail:
+                                if (t == otf[0]) and (msg == otf[1]):
+                                        print "  (skipping known failure)"
+                                        expected += 1
+                                        break # from the *inner* for loop
+                        else:
+                                break # from the *outer* for loop
+
+        print "%d failures (%d expected)" % (failures, expected)
diff --git a/api/examples/glfsxmp.c b/api/examples/glfsxmp.c
new file mode 100644
index 000000000..600d72fb5
--- /dev/null
+++ b/api/examples/glfsxmp.c
@@ -0,0 +1,1598 @@
+#include <stdio.h>
+#include <stdlib.h>
+#include <errno.h>
+#include "api/glfs.h"
+#include "api/glfs-handles.h"
+#include <string.h>
+#include <time.h>
+
+
+int
+test_dirops (glfs_t *fs)
+{
+        glfs_fd_t *fd = NULL;
+        char buf[512];
+        struct dirent *entry = NULL;
+
+        fd = glfs_opendir (fs, "/");
+        if (!fd) {
+                fprintf (stderr, "/: %s\n", strerror (errno));
+                return -1;
+        }
+
+        fprintf (stderr, "Entries:\n");
+        while (glfs_readdir_r (fd, (struct dirent *)buf, &entry), entry) {
+                fprintf (stderr, "%s: %lu\n", entry->d_name, glfs_telldir (fd));
+        }
+
+        glfs_closedir (fd);
+        return 0;
+}
+
+
+int
+test_xattr (glfs_t *fs)
+{
+        char *filename = "/filename2";
+        char buf[512];
+        char *ptr;
+        int ret;
+
+        ret = glfs_setxattr (fs, filename, "user.testkey", "testval", 8, 0);
+        fprintf (stderr, "setxattr(%s): %d (%s)\n", filename, ret,
+                 strerror (errno));
+
+        ret = glfs_setxattr (fs, filename, "user.testkey2", "testval", 8, 0);
+        fprintf (stderr, "setxattr(%s): %d (%s)\n", filename, ret,
+                 strerror (errno));
+
+        ret = glfs_listxattr (fs, filename, buf, 512);
+        fprintf (stderr, "listxattr(%s): %d (%s)\n", filename, ret,
+                 strerror (errno));
+        if (ret < 0)
+                return -1;
+
+        for (ptr = buf; ptr < buf + ret; ptr++) {
+                printf ("key=%s\n", ptr);
+                ptr += strlen (ptr);
+        }
+
+        return 0;
+}
+
+
+int
+test_chdir (glfs_t *fs)
+{
+        int ret = -1;
+        char *topdir = "/topdir";
+        char *linkdir = "/linkdir";
+        char *subdir = "./subdir";
+        char *respath = NULL;
+        char pathbuf[4096];
+
+        ret = glfs_mkdir (fs, topdir, 0755);
+        if (ret) {
+                fprintf (stderr, "mkdir(%s): %s\n", topdir, strerror (errno));
+                return -1;
+        }
+
+        respath = glfs_getcwd (fs, pathbuf, 4096);
+        fprintf (stdout, "getcwd() = %s\n", respath);
+
+        ret = glfs_symlink (fs, topdir, linkdir);
+        if (ret) {
+                fprintf (stderr, "symlink(%s, %s): %s\n", topdir, linkdir, strerror (errno));
+                return -1;
+        }
+
+        ret = glfs_chdir (fs, linkdir);
+        if (ret) {
+                fprintf (stderr, "chdir(%s): %s\n", linkdir, strerror (errno));
+                return -1;
+        }
+
+        respath = glfs_getcwd (fs, pathbuf, 4096);
+        fprintf (stdout, "getcwd() = %s\n", respath);
+
+        respath = glfs_realpath (fs, subdir, pathbuf);
+        if (respath) {
+                fprintf (stderr, "realpath(%s) worked unexpectedly: %s\n", subdir, respath);
+                return -1;
+        }
+
+        ret = glfs_mkdir (fs, subdir, 0755);
+        if (ret) {
+                fprintf (stderr, "mkdir(%s): %s\n", subdir, strerror (errno));
+                return -1;
+        }
+
+        respath = glfs_realpath (fs, subdir, pathbuf);
+        if (!respath) {
+                fprintf (stderr, "realpath(%s): %s\n", subdir, strerror (errno));
+        } else {
+                fprintf (stdout, "realpath(%s) = %s\n", subdir, respath);
+        }
+
+        ret = glfs_chdir (fs, subdir);
+        if (ret) {
+                fprintf (stderr, "chdir(%s): %s\n", subdir, strerror (errno));
+                return -1;
+        }
+
+        respath = glfs_getcwd (fs, pathbuf, 4096);
+        fprintf (stdout, "getcwd() = %s\n", respath);
+
+        respath = glfs_realpath (fs, "/linkdir/subdir", pathbuf);
+        if (!respath) {
+                fprintf (stderr, "realpath(/linkdir/subdir): %s\n", strerror (errno));
+        } else {
+                fprintf (stdout, "realpath(/linkdir/subdir) = %s\n", respath);
+        }
+
+        return 0;
+}
+
+#ifdef DEBUG
+static void
+peek_stat (struct stat *sb)
+{
+        printf ("Dumping stat information:\n");
+        printf ("File type:                ");
+
+        switch (sb->st_mode & S_IFMT) {
+                case S_IFBLK:  printf ("block device\n");            break;
+                case S_IFCHR:  printf ("character device\n");        break;
+                case S_IFDIR:  printf ("directory\n");               break;
+                case S_IFIFO:  printf ("FIFO/pipe\n");               break;
+                case S_IFLNK:  printf ("symlink\n");                 break;
+                case S_IFREG:  printf ("regular file\n");            break;
+                case S_IFSOCK: printf ("socket\n");                  break;
+                default:       printf ("unknown?\n");                break;
+        }
+
+        printf ("I-node number:            %ld\n", (long) sb->st_ino);
+
+        printf ("Mode:                     %lo (octal)\n",
+                (unsigned long) sb->st_mode);
+
+        printf ("Link count:               %ld\n", (long) sb->st_nlink);
+        printf ("Ownership:                UID=%ld   GID=%ld\n",
+                (long) sb->st_uid, (long) sb->st_gid);
+
+        printf ("Preferred I/O block size: %ld bytes\n",
+                (long) sb->st_blksize);
+        printf ("File size:                %lld bytes\n",
+                (long long) sb->st_size);
+        printf ("Blocks allocated:         %lld\n",
+                (long long) sb->st_blocks);
+
+        printf ("Last status change:       %s", ctime(&sb->st_ctime));
+        printf ("Last file access:         %s", ctime(&sb->st_atime));
+        printf ("Last file modification:   %s", ctime(&sb->st_mtime));
+
+        return;
+}
+
+static void
+peek_handle (unsigned char *glid)
+{
+        int i;
+
+        for (i = 0; i < GFAPI_HANDLE_LENGTH; i++)
+        {
+                printf (":%02x:", glid[i]);
+        }
+        printf ("\n");
+}
+#else /* DEBUG */
+static void
+peek_stat (struct stat *sb)
+{
+        return;
+}
+
+static void
+peek_handle (unsigned char *id)
+{
+        return;
+}
+#endif /* DEBUG */
+
+glfs_t    *fs = NULL;
+char      *full_parent_name = "/testdir", *parent_name = "testdir";
+
+void
+test_h_unlink (void)
+{
+        char               *my_dir = "unlinkdir";
+        char               *my_file = "file.txt";
+        char               *my_subdir = "dir1";
+        struct glfs_object *parent = NULL, *leaf = NULL, *dir = NULL,
+                           *subdir = NULL, *subleaf = NULL;
+        struct stat         sb;
+        int                 ret;
+
+        printf ("glfs_h_unlink tests: In Progress\n");
+
+        /* Prepare tests */
+        parent = glfs_h_lookupat (fs, NULL, full_parent_name, &sb);
+        if (parent == NULL) {
+                fprintf (stderr, "glfs_h_lookupat: error on lookup of %s: from (%p),%s\n",
+                         full_parent_name, NULL, strerror (errno));
+                printf ("glfs_h_lookupat tests: FAILED\n");
+                goto out;
+        }
+        peek_stat (&sb);
+
+        dir = glfs_h_mkdir (fs, parent, my_dir, 0644, &sb);
+        if (dir == NULL) {
+                fprintf (stderr, "glfs_h_mkdir: error creating %s: from (%p),%s\n",
+                         my_dir, parent, strerror (errno));
+                printf ("glfs_h_unlink tests: FAILED\n");
+                goto out;
+        }
+
+        leaf = glfs_h_creat (fs, dir, my_file, O_CREAT, 0644, &sb);
+        if (leaf == NULL) {
+                fprintf (stderr, "glfs_h_creat: error creating %s: from (%p),%s\n",
+                         my_file, dir, strerror (errno));
+                printf ("glfs_h_unlink tests: FAILED\n");
+                goto out;
+        }
+
+        subdir = glfs_h_mkdir (fs, dir, my_subdir, 0644, &sb);
+        if (subdir == NULL) {
+                fprintf (stderr, "glfs_h_mkdir: error creating %s: from (%p),%s\n",
+                         my_subdir, dir, strerror (errno));
+                printf ("glfs_h_unlink tests: FAILED\n");
+                goto out;
+        }
+
+        subleaf = glfs_h_creat (fs, subdir, my_file, O_CREAT, 0644, &sb);
+        if (subleaf == NULL) {
+                fprintf (stderr, "glfs_h_creat: error creating %s: from (%p),%s\n",
+                         my_file, subdir, strerror (errno));
+                printf ("glfs_h_unlink tests: FAILED\n");
+                goto out;
+        }
+
+        /* unlink non empty directory */
+        ret = glfs_h_unlink (fs, dir, my_subdir);
+        if ((ret && errno != ENOTEMPTY) || (ret == 0)) {
+                fprintf (stderr, "glfs_h_unlink: error unlinking %s: it is non empty: %s\n",
+                         my_subdir, strerror (errno));
+                printf ("glfs_h_unlink tests: FAILED\n");
+                goto out;
+        }
+
+        /* unlink regular file */
+        ret = glfs_h_unlink (fs, subdir, my_file);
+        if (ret) {
+                fprintf (stderr, "glfs_h_unlink: error unlinking %s: from (%p),%s\n",
+                         my_file, subdir, strerror (errno));
+                printf ("glfs_h_unlink tests: FAILED\n");
+                goto out;
+        }
+
+        /* unlink directory */
+        ret = glfs_h_unlink (fs, dir, my_subdir);
+        if (ret) {
+                fprintf (stderr, "glfs_h_unlink: error unlinking %s: from (%p),%s\n",
+                         my_subdir, dir, strerror (errno));
+                printf ("glfs_h_unlink tests: FAILED\n");
+                goto out;
+        }
+
+        /* unlink regular file */
+        ret = glfs_h_unlink (fs, dir, my_file);
+        if (ret) {
+                fprintf (stderr, "glfs_h_unlink: error unlinking %s: from (%p),%s\n",
+                         my_file, dir, strerror (errno));
+                printf ("glfs_h_unlink tests: FAILED\n");
+                goto out;
+        }
+
+        /* unlink non-existant regular file */
+        ret = glfs_h_unlink (fs, dir, my_file);
+        if ((ret && errno != ENOENT) || (ret == 0)) {
+                fprintf (stderr, "glfs_h_unlink: error unlinking non-existant %s: invalid errno ,%d, %s\n",
+                         my_file, ret, strerror (errno));
+                printf ("glfs_h_unlink tests: FAILED\n");
+                goto out;
+        }
+
+        /* unlink non-existant directory */
+        ret = glfs_h_unlink (fs, dir, my_subdir);
+        if ((ret && errno != ENOENT) || (ret == 0)) {
+                fprintf (stderr, "glfs_h_unlink: error unlinking non-existant %s:  invalid errno ,%d, %s\n",
+                         my_subdir, ret, strerror (errno));
+                printf ("glfs_h_unlink tests: FAILED\n");
+                goto out;
+        }
+
+        /* unlink directory */
+        ret = glfs_h_unlink (fs, parent, my_dir);
+        if (ret) {
+                fprintf (stderr, "glfs_h_unlink: error unlinking %s: from (%p),%s\n",
+                         my_dir, dir, strerror (errno));
+                printf ("glfs_h_unlink tests: FAILED\n");
+                goto out;
+        }
+
+        printf ("glfs_h_unlink tests: PASSED\n");
+
+out:
+        if (dir)
+                glfs_h_close (dir);
+        if (leaf)
+                glfs_h_close (leaf);
+        if (subdir)
+                glfs_h_close (subdir);
+        if (subleaf)
+                glfs_h_close (subleaf);
+        if (parent)
+                glfs_h_close (parent);
+
+        return;
+}
+
+void
+test_h_getsetattrs (void)
+{
+        char               *my_dir = "attrdir";
+        char               *my_file = "attrfile.txt";
+        struct glfs_object *parent = NULL, *leaf = NULL, *dir = NULL;
+        struct stat         sb, retsb;
+        int                 ret, valid;
+        struct timespec     timestamp;
+
+        printf("glfs_h_getattrs and setattrs tests: In Progress\n");
+
+        /* Prepare tests */
+        parent = glfs_h_lookupat (fs, NULL, full_parent_name, &sb);
+        if (parent == NULL) {
+                fprintf (stderr, "glfs_h_lookupat: error on lookup of %s: from (%p),%s\n",
+                         full_parent_name, NULL, strerror (errno));
+                printf ("glfs_h_lookupat tests: FAILED\n");
+                goto out;
+        }
+        peek_stat (&sb);
+
+        dir = glfs_h_mkdir (fs, parent, my_dir, 0644, &sb);
+        if (dir == NULL) {
+                fprintf (stderr, "glfs_h_mkdir: error creating %s: from (%p),%s\n",
+                         my_dir, parent, strerror (errno));
+                printf ("glfs_h_unlink tests: FAILED\n");
+                goto out;
+        }
+        peek_stat (&sb);
+
+        leaf = glfs_h_creat (fs, dir, my_file, O_CREAT, 0644, &sb);
+        if (leaf == NULL) {
+                fprintf (stderr, "glfs_h_creat: error creating %s: from (%p),%s\n",
+                         my_file, dir, strerror (errno));
+                printf ("glfs_h_unlink tests: FAILED\n");
+                goto out;
+        }
+        peek_stat (&sb);
+
+        ret = glfs_h_getattrs (fs, dir, &retsb);
+        if (ret != 0) {
+                fprintf (stderr, "glfs_h_getattrs: error %s: from (%p),%s\n",
+                         my_dir, dir, strerror (errno));
+                printf ("glfs_h_getattrs and setattrs tests: FAILED\n");
+                goto out;
+        }
+        peek_stat (&retsb);
+        /* TODO: Compare stat information */
+
+        retsb.st_mode = 00666;
+        retsb.st_uid = 1000;
+        retsb.st_gid = 1001;
+        ret = clock_gettime (CLOCK_REALTIME, &timestamp);
+        if(ret != 0) {
+                fprintf (stderr, "clock_gettime: error %s\n", strerror (errno));
+                printf ("glfs_h_getattrs and setattrs tests: FAILED\n");
+                goto out;
+        }
+        retsb.st_atim = timestamp;
+        retsb.st_mtim = timestamp;
+        valid = GFAPI_SET_ATTR_MODE | GFAPI_SET_ATTR_UID | GFAPI_SET_ATTR_GID |
+        GFAPI_SET_ATTR_ATIME | GFAPI_SET_ATTR_MTIME;
+        peek_stat (&retsb);
+
+        ret = glfs_h_setattrs (fs, dir, &retsb, valid);
+        if (ret != 0) {
+                fprintf (stderr, "glfs_h_setattrs: error %s: from (%p),%s\n",
+                         my_dir, dir, strerror (errno));
+                printf ("glfs_h_getattrs and setattrs tests: FAILED\n");
+                goto out;
+        }
+
+        memset(&retsb, 0, sizeof (struct stat));
+        ret = glfs_h_stat (fs, dir, &retsb);
+        if (ret != 0) {
+                fprintf (stderr, "glfs_h_stat: error %s: from (%p),%s\n",
+                         my_dir, dir, strerror (errno));
+                printf ("glfs_h_getattrs and setattrs tests: FAILED\n");
+                goto out;
+        }
+        peek_stat (&retsb);
+
+        printf ("glfs_h_getattrs and setattrs tests: PASSED\n");
+out:
+        if (parent)
+                glfs_h_close (parent);
+        if (leaf)
+                glfs_h_close (leaf);
+        if (dir)
+                glfs_h_close (dir);
+
+        return;
+}
+
+void
+test_h_truncate (void)
+{
+        char               *my_dir = "truncatedir";
+        char               *my_file = "file.txt";
+        struct glfs_object *root = NULL, *parent = NULL, *leaf = NULL;
+        struct stat         sb;
+        glfs_fd_t          *fd = NULL;
+        char                buf[32];
+        off_t               offset = 0;
+        int                 ret = 0;
+
+        printf("glfs_h_truncate tests: In Progress\n");
+
+        /* Prepare tests */
+        root = glfs_h_lookupat (fs, NULL, full_parent_name, &sb);
+        if (root == NULL) {
+                fprintf (stderr, "glfs_h_lookupat: error on lookup of %s: from (%p),%s\n",
+                         full_parent_name, NULL, strerror (errno));
+                printf ("glfs_h_truncate tests: FAILED\n");
+                goto out;
+        }
+        peek_stat (&sb);
+
+        parent = glfs_h_mkdir (fs, root, my_dir, 0644, &sb);
+        if (parent == NULL) {
+                fprintf (stderr, "glfs_h_mkdir: error creating %s: from (%p),%s\n",
+                         my_dir, root, strerror (errno));
+                printf ("glfs_h_truncate tests: FAILED\n");
+                goto out;
+        }
+        peek_stat (&sb);
+
+        leaf = glfs_h_creat (fs, parent, my_file, O_CREAT, 0644, &sb);
+        if (leaf == NULL) {
+                fprintf (stderr, "glfs_h_creat: error creating %s: from (%p),%s\n",
+                         my_file, parent, strerror (errno));
+                printf ("glfs_h_truncate tests: FAILED\n");
+                goto out;
+        }
+        peek_stat (&sb);
+
+        fd = glfs_h_open (fs, leaf, O_RDWR);
+        if (fd == NULL) {
+                fprintf (stderr, "glfs_h_open: error on open of %s: %s\n",
+                         my_file, strerror (errno));
+                printf ("glfs_h_truncate tests: FAILED\n");
+                goto out;
+        }
+
+        memcpy (buf, "abcdefghijklmnopqrstuvwxyz012345", 32);
+        ret = glfs_write (fd, buf, 32, 0);
+
+        /* run tests */
+        /* truncate lower */
+        offset = 30;
+        ret = glfs_h_truncate (fs, leaf, offset);
+        if (ret != 0) {
+                fprintf (stderr, "glfs_h_truncate: error creating %s: from (%p),%s\n",
+                         my_file, parent, strerror (errno));
+                printf ("glfs_h_truncate tests: FAILED\n");
+                goto out;
+        }
+        ret = glfs_h_getattrs (fs, leaf, &sb);
+        if (ret != 0) {
+                fprintf (stderr, "glfs_h_getattrs: error for %s (%p),%s\n",
+                         my_file, leaf, strerror (errno));
+                printf ("glfs_h_truncate tests: FAILED\n");
+                goto out;
+        }
+        if (sb.st_size != offset) {
+                fprintf (stderr, "glfs_h_truncate: post size mismatch\n");
+                printf ("glfs_h_truncate tests: FAILED\n");
+                goto out;
+        }
+
+        /* truncate higher */
+        offset = 32;
+        ret = glfs_h_truncate (fs, leaf, offset);
+        if (ret != 0) {
+                fprintf (stderr, "glfs_h_truncate: error creating %s: from (%p),%s\n",
+                         my_file, parent, strerror (errno));
+                printf ("glfs_h_truncate tests: FAILED\n");
+                goto out;
+        }
+        ret = glfs_h_getattrs (fs, leaf, &sb);
+        if (ret != 0) {
+                fprintf (stderr, "glfs_h_getattrs: error for %s (%p),%s\n",
+                         my_file, leaf, strerror (errno));
+                printf ("glfs_h_truncate tests: FAILED\n");
+                goto out;
+        }
+        if (sb.st_size != offset) {
+                fprintf (stderr, "glfs_h_truncate: post size mismatch\n");
+                printf ("glfs_h_truncate tests: FAILED\n");
+                goto out;
+        }
+
+        /* truncate equal */
+        offset = 30;
+        ret = glfs_h_truncate (fs, leaf, offset);
+        if (ret != 0) {
+                fprintf (stderr, "glfs_h_truncate: error creating %s: from (%p),%s\n",
+                         my_file, parent, strerror (errno));
+                printf ("glfs_h_truncate tests: FAILED\n");
+                goto out;
+        }
+        ret = glfs_h_getattrs (fs, leaf, &sb);
+        if (ret != 0) {
+                fprintf (stderr, "glfs_h_getattrs: error for %s (%p),%s\n",
+                         my_file, leaf, strerror (errno));
+                printf ("glfs_h_truncate tests: FAILED\n");
+                goto out;
+        }
+        if (sb.st_size != offset) {
+                fprintf (stderr, "glfs_h_truncate: post size mismatch\n");
+                printf ("glfs_h_truncate tests: FAILED\n");
+                goto out;
+        }
+
+        printf ("glfs_h_truncate tests: PASSED\n");
+out:
+        if (fd)
+                glfs_close (fd);
+        if (root)
+                glfs_h_close (root);
+        if (parent)
+                glfs_h_close (parent);
+        if (leaf)
+                glfs_h_close (leaf);
+
+        return;
+}
+
+void
+test_h_links (void)
+{
+        char               *my_dir = "linkdir";
+        char               *my_file = "file.txt";
+        char               *my_symlnk = "slnk.txt";
+        char               *my_lnk = "lnk.txt";
+        char               *linksrc_dir = "dir1";
+        char               *linktgt_dir = "dir2";
+        struct glfs_object *root = NULL, *parent = NULL, *leaf = NULL,
+                           *dirsrc = NULL, *dirtgt = NULL, *dleaf = NULL;
+        struct glfs_object *ln1 = NULL;
+        struct stat         sb;
+        int                 ret;
+        char               *buf = NULL;
+
+        printf("glfs_h_link(s) tests: In Progress\n");
+
+        /* Prepare tests */
+        root = glfs_h_lookupat (fs, NULL, full_parent_name, &sb);
+        if (root == NULL) {
+                fprintf (stderr, "glfs_h_lookupat: error on lookup of %s: from (%p),%s\n",
+                         full_parent_name, NULL, strerror (errno));
+                printf ("glfs_h_link(s) tests: FAILED\n");
+                goto out;
+        }
+        peek_stat (&sb);
+
+        parent = glfs_h_mkdir (fs, root, my_dir, 0644, &sb);
+        if (parent == NULL) {
+                fprintf (stderr, "glfs_h_mkdir: error creating %s: from (%p),%s\n",
+                         my_dir, root, strerror (errno));
+                printf ("glfs_h_link(s) tests: FAILED\n");
+                goto out;
+        }
+        peek_stat (&sb);
+
+        leaf = glfs_h_creat (fs, parent, my_file, O_CREAT, 0644, &sb);
+        if (leaf == NULL) {
+                fprintf (stderr, "glfs_h_creat: error creating %s: from (%p),%s\n",
+                         my_file, parent, strerror (errno));
+                printf ("glfs_h_link(s) tests: FAILED\n");
+                goto out;
+        }
+        peek_stat (&sb);
+
+        dirsrc = glfs_h_mkdir (fs, parent, linksrc_dir, 0644, &sb);
+        if (dirsrc == NULL) {
+                fprintf (stderr, "glfs_h_mkdir: error creating %s: from (%p),%s\n",
+                         linksrc_dir, parent, strerror (errno));
+                printf ("glfs_h_link(s) tests: FAILED\n");
+                goto out;
+        }
+        peek_stat (&sb);
+
+        dirtgt = glfs_h_mkdir (fs, parent, linktgt_dir, 0644, &sb);
+        if (dirtgt == NULL) {
+                fprintf (stderr, "glfs_h_mkdir: error creating %s: from (%p),%s\n",
+                         linktgt_dir, parent, strerror (errno));
+                printf ("glfs_h_link(s) tests: FAILED\n");
+                goto out;
+        }
+        peek_stat (&sb);
+
+        dleaf = glfs_h_creat (fs, dirsrc, my_file, O_CREAT, 0644, &sb);
+        if (dleaf == NULL) {
+                fprintf (stderr, "glfs_h_creat: error creating %s: from (%p),%s\n",
+                         my_file, dirsrc, strerror (errno));
+                printf ("glfs_h_link(s) tests: FAILED\n");
+                goto out;
+        }
+        peek_stat (&sb);
+
+        /* run tests */
+        /* sym link: /testdir/linkdir/file.txt to ./slnk.txt */
+        ln1 = glfs_h_symlink (fs, parent, my_symlnk, "./file.txt", &sb);
+        if (ln1 == NULL) {
+                fprintf (stderr, "glfs_h_symlink: error creating %s: from (%p),%s\n",
+                         my_symlnk, parent, strerror (errno));
+                printf ("glfs_h_link(s) tests: FAILED\n");
+                goto out;
+        }
+        peek_stat (&sb);
+
+        buf = calloc (1024, sizeof(char));
+        if (buf == NULL) {
+                fprintf (stderr, "Error allocating memory\n");
+                printf ("glfs_h_link(s) tests: FAILED\n");
+                goto out;
+        }
+
+        ret = glfs_h_readlink (fs, ln1, buf, 1024);
+        if (ret <= 0) {
+                fprintf (stderr, "glfs_h_readlink: error reading %s: from (%p),%s\n",
+                         my_symlnk, ln1, strerror (errno));
+                printf ("glfs_h_link(s) tests: FAILED\n");
+                goto out;
+        }
+        if (!(strncmp (buf, my_symlnk, strlen (my_symlnk)))) {
+                fprintf (stderr, "glfs_h_readlink: error mismatch in link name: actual %s: retrieved %s\n",
+                         my_symlnk, buf);
+                printf ("glfs_h_link(s) tests: FAILED\n");
+                goto out;
+        }
+
+        /* link: /testdir/linkdir/file.txt to ./lnk.txt */
+        ret = glfs_h_link (fs, leaf, parent, my_lnk);
+        if (ret != 0) {
+                fprintf (stderr, "glfs_h_link: error creating %s: from (%p),%s\n",
+                         my_lnk, parent, strerror (errno));
+                printf ("glfs_h_link(s) tests: FAILED\n");
+                goto out;
+        }
+        /* TODO: Should write content to a file and read from the link */
+
+        /* link: /testdir/linkdir/dir1/file.txt to ../dir2/slnk.txt */
+        ret = glfs_h_link (fs, dleaf, dirtgt, my_lnk);
+        if (ret != 0) {
+                fprintf (stderr, "glfs_h_link: error creating %s: from (%p),%s\n",
+                         my_lnk, dirtgt, strerror (errno));
+                printf ("glfs_h_link(s) tests: FAILED\n");
+                goto out;
+        }
+        /* TODO: Should write content to a file and read from the link */
+
+        printf ("glfs_h_link(s) tests: PASSED\n");
+
+out:
+        if (root)
+                glfs_h_close (root);
+        if (parent)
+                glfs_h_close (parent);
+        if (leaf)
+                glfs_h_close (leaf);
+        if (dirsrc)
+                glfs_h_close (dirsrc);
+        if (dirtgt)
+                glfs_h_close (dirtgt);
+        if (dleaf)
+                glfs_h_close (dleaf);
+        if (ln1)
+                glfs_h_close (ln1);
+        if (buf)
+                free (buf);
+
+        return;
+}
+
+void
+test_h_rename (void)
+{
+        char               *my_dir = "renamedir";
+        char               *my_file = "file.txt";
+        char               *src_dir = "dir1";
+        char               *tgt_dir = "dir2";
+        struct glfs_object *root = NULL, *parent = NULL, *leaf = NULL,
+                           *dirsrc = NULL, *dirtgt = NULL, *dleaf = NULL;
+        struct stat         sb;
+        int                 ret;
+
+        printf("glfs_h_rename tests: In Progress\n");
+
+        /* Prepare tests */
+        root = glfs_h_lookupat (fs, NULL, full_parent_name, &sb);
+        if (root == NULL) {
+                fprintf (stderr, "glfs_h_lookupat: error on lookup of %s: from (%p),%s\n",
+                         full_parent_name, NULL, strerror (errno));
+                printf ("glfs_h_rename tests: FAILED\n");
+                goto out;
+        }
+        peek_stat (&sb);
+
+        parent = glfs_h_mkdir (fs, root, my_dir, 0644, &sb);
+        if (parent == NULL) {
+                fprintf (stderr, "glfs_h_mkdir: error creating %s: from (%p),%s\n",
+                         my_dir, root, strerror (errno));
+                printf ("glfs_h_rename tests: FAILED\n");
+                goto out;
+        }
+        peek_stat (&sb);
+
+        leaf = glfs_h_creat (fs, parent, my_file, O_CREAT, 0644, &sb);
+        if (leaf == NULL) {
+                fprintf (stderr, "glfs_h_creat: error creating %s: from (%p),%s\n",
+                         my_file, parent, strerror (errno));
+                printf ("glfs_h_rename tests: FAILED\n");
+                goto out;
+        }
+        peek_stat (&sb);
+
+        dirsrc = glfs_h_mkdir (fs, parent, src_dir, 0644, &sb);
+        if (dirsrc == NULL) {
+                fprintf (stderr, "glfs_h_mkdir: error creating %s: from (%p),%s\n",
+                         src_dir, parent, strerror (errno));
+                printf ("glfs_h_rename tests: FAILED\n");
+                goto out;
+        }
+        peek_stat (&sb);
+
+        dirtgt = glfs_h_mkdir (fs, parent, tgt_dir, 0644, &sb);
+        if (dirtgt == NULL) {
+                fprintf (stderr, "glfs_h_mkdir: error creating %s: from (%p),%s\n",
+                         tgt_dir, parent, strerror (errno));
+                printf ("glfs_h_rename tests: FAILED\n");
+                goto out;
+        }
+        peek_stat (&sb);
+
+        dleaf = glfs_h_creat (fs, dirsrc, my_file, O_CREAT, 0644, &sb);
+        if (dleaf == NULL) {
+                fprintf (stderr, "glfs_h_creat: error creating %s: from (%p),%s\n",
+                         my_file, dirsrc, strerror (errno));
+                printf ("glfs_h_rename tests: FAILED\n");
+                goto out;
+        }
+        peek_stat (&sb);
+
+        /* run tests */
+        /* Rename file.txt -> file1.txt */
+        ret = glfs_h_rename (fs, parent, "file.txt", parent, "file1.txt");
+        if (ret != 0) {
+                fprintf (stderr, "glfs_h_rename: error renaming %s to %s (%s)\n",
+                         "file.txt", "file1.txt", strerror (errno));
+                printf ("glfs_h_rename tests: FAILED\n");
+                goto out;
+        }
+
+        /* rename dir1/file.txt -> file.txt */
+        ret = glfs_h_rename (fs, dirsrc, "file.txt", parent, "file.txt");
+        if (ret != 0) {
+                fprintf (stderr, "glfs_h_rename: error renaming %s/%s to %s (%s)\n",
+                         src_dir, "file.txt", "file.txt", strerror (errno));
+                printf ("glfs_h_rename tests: FAILED\n");
+                goto out;
+        }
+
+        /* rename file1.txt -> file.txt (exists) */
+        ret = glfs_h_rename (fs, parent, "file1.txt", parent, "file.txt");
+        if (ret != 0) {
+                fprintf (stderr, "glfs_h_rename: error renaming %s to %s (%s)\n",
+                         "file.txt", "file.txt", strerror (errno));
+                printf ("glfs_h_rename tests: FAILED\n");
+                goto out;
+        }
+
+        /* rename dir1 -> dir3 */
+        ret = glfs_h_rename (fs, parent, "dir1", parent, "dir3");
+        if (ret != 0) {
+                fprintf (stderr, "glfs_h_rename: error renaming %s to %s (%s)\n",
+                         "dir1", "dir3", strerror (errno));
+                printf ("glfs_h_rename tests: FAILED\n");
+                goto out;
+        }
+
+        /* rename dir2 ->dir3 (exists) */
+        ret = glfs_h_rename (fs, parent, "dir2", parent, "dir3");
+        if (ret != 0) {
+                fprintf (stderr, "glfs_h_rename: error renaming %s to %s (%s)\n",
+                         "dir2", "dir3", strerror (errno));
+                printf ("glfs_h_rename tests: FAILED\n");
+                goto out;
+        }
+
+        /* rename file.txt -> dir3 (fail) */
+        ret = glfs_h_rename (fs, parent, "file.txt", parent, "dir3");
+        if (ret == 0) {
+                fprintf (stderr, "glfs_h_rename: NO error renaming %s to %s (%s)\n",
+                         "file.txt", "dir3", strerror (errno));
+                printf ("glfs_h_rename tests: FAILED\n");
+                goto out;
+        }
+
+        /* rename dir3 -> file.txt (fail) */
+        ret = glfs_h_rename (fs, parent, "dir3", parent, "file.txt");
+        if (ret == 0) {
+                fprintf (stderr, "glfs_h_rename: NO error renaming %s to %s (%s)\n",
+                         "dir3", "file.txt", strerror (errno));
+                printf ("glfs_h_rename tests: FAILED\n");
+                goto out;
+        }
+
+        printf ("glfs_h_rename tests: PASSED\n");
+
+out:
+        if (root)
+                glfs_h_close (root);
+        if (parent)
+                glfs_h_close (parent);
+        if (leaf)
+                glfs_h_close (leaf);
+        if (dirsrc)
+                glfs_h_close (dirsrc);
+        if (dirtgt)
+                glfs_h_close (dirtgt);
+        if (dleaf)
+                glfs_h_close (dleaf);
+
+        return;
+}
+
+void
+assimilatetime (struct timespec *ts, struct timespec ts_st,
+                struct timespec ts_ed)
+{
+        if ((ts_ed.tv_nsec - ts_st.tv_nsec) < 0) {
+                ts->tv_sec += ts_ed.tv_sec - ts_st.tv_sec - 1;
+                ts->tv_nsec += 1000000000 + ts_ed.tv_nsec - ts_st.tv_nsec;
+        } else {
+                ts->tv_sec += ts_ed.tv_sec - ts_st.tv_sec;
+                ts->tv_nsec += ts_ed.tv_nsec - ts_st.tv_nsec;
+        }
+
+        if (ts->tv_nsec > 1000000000) {
+                ts->tv_nsec = ts->tv_nsec - 1000000000;
+                ts->tv_sec += 1;
+        }
+
+        return;
+}
+
+#define MAX_FILES_CREATE 10
+#define MAXPATHNAME      512
+void
+test_h_performance (void)
+{
+        char               *my_dir = "perftest",
+                           *full_dir_path="/testdir/perftest";
+        char               *my_file = "file_", my_file_name[MAXPATHNAME];
+        struct glfs_object *parent = NULL, *leaf = NULL, *dir = NULL;
+        struct stat         sb;
+        int                 ret, i;
+        struct glfs_fd     *fd;
+        struct timespec     c_ts = {0, 0}, c_ts_st, c_ts_ed;
+        struct timespec     o_ts = {0, 0}, o_ts_st, o_ts_ed;
+
+        printf("glfs_h_performance tests: In Progress\n");
+
+        /* Prepare tests */
+        parent = glfs_h_lookupat (fs, NULL, full_parent_name, &sb);
+        if (parent == NULL) {
+                fprintf (stderr, "glfs_h_lookupat: error on lookup of %s: from (%p),%s\n",
+                         full_parent_name, NULL, strerror (errno));
+                printf ("glfs_h_performance tests: FAILED\n");
+                goto out;
+        }
+
+        dir = glfs_h_mkdir (fs, parent, my_dir, 0644, &sb);
+        if (dir == NULL) {
+                fprintf (stderr, "glfs_h_mkdir: error creating %s: from (%p),%s\n",
+                         my_dir, parent, strerror (errno));
+                printf ("glfs_h_performance tests: FAILED\n");
+                goto out;
+        }
+        peek_stat (&sb);
+
+        /* create performance */
+        ret = clock_gettime (CLOCK_REALTIME, &o_ts_st);
+        if(ret != 0) {
+                fprintf (stderr, "clock_gettime: error %s\n", strerror (errno));
+                printf ("glfs_h_getattrs and setattrs tests: FAILED\n");
+                goto out;
+        }
+
+        for (i = 0; i < MAX_FILES_CREATE; i++) {
+                sprintf (my_file_name, "%s%d", my_file, i);
+
+                ret = clock_gettime (CLOCK_REALTIME, &c_ts_st);
+                if(ret != 0) {
+                        fprintf (stderr, "clock_gettime: error %s\n",
+                                 strerror (errno));
+                        printf ("glfs_h_getattrs and setattrs tests: FAILED\n");
+                        goto out;
+                }
+
+                leaf = glfs_h_lookupat (fs, dir, my_file_name, &sb);
+                if (leaf != NULL) {
+                        fprintf (stderr, "glfs_h_lookup: exists %s\n",
+                                 my_file_name);
+                        printf ("glfs_h_performance tests: FAILED\n");
+                        goto out;
+                }
+
+                leaf = glfs_h_creat (fs, dir, my_file_name, O_CREAT, 0644, &sb);
+                if (leaf == NULL) {
+                        fprintf (stderr, "glfs_h_creat: error creating %s: from (%p),%s\n",
+                                 my_file, dir, strerror (errno));
+                        printf ("glfs_h_performance tests: FAILED\n");
+                        goto out;
+                }
+
+                ret = clock_gettime (CLOCK_REALTIME, &c_ts_ed);
+                if(ret != 0) {
+                        fprintf (stderr, "clock_gettime: error %s\n",
+                                 strerror (errno));
+                        printf ("glfs_h_getattrs and setattrs tests: FAILED\n");
+                        goto out;
+                }
+
+                assimilatetime (&c_ts, c_ts_st, c_ts_ed);
+                glfs_h_close (leaf); leaf = NULL;
+        }
+
+        ret = clock_gettime (CLOCK_REALTIME, &o_ts_ed);
+        if(ret != 0) {
+                fprintf (stderr, "clock_gettime: error %s\n", strerror (errno));
+                printf ("glfs_h_getattrs and setattrs tests: FAILED\n");
+                goto out;
+        }
+
+        assimilatetime (&o_ts, o_ts_st, o_ts_ed);
+
+        printf ("Creation performance (handle based):\n\t# empty files:%d\n",
+                MAX_FILES_CREATE);
+        printf ("\tOverall time:\n\t\tSecs:%ld\n\t\tnSecs:%ld\n",
+                o_ts.tv_sec, o_ts.tv_nsec);
+        printf ("\tcreate call time time:\n\t\tSecs:%ld\n\t\tnSecs:%ld\n",
+                c_ts.tv_sec, c_ts.tv_nsec);
+
+        /* create using path */
+        c_ts.tv_sec = o_ts.tv_sec = 0;
+        c_ts.tv_nsec = o_ts.tv_nsec = 0;
+
+        sprintf (my_file_name, "%s1", full_dir_path);
+        ret = glfs_mkdir (fs, my_file_name, 0644);
+        if (ret != 0) {
+                fprintf (stderr, "glfs_mkdir: error creating %s: from (%p),%s\n",
+                         my_dir, parent, strerror (errno));
+                printf ("glfs_h_performance tests: FAILED\n");
+                goto out;
+        }
+        peek_stat (&sb);
+
+        ret = clock_gettime (CLOCK_REALTIME, &o_ts_st);
+        if(ret != 0) {
+                fprintf (stderr, "clock_gettime: error %s\n", strerror (errno));
+                printf ("glfs_h_getattrs and setattrs tests: FAILED\n");
+                goto out;
+        }
+
+        for (i = 0; i < MAX_FILES_CREATE; i++) {
+                sprintf (my_file_name, "%s1/%sn%d", full_dir_path, my_file, i);
+
+                ret = clock_gettime (CLOCK_REALTIME, &c_ts_st);
+                if(ret != 0) {
+                        fprintf (stderr, "clock_gettime: error %s\n",
+                                 strerror (errno));
+                        printf ("glfs_h_getattrs and setattrs tests: FAILED\n");
+                        goto out;
+                }
+
+                ret = glfs_stat (fs, my_file_name, &sb);
+                if (ret == 0) {
+                        fprintf (stderr, "glfs_stat: exists %s\n",
+                                 my_file_name);
+                        printf ("glfs_h_performance tests: FAILED\n");
+                        goto out;
+                }
+
+                fd = glfs_creat (fs, my_file_name, O_CREAT, 0644);
+                if (fd == NULL) {
+                        fprintf (stderr, "glfs_creat: error creating %s: from (%p),%s\n",
+                                 my_file, dir, strerror (errno));
+                        printf ("glfs_h_performance tests: FAILED\n");
+                        goto out;
+                }
+
+                ret = clock_gettime (CLOCK_REALTIME, &c_ts_ed);
+                if(ret != 0) {
+                        fprintf (stderr, "clock_gettime: error %s\n",
+                                 strerror (errno));
+                        printf ("glfs_h_getattrs and setattrs tests: FAILED\n");
+                        goto out;
+                }
+
+                assimilatetime (&c_ts, c_ts_st, c_ts_ed);
+                glfs_close (fd);
+        }
+
+        ret = clock_gettime (CLOCK_REALTIME, &o_ts_ed);
+        if(ret != 0) {
+                fprintf (stderr, "clock_gettime: error %s\n", strerror (errno));
+                printf ("glfs_h_getattrs and setattrs tests: FAILED\n");
+                goto out;
+        }
+
+        assimilatetime (&o_ts, o_ts_st, o_ts_ed);
+
+        printf ("Creation performance (path based):\n\t# empty files:%d\n",
+                MAX_FILES_CREATE);
+        printf ("\tOverall time:\n\t\tSecs:%ld\n\t\tnSecs:%ld\n",
+                o_ts.tv_sec, o_ts.tv_nsec);
+        printf ("\tcreate call time time:\n\t\tSecs:%ld\n\t\tnSecs:%ld\n",
+                c_ts.tv_sec, c_ts.tv_nsec);
+out:
+        return;
+}
+
+int
+test_handleops (int argc, char *argv[])
+{
+        int                 ret = 0;
+        glfs_fd_t          *fd = NULL;
+        struct stat         sb = {0, };
+        struct glfs_object *root = NULL, *parent = NULL, *leaf = NULL,
+                           *tmp = NULL;
+        char                readbuf[32], writebuf[32];
+        unsigned char       leaf_handle[GFAPI_HANDLE_LENGTH];
+
+        char *full_leaf_name = "/testdir/testfile.txt",
+             *leaf_name = "testfile.txt",
+             *relative_leaf_name = "testdir/testfile.txt";
+        char *leaf_name1 = "testfile1.txt";
+        char *full_newparent_name = "/testdir/dir1",
+             *newparent_name = "dir1";
+        char *full_newnod_name = "/testdir/nod1",
+             *newnod_name = "nod1";
+
+        /* Initialize test area */
+        ret = glfs_mkdir (fs, full_parent_name, 0644);
+        if (ret != 0 && errno != EEXIST) {
+                fprintf (stderr, "%s: (%p) %s\n", full_parent_name, fd,
+                        strerror (errno));
+                printf ("Test initialization failed on volume %s\n", argv[1]);
+                goto out;
+        }
+        else if (ret != 0) {
+                printf ("Found test directory %s to be existing\n",
+                        full_parent_name);
+                printf ("Cleanup test directory and restart tests\n");
+                goto out;
+        }
+
+        fd = glfs_creat (fs, full_leaf_name, O_CREAT, 0644);
+        if (fd == NULL) {
+                fprintf (stderr, "%s: (%p) %s\n", full_leaf_name, fd,
+                        strerror (errno));
+                printf ("Test initialization failed on volume %s\n", argv[1]);
+                goto out;
+        }
+        glfs_close (fd);
+
+        printf ("Initialized the test area, within volume %s\n", argv[1]);
+
+        /* Handle based APIs test area */
+
+        /* glfs_lookupat test */
+        printf ("glfs_h_lookupat tests: In Progress\n");
+        /* start at root of the volume */
+        root = glfs_h_lookupat (fs, NULL, "/", &sb);
+        if (root == NULL) {
+                fprintf (stderr, "glfs_h_lookupat: error on lookup of %s: from (%p),%s\n",
+                         "/", NULL, strerror (errno));
+                printf ("glfs_h_lookupat tests: FAILED\n");
+                goto out;
+        }
+        peek_stat (&sb);
+
+        /* lookup a parent within root */
+        parent = glfs_h_lookupat (fs, root, parent_name, &sb);
+        if (parent == NULL) {
+                fprintf (stderr, "glfs_h_lookupat: error on lookup of %s: from (%p),%s\n",
+                         parent_name, root, strerror (errno));
+                printf ("glfs_h_lookupat tests: FAILED\n");
+                goto out;
+        }
+        peek_stat (&sb);
+
+        /* lookup a leaf/child within the parent */
+        leaf = glfs_h_lookupat (fs, parent, leaf_name, &sb);
+        if (leaf == NULL) {
+                fprintf (stderr, "glfs_h_lookupat: error on lookup of %s: from (%p),%s\n",
+                         leaf_name, parent, strerror (errno));
+                printf ("glfs_h_lookupat tests: FAILED\n");
+                goto out;
+        }
+        peek_stat (&sb);
+
+        /* reset */
+        glfs_h_close (root); root = NULL;
+        glfs_h_close (leaf); leaf = NULL;
+        glfs_h_close (parent); parent = NULL;
+
+        /* check absolute paths */
+        root = glfs_h_lookupat (fs, NULL, "/", &sb);
+        if (root == NULL) {
+                fprintf (stderr, "glfs_h_lookupat: error on lookup of %s: from (%p),%s\n",
+                         "/", NULL, strerror (errno));
+                printf ("glfs_h_lookupat tests: FAILED\n");
+                goto out;
+        }
+        peek_stat (&sb);
+
+        parent = glfs_h_lookupat (fs, NULL, full_parent_name, &sb);
+        if (parent == NULL) {
+                fprintf (stderr, "glfs_h_lookupat: error on lookup of %s: from (%p),%s\n",
+                         full_parent_name, root, strerror (errno));
+                printf ("glfs_h_lookupat tests: FAILED\n");
+                goto out;
+        }
+        peek_stat (&sb);
+
+        leaf = glfs_h_lookupat (fs, NULL, full_leaf_name, &sb);
+        if (leaf == NULL) {
+                fprintf (stderr, "glfs_h_lookupat: error on lookup of %s: from (%p),%s\n",
+                         full_leaf_name, parent, strerror (errno));
+                printf ("glfs_h_lookupat tests: FAILED\n");
+                goto out;
+        }
+        peek_stat (&sb);
+
+        /* reset */
+        glfs_h_close (leaf); leaf = NULL;
+
+        /* check multiple component paths */
+        leaf = glfs_h_lookupat (fs, root, relative_leaf_name, &sb);
+        if (leaf == NULL) {
+                fprintf (stderr, "glfs_h_lookupat: error on lookup of %s: from (%p),%s\n",
+                         relative_leaf_name, parent, strerror (errno));
+                goto out;
+        }
+        peek_stat (&sb);
+
+        /* reset */
+        glfs_h_close (root); root = NULL;
+        glfs_h_close (parent); parent = NULL;
+
+        /* check symlinks in path */
+
+        /* TODO: -ve test cases */
+        /* parent invalid
+        * path invalid
+        * path does not exist after some components
+        * no parent, but relative path
+        * parent and full path? -ve?
+        */
+
+        printf ("glfs_h_lookupat tests: PASSED\n");
+
+        /* glfs_openat test */
+        printf ("glfs_h_open tests: In Progress\n");
+        fd = glfs_h_open (fs, leaf, O_RDWR);
+        if (fd == NULL) {
+                fprintf (stderr, "glfs_h_open: error on open of %s: %s\n",
+                         full_leaf_name, strerror (errno));
+                printf ("glfs_h_open tests: FAILED\n");
+                goto out;
+        }
+
+        /* test read/write based on fd */
+        memcpy (writebuf, "abcdefghijklmnopqrstuvwxyz012345", 32);
+        ret = glfs_write (fd, writebuf, 32, 0);
+
+        glfs_lseek (fd, 0, SEEK_SET);
+
+        ret = glfs_read (fd, readbuf, 32, 0);
+        if (memcmp (readbuf, writebuf, 32)) {
+                printf ("Failed to read what I wrote: %s %s\n", readbuf,
+                        writebuf);
+                glfs_close (fd);
+                printf ("glfs_h_open tests: FAILED\n");
+                goto out;
+        }
+
+        glfs_h_close (leaf); leaf = NULL;
+        glfs_close (fd);
+
+        printf ("glfs_h_open tests: PASSED\n");
+
+        /* Create tests */
+        printf ("glfs_h_creat tests: In Progress\n");
+        parent = glfs_h_lookupat (fs, NULL, full_parent_name, &sb);
+        if (parent == NULL) {
+                fprintf (stderr, "glfs_h_lookupat: error on lookup of %s: from (%p),%s\n",
+                         full_parent_name, root, strerror (errno));
+                printf ("glfs_h_creat tests: FAILED\n");
+                goto out;
+        }
+        peek_stat (&sb);
+
+        leaf = glfs_h_creat (fs, parent, leaf_name1, O_CREAT, 0644, &sb);
+        if (leaf == NULL) {
+                fprintf (stderr, "glfs_h_creat: error on create of %s: from (%p),%s\n",
+                         leaf_name1, parent, strerror (errno));
+                printf ("glfs_h_creat tests: FAILED\n");
+                goto out;
+        }
+        peek_stat (&sb);
+
+        glfs_h_close (leaf); leaf = NULL;
+
+        leaf = glfs_h_creat (fs, parent, leaf_name1, O_CREAT | O_EXCL, 0644,
+                            &sb);
+        if (leaf != NULL || errno != EEXIST) {
+                fprintf (stderr, "glfs_h_creat: existing file, leaf = (%p), errno = %s\n",
+                        leaf, strerror (errno));
+                printf ("glfs_h_creat tests: FAILED\n");
+                if (leaf != NULL) {
+                        glfs_h_close (leaf); leaf = NULL;
+                }
+        }
+
+        tmp = glfs_h_creat (fs, root, parent_name, O_CREAT, 0644, &sb);
+        if (tmp != NULL || !(errno == EISDIR || errno == EINVAL)) {
+                fprintf (stderr, "glfs_h_creat: dir create, tmp = (%p), errno = %s\n",
+                        leaf, strerror (errno));
+                printf ("glfs_h_creat tests: FAILED\n");
+                if (tmp != NULL) {
+                        glfs_h_close (tmp); tmp = NULL;
+                }
+        }
+
+        /* TODO: Other combinations and -ve cases as applicable */
+        printf ("glfs_h_creat tests: PASSED\n");
+
+        /* extract handle and create from handle test */
+        printf ("glfs_h_extract_handle and glfs_h_create_from_handle tests: In Progress\n");
+        /* TODO: Change the lookup to creat below for a GIFD recovery falure,
+         * that needs to be fixed */
+        leaf = glfs_h_lookupat (fs, parent, leaf_name1, &sb);
+        if (leaf == NULL) {
+                fprintf (stderr, "glfs_h_lookupat: error on lookup of %s: from (%p),%s\n",
+                         leaf_name1, parent, strerror (errno));
+                printf ("glfs_h_extract_handle tests: FAILED\n");
+                goto out;
+        }
+        peek_stat (&sb);
+
+        ret = glfs_h_extract_handle (leaf, leaf_handle,
+                                             GFAPI_HANDLE_LENGTH);
+        if (ret < 0) {
+                fprintf (stderr, "glfs_h_extract_handle: error extracting handle of %s: %s\n",
+                         full_leaf_name, strerror (errno));
+                printf ("glfs_h_extract_handle tests: FAILED\n");
+                goto out;
+        }
+        peek_handle (leaf_handle);
+
+        glfs_h_close (leaf); leaf = NULL;
+
+        leaf = glfs_h_create_from_handle (fs, leaf_handle, GFAPI_HANDLE_LENGTH,
+                                          &sb);
+        if (leaf == NULL) {
+                fprintf (stderr, "glfs_h_create_from_handle: error on create of %s: from (%p),%s\n",
+                         leaf_name1, leaf_handle, strerror (errno));
+                printf ("glfs_h_create_from_handle tests: FAILED\n");
+                goto out;
+        }
+        peek_stat (&sb);
+
+        fd = glfs_h_open (fs, leaf, O_RDWR);
+        if (fd == NULL) {
+                fprintf (stderr, "glfs_h_open: error on open of %s: %s\n",
+                         full_leaf_name, strerror (errno));
+                printf ("glfs_h_create_from_handle tests: FAILED\n");
+                goto out;
+        }
+
+        /* test read/write based on fd */
+        memcpy (writebuf, "abcdefghijklmnopqrstuvwxyz012345", 32);
+        ret = glfs_write (fd, writebuf, 32, 0);
+
+        glfs_lseek (fd, 0, SEEK_SET);
+
+        ret = glfs_read (fd, readbuf, 32, 0);
+        if (memcmp (readbuf, writebuf, 32)) {
+                printf ("Failed to read what I wrote: %s %s\n", writebuf,
+                        writebuf);
+                printf ("glfs_h_create_from_handle tests: FAILED\n");
+                glfs_close (fd);
+                goto out;
+        }
+
+        glfs_close (fd);
+        glfs_h_close (leaf); leaf = NULL;
+        glfs_h_close (parent); parent = NULL;
+
+        printf ("glfs_h_extract_handle and glfs_h_create_from_handle tests: PASSED\n");
+
+        /* Mkdir tests */
+        printf ("glfs_h_mkdir tests: In Progress\n");
+
+        ret = glfs_rmdir (fs, full_newparent_name);
+        if (ret && errno != ENOENT) {
+                fprintf (stderr, "glfs_rmdir: Failed for %s: %s\n",
+                         full_newparent_name, strerror (errno));
+                printf ("glfs_h_mkdir tests: FAILED\n");
+                goto out;
+        }
+
+        parent = glfs_h_lookupat (fs, NULL, full_parent_name, &sb);
+        if (parent == NULL) {
+                fprintf (stderr, "glfs_h_lookupat: error on lookup of %s: from (%p),%s\n",
+                         full_parent_name, root, strerror (errno));
+                printf ("glfs_h_mkdir tests: FAILED\n");
+                goto out;
+        }
+        peek_stat (&sb);
+
+        leaf = glfs_h_mkdir (fs, parent, newparent_name, 0644, &sb);
+        if (leaf == NULL) {
+                fprintf (stderr, "glfs_h_mkdir: error on mkdir of %s: from (%p),%s\n",
+                         newparent_name, parent, strerror (errno));
+                printf ("glfs_h_mkdir tests: FAILED\n");
+                goto out;
+        }
+        peek_stat (&sb);
+
+        glfs_h_close (leaf); leaf = NULL;
+
+        leaf = glfs_h_mkdir (fs, parent, newparent_name, 0644, &sb);
+        if (leaf != NULL || errno != EEXIST) {
+                fprintf (stderr, "glfs_h_mkdir: existing directory, leaf = (%p), errno = %s\n",
+                         leaf, strerror (errno));
+                printf ("glfs_h_mkdir tests: FAILED\n");
+                if (leaf != NULL) {
+                        glfs_h_close (leaf); leaf = NULL;
+                }
+        }
+
+        glfs_h_close (parent); parent = NULL;
+
+        printf ("glfs_h_mkdir tests: PASSED\n");
+
+        /* Mknod tests */
+        printf ("glfs_h_mknod tests: In Progress\n");
+        ret = glfs_unlink (fs, full_newnod_name);
+        if (ret && errno != ENOENT) {
+                fprintf (stderr, "glfs_unlink: Failed for %s: %s\n",
+                         full_newnod_name, strerror (errno));
+                printf ("glfs_h_mknod tests: FAILED\n");
+                goto out;
+        }
+
+        parent = glfs_h_lookupat (fs, NULL, full_parent_name, &sb);
+        if (parent == NULL) {
+                fprintf (stderr, "glfs_h_lookupat: error on lookup of %s: from (%p),%s\n",
+                         full_parent_name, root, strerror (errno));
+                printf ("glfs_h_mknod tests: FAILED\n");
+                goto out;
+        }
+        peek_stat (&sb);
+
+        leaf = glfs_h_mknod (fs, parent, newnod_name, S_IFIFO, 0, &sb);
+        if (leaf == NULL) {
+                fprintf (stderr, "glfs_h_mkdir: error on mkdir of %s: from (%p),%s\n",
+                         newnod_name, parent, strerror (errno));
+                printf ("glfs_h_mknod tests: FAILED\n");
+                goto out;
+        }
+        peek_stat (&sb);
+
+        /* TODO: creat op on a FIFO node hangs, need to check and fix
+        tmp = glfs_h_creat (fs, parent, newnod_name, O_CREAT, 0644, &sb);
+        if (tmp != NULL || errno != EINVAL) {
+                fprintf (stderr, "glfs_h_creat: node create, tmp = (%p), errno = %s\n",
+                        tmp, strerror (errno));
+                printf ("glfs_h_creat/mknod tests: FAILED\n");
+                if (tmp != NULL) {
+                        glfs_h_close(tmp); tmp = NULL;
+                }
+        } */
+
+        glfs_h_close (leaf); leaf = NULL;
+
+        leaf = glfs_h_mknod (fs, parent, newnod_name, 0644, 0, &sb);
+        if (leaf != NULL || errno != EEXIST) {
+                fprintf (stderr, "glfs_h_mknod: existing node, leaf = (%p), errno = %s\n",
+                         leaf, strerror (errno));
+                printf ("glfs_h_mknod tests: FAILED\n");
+                if (leaf != NULL) {
+                        glfs_h_close (leaf); leaf = NULL;
+                }
+        }
+
+        glfs_h_close (parent); parent = NULL;
+
+        printf ("glfs_h_mknod tests: PASSED\n");
+
+        /* unlink tests */
+        test_h_unlink ();
+
+        /* TODO: opendir tests */
+
+        /* getattr tests */
+        test_h_getsetattrs ();
+
+        /* TODO: setattr tests */
+
+        /* truncate tests */
+        test_h_truncate();
+
+        /* link tests */
+        test_h_links ();
+
+        /* rename tests */
+        test_h_rename ();
+
+        /* performance tests */
+        test_h_performance ();
+
+        /* END: New APIs test area */
+
+out:
+        /* Cleanup glfs handles */
+        if (root)
+                glfs_h_close (root);
+        if (parent)
+                glfs_h_close (parent);
+        if (leaf)
+                glfs_h_close (leaf);
+
+        return ret;
+}
+
+int
+main (int argc, char *argv[])
+{
+        glfs_t    *fs2 = NULL;
+        int        ret = 0;
+        glfs_fd_t *fd = NULL;
+        glfs_fd_t *fd2 = NULL;
+        struct stat sb = {0, };
+        char       readbuf[32];
+        char       writebuf[32];
+
+        char      *filename = "/filename2";
+
+        if (argc != 3) {
+                printf ("Expect following args\n\t%s <volname> <hostname>\n", argv[0]);
+                return -1;
+        }
+
+        fs = glfs_new (argv[1]);
+        if (!fs) {
+                fprintf (stderr, "glfs_new: returned NULL\n");
+                return 1;
+        }
+
+//      ret = glfs_set_volfile (fs, "/tmp/posix.vol");
+
+        ret = glfs_set_volfile_server (fs, "tcp", argv[2], 24007);
+
+//      ret = glfs_set_volfile_server (fs, "unix", "/tmp/gluster.sock", 0);
+
+        ret = glfs_set_logging (fs, "/dev/stderr", 7);
+
+        ret = glfs_init (fs);
+
+        fprintf (stderr, "glfs_init: returned %d\n", ret);
+
+        sleep (2);
+
+        fs2 = glfs_new (argv[1]);
+        if (!fs2) {
+                fprintf (stderr, "glfs_new: returned NULL\n");
+                return 1;
+        }
+
+
+//      ret = glfs_set_volfile (fs2, "/tmp/posix.vol");
+
+        ret = glfs_set_volfile_server (fs2, "tcp", argv[2], 24007);
+
+        ret = glfs_set_logging (fs2, "/dev/stderr", 7);
+
+        ret = glfs_init (fs2);
+
+        fprintf (stderr, "glfs_init: returned %d\n", ret);
+
+        ret = glfs_lstat (fs, filename, &sb);
+        fprintf (stderr, "%s: (%d) %s\n", filename, ret, strerror (errno));
+
+        fd = glfs_creat (fs, filename, O_RDWR, 0644);
+        fprintf (stderr, "%s: (%p) %s\n", filename, fd, strerror (errno));
+
+        fd2 = glfs_open (fs2, filename, O_RDWR);
+        fprintf (stderr, "%s: (%p) %s\n", filename, fd, strerror (errno));
+
+        sprintf (writebuf, "hi there\n");
+        ret = glfs_write (fd, writebuf, 32, 0);
+
+        glfs_lseek (fd2, 0, SEEK_SET);
+
+        ret = glfs_read (fd2, readbuf, 32, 0);
+
+        printf ("read %d, %s", ret, readbuf);
+
+        glfs_close (fd);
+        glfs_close (fd2);
+
+        filename = "/filename3";
+        ret = glfs_mknod (fs, filename, S_IFIFO, 0);
+        fprintf (stderr, "%s: (%d) %s\n", filename, ret, strerror (errno));
+
+        ret = glfs_lstat (fs, filename, &sb);
+        fprintf (stderr, "%s: (%d) %s\n", filename, ret, strerror (errno));
+
+
+        ret = glfs_rename (fs, filename, "/filename4");
+        fprintf (stderr, "rename(%s): (%d) %s\n", filename, ret,
+                 strerror (errno));
+
+        ret = glfs_unlink (fs, "/filename4");
+        fprintf (stderr, "unlink(%s): (%d) %s\n", "/filename4", ret,
+                 strerror (errno));
+
+        filename = "/dirname2";
+        ret = glfs_mkdir (fs, filename, 0);
+        fprintf (stderr, "%s: (%d) %s\n", filename, ret, strerror (errno));
+
+        ret = glfs_lstat (fs, filename, &sb);
+        fprintf (stderr, "lstat(%s): (%d) %s\n", filename, ret, strerror (errno));
+
+        ret = glfs_rmdir (fs, filename);
+        fprintf (stderr, "rmdir(%s): (%d) %s\n", filename, ret, strerror (errno));
+
+        test_dirops (fs);
+
+        test_xattr (fs);
+
+        test_chdir (fs);
+
+        test_handleops (argc, argv);
+        // done
+
+        glfs_fini (fs);
+        glfs_fini (fs2);
+
+        return ret;
+}
diff --git a/api/examples/setup.py.in b/api/examples/setup.py.in
new file mode 100644
index 000000000..44b738094
--- /dev/null
+++ b/api/examples/setup.py.in
@@ -0,0 +1,29 @@
+from distutils.core import setup
+
+# generate a __init__.py for the package namespace
+fo = open('__init__.py', 'w')
+fo.write('__version__ = "@PACKAGE_VERSION@"\n')
+fo.close()
+
+DESC = """GlusterFS is a clustered file-system capable of scaling to
+several petabytes. It aggregates various storage bricks over Infiniband
+RDMA or TCP/IP interconnect into one large parallel network file system.
+GlusterFS is one of the most sophisticated file systems in terms of
+features and extensibility.  It borrows a powerful concept called
+Translators from GNU Hurd kernel. Much of the code in GlusterFS is in
+user space and easily manageable.
+
+This package contains the Python interface to the libgfapi library."""
+
+setup(
+    name='glusterfs-api',
+    version='@PACKAGE_VERSION@',
+    description='Python client library for the GlusterFS libgfapi',
+    long_description=DESC,
+    author='Gluster Community',
+    author_email='gluster-devel@nongnu.org',
+    license='LGPLv3',
+    url='http://gluster.org/',
+    package_dir={'gluster':''},
+    packages=['gluster']
+)
diff --git a/api/src/Makefile.am b/api/src/Makefile.am
new file mode 100644
index 000000000..7c5df3e20
--- /dev/null
+++ b/api/src/Makefile.am
@@ -0,0 +1,36 @@
+lib_LTLIBRARIES = libgfapi.la
+noinst_HEADERS = glfs-mem-types.h glfs-internal.h
+libgfapi_HEADERS = glfs.h glfs-handles.h
+libgfapidir = $(includedir)/glusterfs/api
+
+libgfapi_la_SOURCES = glfs.c glfs-mgmt.c glfs-fops.c glfs-resolve.c \
+	glfs-handleops.c
+libgfapi_la_LIBADD = $(top_builddir)/libglusterfs/src/libglusterfs.la \
+	$(top_builddir)/rpc/rpc-lib/src/libgfrpc.la \
+	$(top_builddir)/rpc/xdr/src/libgfxdr.la \
+	$(GF_LDADD)
+
+libgfapi_la_CPPFLAGS = $(GF_CPPFLAGS) -D__USE_FILE_OFFSET64 \
+	-I$(top_srcdir)/libglusterfs/src \
+	-I$(top_srcdir)/rpc/rpc-lib/src \
+	-I$(top_srcdir)/rpc/xdr/src
+
+
+xlator_LTLIBRARIES = api.la
+xlatordir = $(libdir)/glusterfs/$(PACKAGE_VERSION)/xlator/mount
+# workaround for broken parallel install support in automake with LTLIBRARIES
+# http://debbugs.gnu.org/cgi/bugreport.cgi?bug=7328
+install_xlatorLTLIBRARIES = install-xlatorLTLIBRARIES
+$(install_xlatorLTLIBRARIES): install-libLTLIBRARIES
+
+api_la_SOURCES = glfs-master.c
+api_la_DEPENDENCIES = libgfapi.la
+api_la_LDFLAGS = -module -avoid-version
+api_la_LIBADD = $(top_builddir)/libglusterfs/src/libglusterfs.la \
+	$(top_builddir)/rpc/rpc-lib/src/libgfrpc.la \
+	$(top_builddir)/rpc/xdr/src/libgfxdr.la \
+	$(top_builddir)/api/src/libgfapi.la
+
+AM_CPPFLAGS = $(GF_CPPFLAGS) -I$(top_srcdir)/libglusterfs/src
+
+AM_CFLAGS = -Wall $(GF_CFLAGS)
diff --git a/api/src/glfs-fops.c b/api/src/glfs-fops.c
new file mode 100644
index 000000000..10bb7d38b
--- /dev/null
+++ b/api/src/glfs-fops.c
@@ -0,0 +1,3252 @@
+/*
+  Copyright (c) 2012 Red Hat, Inc. <http://www.redhat.com>
+  This file is part of GlusterFS.
+
+  This file is licensed to you under your choice of the GNU Lesser
+  General Public License, version 3 or any later version (LGPLv3 or
+  later), or the GNU General Public License, version 2 (GPLv2), in all
+  cases as published by the Free Software Foundation.
+*/
+
+
+#include "glfs-internal.h"
+#include "glfs-mem-types.h"
+#include "syncop.h"
+#include "glfs.h"
+#include <limits.h>
+
+#ifdef NAME_MAX
+#define GF_NAME_MAX NAME_MAX
+#else
+#define GF_NAME_MAX 255
+#endif
+
+#define READDIRBUF_SIZE (sizeof(struct dirent) + GF_NAME_MAX + 1)
+
+int
+glfs_loc_link (loc_t *loc, struct iatt *iatt)
+{
+	int ret = -1;
+	inode_t *linked_inode = NULL;
+
+	if (!loc->inode) {
+		errno = EINVAL;
+		return -1;
+	}
+
+	linked_inode = inode_link (loc->inode, loc->parent, loc->name, iatt);
+	if (linked_inode) {
+		inode_lookup (linked_inode);
+		inode_unref (linked_inode);
+		ret = 0;
+	} else {
+		ret = -1;
+		errno = ENOMEM;
+	}
+
+	return ret;
+}
+
+
+void
+glfs_iatt_to_stat (struct glfs *fs, struct iatt *iatt, struct stat *stat)
+{
+	iatt_to_stat (iatt, stat);
+	stat->st_dev = fs->dev_id;
+}
+
+
+int
+glfs_loc_unlink (loc_t *loc)
+{
+	inode_unlink (loc->inode, loc->parent, loc->name);
+
+	return 0;
+}
+
+
+struct glfs_fd *
+glfs_open (struct glfs *fs, const char *path, int flags)
+{
+	int              ret = -1;
+	struct glfs_fd  *glfd = NULL;
+	xlator_t        *subvol = NULL;
+	loc_t            loc = {0, };
+	struct iatt      iatt = {0, };
+	int              reval = 0;
+
+	__glfs_entry_fs (fs);
+
+	subvol = glfs_active_subvol (fs);
+	if (!subvol) {
+		ret = -1;
+		errno = EIO;
+		goto out;
+	}
+
+	glfd = glfs_fd_new (fs);
+	if (!glfd)
+		goto out;
+
+retry:
+	ret = glfs_resolve (fs, subvol, path, &loc, &iatt, reval);
+
+	ESTALE_RETRY (ret, errno, reval, &loc, retry);
+
+	if (ret)
+		goto out;
+
+	if (IA_ISDIR (iatt.ia_type)) {
+		ret = -1;
+		errno = EISDIR;
+		goto out;
+	}
+
+	if (!IA_ISREG (iatt.ia_type)) {
+		ret = -1;
+		errno = EINVAL;
+		goto out;
+	}
+
+	if (glfd->fd) {
+		/* Retry. Safe to touch glfd->fd as we
+		   still have not glfs_fd_bind() yet.
+		*/
+		fd_unref (glfd->fd);
+		glfd->fd = NULL;
+	}
+
+	glfd->fd = fd_create (loc.inode, getpid());
+	if (!glfd->fd) {
+		ret = -1;
+		errno = ENOMEM;
+		goto out;
+	}
+
+	ret = syncop_open (subvol, &loc, flags, glfd->fd);
+
+	ESTALE_RETRY (ret, errno, reval, &loc, retry);
+out:
+	loc_wipe (&loc);
+
+	if (ret && glfd) {
+		glfs_fd_destroy (glfd);
+		glfd = NULL;
+	} else if (glfd) {
+                glfd->fd->flags = flags;
+		fd_bind (glfd->fd);
+		glfs_fd_bind (glfd);
+	}
+
+	glfs_subvol_done (fs, subvol);
+
+	return glfd;
+}
+
+
+int
+glfs_close (struct glfs_fd *glfd)
+{
+	xlator_t  *subvol = NULL;
+	int        ret = -1;
+	fd_t      *fd = NULL;
+	struct glfs *fs = NULL;
+
+	__glfs_entry_fd (glfd);
+
+	subvol = glfs_active_subvol (glfd->fs);
+        if (!subvol) {
+                ret = -1;
+                errno = EIO;
+                goto out;
+        }
+
+	fd = glfs_resolve_fd (glfd->fs, subvol, glfd);
+	if (!fd) {
+		ret = -1;
+		errno = EBADFD;
+		goto out;
+	}
+
+	ret = syncop_flush (subvol, fd);
+out:
+	fs = glfd->fs;
+	glfs_fd_destroy (glfd);
+
+	if (fd)
+		fd_unref (fd);
+
+	glfs_subvol_done (fs, subvol);
+
+	return ret;
+}
+
+
+int
+glfs_lstat (struct glfs *fs, const char *path, struct stat *stat)
+{
+	int              ret = -1;
+	xlator_t        *subvol = NULL;
+	loc_t            loc = {0, };
+	struct iatt      iatt = {0, };
+	int              reval = 0;
+
+	__glfs_entry_fs (fs);
+
+	subvol = glfs_active_subvol (fs);
+	if (!subvol) {
+		ret = -1;
+		errno = EIO;
+		goto out;
+	}
+retry:
+	ret = glfs_lresolve (fs, subvol, path, &loc, &iatt, reval);
+
+	ESTALE_RETRY (ret, errno, reval, &loc, retry);
+
+	if (ret == 0 && stat)
+		glfs_iatt_to_stat (fs, &iatt, stat);
+out:
+	loc_wipe (&loc);
+
+	glfs_subvol_done (fs, subvol);
+
+	return ret;
+}
+
+
+int
+glfs_stat (struct glfs *fs, const char *path, struct stat *stat)
+{
+	int              ret = -1;
+	xlator_t        *subvol = NULL;
+	loc_t            loc = {0, };
+	struct iatt      iatt = {0, };
+	int              reval = 0;
+
+	__glfs_entry_fs (fs);
+
+	subvol = glfs_active_subvol (fs);
+	if (!subvol) {
+		ret = -1;
+		errno = EIO;
+		goto out;
+	}
+retry:
+	ret = glfs_resolve (fs, subvol, path, &loc, &iatt, reval);
+
+	ESTALE_RETRY (ret, errno, reval, &loc, retry);
+
+	if (ret == 0 && stat)
+		glfs_iatt_to_stat (fs, &iatt, stat);
+out:
+	loc_wipe (&loc);
+
+	glfs_subvol_done (fs, subvol);
+
+	return ret;
+}
+
+
+int
+glfs_fstat (struct glfs_fd *glfd, struct stat *stat)
+{
+	int              ret = -1;
+	xlator_t        *subvol = NULL;
+	struct iatt      iatt = {0, };
+	fd_t            *fd = NULL;
+
+	__glfs_entry_fd (glfd);
+
+	subvol = glfs_active_subvol (glfd->fs);
+	if (!subvol) {
+		ret = -1;
+		errno = EIO;
+		goto out;
+	}
+
+	fd = glfs_resolve_fd (glfd->fs, subvol, glfd);
+	if (!fd) {
+		ret = -1;
+		errno = EBADFD;
+		goto out;
+	}
+
+	ret = syncop_fstat (subvol, fd, &iatt);
+
+	if (ret == 0 && stat)
+		glfs_iatt_to_stat (glfd->fs, &iatt, stat);
+out:
+	if (fd)
+		fd_unref (fd);
+
+	glfs_subvol_done (glfd->fs, subvol);
+
+	return ret;
+}
+
+
+struct glfs_fd *
+glfs_creat (struct glfs *fs, const char *path, int flags, mode_t mode)
+{
+	int              ret = -1;
+	struct glfs_fd  *glfd = NULL;
+	xlator_t        *subvol = NULL;
+	loc_t            loc = {0, };
+	struct iatt      iatt = {0, };
+	uuid_t           gfid;
+	dict_t          *xattr_req = NULL;
+	int              reval = 0;
+
+	__glfs_entry_fs (fs);
+
+	subvol = glfs_active_subvol (fs);
+	if (!subvol) {
+		ret = -1;
+		errno = EIO;
+		goto out;
+	}
+
+	xattr_req = dict_new ();
+	if (!xattr_req) {
+		ret = -1;
+		errno = ENOMEM;
+		goto out;
+	}
+
+	uuid_generate (gfid);
+	ret = dict_set_static_bin (xattr_req, "gfid-req", gfid, 16);
+	if (ret) {
+		ret = -1;
+		errno = ENOMEM;
+		goto out;
+	}
+
+	glfd = glfs_fd_new (fs);
+	if (!glfd)
+		goto out;
+
+	/* This must be glfs_resolve() and NOT glfs_lresolve().
+	   That is because open("name", O_CREAT) where "name"
+	   is a danging symlink must create the dangling
+	   destinataion.
+	*/
+retry:
+	ret = glfs_resolve (fs, subvol, path, &loc, &iatt, reval);
+
+	ESTALE_RETRY (ret, errno, reval, &loc, retry);
+
+	if (ret == -1 && errno != ENOENT)
+		/* Any other type of error is fatal */
+		goto out;
+
+	if (ret == -1 && errno == ENOENT && !loc.parent)
+		/* The parent directory or an ancestor even
+		   higher does not exist
+		*/
+		goto out;
+
+	if (loc.inode) {
+		if (flags & O_EXCL) {
+			ret = -1;
+			errno = EEXIST;
+			goto out;
+		}
+
+		if (IA_ISDIR (iatt.ia_type)) {
+			ret = -1;
+			errno = EISDIR;
+			goto out;
+		}
+
+		if (!IA_ISREG (iatt.ia_type)) {
+			ret = -1;
+			errno = EINVAL;
+			goto out;
+		}
+	}
+
+	if (ret == -1 && errno == ENOENT) {
+		loc.inode = inode_new (loc.parent->table);
+		if (!loc.inode) {
+			ret = -1;
+			errno = ENOMEM;
+			goto out;
+		}
+	}
+
+	if (glfd->fd) {
+		/* Retry. Safe to touch glfd->fd as we
+		   still have not glfs_fd_bind() yet.
+		*/
+		fd_unref (glfd->fd);
+		glfd->fd = NULL;
+	}
+
+	glfd->fd = fd_create (loc.inode, getpid());
+	if (!glfd->fd) {
+		ret = -1;
+		errno = ENOMEM;
+		goto out;
+	}
+
+	if (ret == 0) {
+		ret = syncop_open (subvol, &loc, flags, glfd->fd);
+	} else {
+		ret = syncop_create (subvol, &loc, flags, mode, glfd->fd,
+				     xattr_req, &iatt);
+	}
+
+	ESTALE_RETRY (ret, errno, reval, &loc, retry);
+
+	if (ret == 0)
+		ret = glfs_loc_link (&loc, &iatt);
+out:
+	loc_wipe (&loc);
+
+	if (xattr_req)
+		dict_unref (xattr_req);
+
+	if (ret && glfd) {
+		glfs_fd_destroy (glfd);
+		glfd = NULL;
+	} else if (glfd) {
+                glfd->fd->flags = flags;
+		fd_bind (glfd->fd);
+		glfs_fd_bind (glfd);
+	}
+
+	glfs_subvol_done (fs, subvol);
+
+	return glfd;
+}
+
+
+off_t
+glfs_lseek (struct glfs_fd *glfd, off_t offset, int whence)
+{
+	struct stat sb = {0, };
+	int         ret = -1;
+
+	__glfs_entry_fd (glfd);
+
+	switch (whence) {
+	case SEEK_SET:
+		glfd->offset = offset;
+		break;
+	case SEEK_CUR:
+		glfd->offset += offset;
+		break;
+	case SEEK_END:
+		ret = glfs_fstat (glfd, &sb);
+		if (ret) {
+			/* seek cannot fail :O */
+			break;
+		}
+		glfd->offset = sb.st_size + offset;
+		break;
+	}
+
+	return glfd->offset;
+}
+
+
+//////////////
+
+ssize_t
+glfs_preadv (struct glfs_fd *glfd, const struct iovec *iovec, int iovcnt,
+	     off_t offset, int flags)
+{
+	xlator_t       *subvol = NULL;
+	ssize_t         ret = -1;
+	ssize_t         size = -1;
+	struct iovec   *iov = NULL;
+	int             cnt = 0;
+	struct iobref  *iobref = NULL;
+	fd_t           *fd = NULL;
+
+	__glfs_entry_fd (glfd);
+
+	subvol = glfs_active_subvol (glfd->fs);
+	if (!subvol) {
+		ret = -1;
+		errno = EIO;
+		goto out;
+	}
+
+	fd = glfs_resolve_fd (glfd->fs, subvol, glfd);
+	if (!fd) {
+		ret = -1;
+		errno = EBADFD;
+		goto out;
+	}
+
+	size = iov_length (iovec, iovcnt);
+
+	ret = syncop_readv (subvol, fd, size, offset, 0, &iov, &cnt, &iobref);
+	if (ret <= 0)
+		goto out;
+
+	size = iov_copy (iovec, iovcnt, iov, cnt); /* FIXME!!! */
+
+	glfd->offset = (offset + size);
+
+	ret = size;
+out:
+        if (iov)
+                GF_FREE (iov);
+        if (iobref)
+                iobref_unref (iobref);
+
+	if (fd)
+		fd_unref (fd);
+
+	glfs_subvol_done (glfd->fs, subvol);
+
+	return ret;
+}
+
+
+ssize_t
+glfs_read (struct glfs_fd *glfd, void *buf, size_t count, int flags)
+{
+	struct iovec iov = {0, };
+	ssize_t      ret = 0;
+
+	iov.iov_base = buf;
+	iov.iov_len = count;
+
+	ret = glfs_preadv (glfd, &iov, 1, glfd->offset, flags);
+
+	return ret;
+}
+
+
+ssize_t
+glfs_pread (struct glfs_fd *glfd, void *buf, size_t count, off_t offset,
+	    int flags)
+{
+	struct iovec iov = {0, };
+	ssize_t      ret = 0;
+
+	iov.iov_base = buf;
+	iov.iov_len = count;
+
+	ret = glfs_preadv (glfd, &iov, 1, offset, flags);
+
+	return ret;
+}
+
+
+ssize_t
+glfs_readv (struct glfs_fd *glfd, const struct iovec *iov, int count,
+	    int flags)
+{
+	ssize_t      ret = 0;
+
+	ret = glfs_preadv (glfd, iov, count, glfd->offset, flags);
+
+	return ret;
+}
+
+
+struct glfs_io {
+	struct glfs_fd      *glfd;
+	int                  op;
+	off_t                offset;
+	struct iovec        *iov;
+	int                  count;
+	int                  flags;
+	glfs_io_cbk          fn;
+	void                *data;
+};
+
+
+static int
+glfs_io_async_cbk (int ret, call_frame_t *frame, void *data)
+{
+	struct glfs_io  *gio = data;
+
+	gio->fn (gio->glfd, ret, gio->data);
+
+	GF_FREE (gio->iov);
+	GF_FREE (gio);
+
+	return 0;
+}
+
+
+static int
+glfs_io_async_task (void *data)
+{
+	struct glfs_io *gio = data;
+	ssize_t         ret = 0;
+
+	switch (gio->op) {
+	case GF_FOP_WRITE:
+		ret = glfs_pwritev (gio->glfd, gio->iov, gio->count,
+				    gio->offset, gio->flags);
+		break;
+	case GF_FOP_FTRUNCATE:
+		ret = glfs_ftruncate (gio->glfd, gio->offset);
+		break;
+	case GF_FOP_FSYNC:
+		if (gio->flags)
+			ret = glfs_fdatasync (gio->glfd);
+		else
+			ret = glfs_fsync (gio->glfd);
+		break;
+	case GF_FOP_DISCARD:
+		ret = glfs_discard (gio->glfd, gio->offset, gio->count);
+		break;
+        case GF_FOP_ZEROFILL:
+                ret = glfs_zerofill(gio->glfd, gio->offset, gio->count);
+                break;
+	}
+
+	return (int) ret;
+}
+
+
+int
+glfs_preadv_async_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+		       int op_ret, int op_errno, struct iovec *iovec,
+		       int count, struct iatt *stbuf, struct iobref *iobref,
+		       dict_t *xdata)
+{
+	struct glfs_io *gio = NULL;
+	xlator_t       *subvol = NULL;
+	struct glfs    *fs = NULL;
+	struct glfs_fd *glfd = NULL;
+
+
+	gio = frame->local;
+	frame->local = NULL;
+	subvol = cookie;
+	glfd = gio->glfd;
+	fs = glfd->fs;
+
+	if (op_ret <= 0)
+		goto out;
+
+	op_ret = iov_copy (gio->iov, gio->count, iovec, count);
+
+	glfd->offset = gio->offset + op_ret;
+out:
+	errno = op_errno;
+	gio->fn (gio->glfd, op_ret, gio->data);
+
+	GF_FREE (gio->iov);
+	GF_FREE (gio);
+	STACK_DESTROY (frame->root);
+	glfs_subvol_done (fs, subvol);
+
+	return 0;
+}
+
+
+int
+glfs_preadv_async (struct glfs_fd *glfd, const struct iovec *iovec, int count,
+		   off_t offset, int flags, glfs_io_cbk fn, void *data)
+{
+	struct glfs_io *gio = NULL;
+	int             ret = 0;
+	call_frame_t   *frame = NULL;
+	xlator_t       *subvol = NULL;
+	glfs_t         *fs = NULL;
+	fd_t           *fd = NULL;
+
+	__glfs_entry_fd (glfd);
+
+	subvol = glfs_active_subvol (glfd->fs);
+	if (!subvol) {
+		ret = -1;
+		errno = EIO;
+		goto out;
+	}
+
+	fd = glfs_resolve_fd (glfd->fs, subvol, glfd);
+	if (!fd) {
+		ret = -1;
+		errno = EBADFD;
+		goto out;
+	}
+
+	fs = glfd->fs;
+
+	frame = syncop_create_frame (THIS);
+	if (!frame) {
+		ret = -1;
+		errno = ENOMEM;
+		goto out;
+	}
+
+	gio = GF_CALLOC (1, sizeof (*gio), glfs_mt_glfs_io_t);
+	if (!gio) {
+		ret = -1;
+		errno = ENOMEM;
+		goto out;
+	}
+
+	gio->iov = iov_dup (iovec, count);
+	if (!gio->iov) {
+		ret = -1;
+		errno = ENOMEM;
+		goto out;
+	}
+
+	gio->op     = GF_FOP_READ;
+	gio->glfd   = glfd;
+	gio->count  = count;
+	gio->offset = offset;
+	gio->flags  = flags;
+	gio->fn     = fn;
+	gio->data   = data;
+
+	frame->local = gio;
+
+	STACK_WIND_COOKIE (frame, glfs_preadv_async_cbk, subvol, subvol,
+			   subvol->fops->readv, fd, iov_length (iovec, count),
+			   offset, flags, NULL);
+
+out:
+	if (ret) {
+		GF_FREE (gio->iov);
+		GF_FREE (gio);
+		STACK_DESTROY (frame->root);
+		glfs_subvol_done (fs, subvol);
+	}
+
+	if (fd)
+		fd_unref (fd);
+
+	return ret;
+}
+
+
+int
+glfs_read_async (struct glfs_fd *glfd, void *buf, size_t count, int flags,
+		 glfs_io_cbk fn, void *data)
+{
+	struct iovec iov = {0, };
+	ssize_t      ret = 0;
+
+	iov.iov_base = buf;
+	iov.iov_len = count;
+
+	ret = glfs_preadv_async (glfd, &iov, 1, glfd->offset, flags, fn, data);
+
+	return ret;
+}
+
+
+int
+glfs_pread_async (struct glfs_fd *glfd, void *buf, size_t count, off_t offset,
+		  int flags, glfs_io_cbk fn, void *data)
+{
+	struct iovec iov = {0, };
+	ssize_t      ret = 0;
+
+	iov.iov_base = buf;
+	iov.iov_len = count;
+
+	ret = glfs_preadv_async (glfd, &iov, 1, offset, flags, fn, data);
+
+	return ret;
+}
+
+
+int
+glfs_readv_async (struct glfs_fd *glfd, const struct iovec *iov, int count,
+		  int flags, glfs_io_cbk fn, void *data)
+{
+	ssize_t      ret = 0;
+
+	ret = glfs_preadv_async (glfd, iov, count, glfd->offset, flags,
+				 fn, data);
+	return ret;
+}
+
+///// writev /////
+
+ssize_t
+glfs_pwritev (struct glfs_fd *glfd, const struct iovec *iovec, int iovcnt,
+	      off_t offset, int flags)
+{
+	xlator_t       *subvol = NULL;
+	int             ret = -1;
+	size_t          size = -1;
+	struct iobref  *iobref = NULL;
+	struct iobuf   *iobuf = NULL;
+	struct iovec    iov = {0, };
+	fd_t           *fd = NULL;
+
+	__glfs_entry_fd (glfd);
+
+	subvol = glfs_active_subvol (glfd->fs);
+	if (!subvol) {
+		ret = -1;
+		errno = EIO;
+		goto out;
+	}
+
+	fd = glfs_resolve_fd (glfd->fs, subvol, glfd);
+	if (!fd) {
+		ret = -1;
+		errno = EBADFD;
+		goto out;
+	}
+
+	size = iov_length (iovec, iovcnt);
+
+	iobuf = iobuf_get2 (subvol->ctx->iobuf_pool, size);
+	if (!iobuf) {
+		ret = -1;
+		errno = ENOMEM;
+		goto out;
+	}
+
+	iobref = iobref_new ();
+	if (!iobref) {
+		iobuf_unref (iobuf);
+		errno = ENOMEM;
+		ret = -1;
+		goto out;
+	}
+
+	ret = iobref_add (iobref, iobuf);
+	if (ret) {
+		iobuf_unref (iobuf);
+		iobref_unref (iobref);
+		errno = ENOMEM;
+		ret = -1;
+		goto out;
+	}
+
+	iov_unload (iobuf_ptr (iobuf), iovec, iovcnt);  /* FIXME!!! */
+
+	iov.iov_base = iobuf_ptr (iobuf);
+	iov.iov_len = size;
+
+	ret = syncop_writev (subvol, fd, &iov, 1, offset, iobref, flags);
+
+	iobuf_unref (iobuf);
+	iobref_unref (iobref);
+
+	if (ret <= 0)
+		goto out;
+
+	glfd->offset = (offset + size);
+
+out:
+	if (fd)
+		fd_unref (fd);
+
+	glfs_subvol_done (glfd->fs, subvol);
+
+	return ret;
+}
+
+
+ssize_t
+glfs_write (struct glfs_fd *glfd, const void *buf, size_t count, int flags)
+{
+	struct iovec iov = {0, };
+	ssize_t      ret = 0;
+
+	iov.iov_base = (void *) buf;
+	iov.iov_len = count;
+
+	ret = glfs_pwritev (glfd, &iov, 1, glfd->offset, flags);
+
+	return ret;
+}
+
+
+
+ssize_t
+glfs_writev (struct glfs_fd *glfd, const struct iovec *iov, int count,
+	     int flags)
+{
+	ssize_t      ret = 0;
+
+	ret = glfs_pwritev (glfd, iov, count, glfd->offset, flags);
+
+	return ret;
+}
+
+
+ssize_t
+glfs_pwrite (struct glfs_fd *glfd, const void *buf, size_t count, off_t offset,
+	     int flags)
+{
+	struct iovec iov = {0, };
+	ssize_t      ret = 0;
+
+	iov.iov_base = (void *) buf;
+	iov.iov_len = count;
+
+	ret = glfs_pwritev (glfd, &iov, 1, offset, flags);
+
+	return ret;
+}
+
+
+int
+glfs_pwritev_async (struct glfs_fd *glfd, const struct iovec *iovec, int count,
+		    off_t offset, int flags, glfs_io_cbk fn, void *data)
+{
+	struct glfs_io *gio = NULL;
+	int             ret = 0;
+
+	gio = GF_CALLOC (1, sizeof (*gio), glfs_mt_glfs_io_t);
+	if (!gio) {
+		errno = ENOMEM;
+		return -1;
+	}
+
+	gio->iov = iov_dup (iovec, count);
+	if (!gio->iov) {
+		GF_FREE (gio);
+		errno = ENOMEM;
+		return -1;
+	}
+
+	gio->op     = GF_FOP_WRITE;
+	gio->glfd   = glfd;
+	gio->count  = count;
+	gio->offset = offset;
+	gio->flags  = flags;
+	gio->fn     = fn;
+	gio->data   = data;
+
+	ret = synctask_new (glfs_from_glfd (glfd)->ctx->env,
+			    glfs_io_async_task, glfs_io_async_cbk,
+			    NULL, gio);
+
+	if (ret) {
+		GF_FREE (gio->iov);
+		GF_FREE (gio);
+	}
+
+	return ret;
+}
+
+
+int
+glfs_write_async (struct glfs_fd *glfd, const void *buf, size_t count, int flags,
+		  glfs_io_cbk fn, void *data)
+{
+	struct iovec iov = {0, };
+	ssize_t      ret = 0;
+
+	iov.iov_base = (void *) buf;
+	iov.iov_len = count;
+
+	ret = glfs_pwritev_async (glfd, &iov, 1, glfd->offset, flags, fn, data);
+
+	return ret;
+}
+
+
+int
+glfs_pwrite_async (struct glfs_fd *glfd, const void *buf, int count,
+		   off_t offset, int flags, glfs_io_cbk fn, void *data)
+{
+	struct iovec iov = {0, };
+	ssize_t      ret = 0;
+
+	iov.iov_base = (void *) buf;
+	iov.iov_len = count;
+
+	ret = glfs_pwritev_async (glfd, &iov, 1, offset, flags, fn, data);
+
+	return ret;
+}
+
+
+int
+glfs_writev_async (struct glfs_fd *glfd, const struct iovec *iov, int count,
+		   int flags, glfs_io_cbk fn, void *data)
+{
+	ssize_t      ret = 0;
+
+	ret = glfs_pwritev_async (glfd, iov, count, glfd->offset, flags,
+				  fn, data);
+	return ret;
+}
+
+
+int
+glfs_fsync (struct glfs_fd *glfd)
+{
+	int              ret = -1;
+	xlator_t        *subvol = NULL;
+	fd_t            *fd = NULL;
+
+	__glfs_entry_fd (glfd);
+
+	subvol = glfs_active_subvol (glfd->fs);
+	if (!subvol) {
+		ret = -1;
+		errno = EIO;
+		goto out;
+	}
+
+	fd = glfs_resolve_fd (glfd->fs, subvol, glfd);
+	if (!fd) {
+		ret = -1;
+		errno = EBADFD;
+		goto out;
+	}
+
+	ret = syncop_fsync (subvol, fd, 0);
+out:
+	if (fd)
+		fd_unref (fd);
+
+	glfs_subvol_done (glfd->fs, subvol);
+
+	return ret;
+}
+
+
+static int
+glfs_fsync_async_common (struct glfs_fd *glfd, glfs_io_cbk fn, void *data,
+			 int dataonly)
+{
+	struct glfs_io *gio = NULL;
+	int             ret = 0;
+
+	gio = GF_CALLOC (1, sizeof (*gio), glfs_mt_glfs_io_t);
+	if (!gio) {
+		errno = ENOMEM;
+		return -1;
+	}
+
+	gio->op     = GF_FOP_FSYNC;
+	gio->glfd   = glfd;
+	gio->flags  = dataonly;
+	gio->fn     = fn;
+	gio->data   = data;
+
+	ret = synctask_new (glfs_from_glfd (glfd)->ctx->env,
+			    glfs_io_async_task, glfs_io_async_cbk,
+			    NULL, gio);
+
+	if (ret) {
+		GF_FREE (gio->iov);
+		GF_FREE (gio);
+	}
+
+	return ret;
+
+}
+
+
+int
+glfs_fsync_async (struct glfs_fd *glfd, glfs_io_cbk fn, void *data)
+{
+	return glfs_fsync_async_common (glfd, fn, data, 0);
+}
+
+
+int
+glfs_fdatasync (struct glfs_fd *glfd)
+{
+	int              ret = -1;
+	xlator_t        *subvol = NULL;
+	fd_t            *fd = NULL;
+
+	__glfs_entry_fd (glfd);
+
+	subvol = glfs_active_subvol (glfd->fs);
+	if (!subvol) {
+		ret = -1;
+		errno = EIO;
+		goto out;
+	}
+
+	fd = glfs_resolve_fd (glfd->fs, subvol, glfd);
+	if (!fd) {
+		ret = -1;
+		errno = EBADFD;
+		goto out;
+	}
+
+	ret = syncop_fsync (subvol, fd, 1);
+out:
+	if (fd)
+		fd_unref (fd);
+
+	glfs_subvol_done (glfd->fs, subvol);
+
+	return ret;
+}
+
+
+int
+glfs_fdatasync_async (struct glfs_fd *glfd, glfs_io_cbk fn, void *data)
+{
+	return glfs_fsync_async_common (glfd, fn, data, 1);
+}
+
+
+int
+glfs_ftruncate (struct glfs_fd *glfd, off_t offset)
+{
+	int              ret = -1;
+	xlator_t        *subvol = NULL;
+	fd_t            *fd = NULL;
+
+	__glfs_entry_fd (glfd);
+
+	subvol = glfs_active_subvol (glfd->fs);
+	if (!subvol) {
+		ret = -1;
+		errno = EIO;
+		goto out;
+	}
+
+	fd = glfs_resolve_fd (glfd->fs, subvol, glfd);
+	if (!fd) {
+		ret = -1;
+		errno = EBADFD;
+		goto out;
+	}
+
+	ret = syncop_ftruncate (subvol, fd, offset);
+out:
+	if (fd)
+		fd_unref (fd);
+
+	glfs_subvol_done (glfd->fs, subvol);
+
+	return ret;
+}
+
+
+int
+glfs_ftruncate_async (struct glfs_fd *glfd, off_t offset,
+		      glfs_io_cbk fn, void *data)
+{
+	struct glfs_io *gio = NULL;
+	int             ret = 0;
+
+	gio = GF_CALLOC (1, sizeof (*gio), glfs_mt_glfs_io_t);
+	if (!gio) {
+		errno = ENOMEM;
+		return -1;
+	}
+
+	gio->op     = GF_FOP_FTRUNCATE;
+	gio->glfd   = glfd;
+	gio->offset = offset;
+	gio->fn     = fn;
+	gio->data   = data;
+
+	ret = synctask_new (glfs_from_glfd (glfd)->ctx->env,
+			    glfs_io_async_task, glfs_io_async_cbk,
+			    NULL, gio);
+
+	if (ret) {
+		GF_FREE (gio->iov);
+		GF_FREE (gio);
+	}
+
+	return ret;
+}
+
+
+int
+glfs_access (struct glfs *fs, const char *path, int mode)
+{
+	int              ret = -1;
+	xlator_t        *subvol = NULL;
+	loc_t            loc = {0, };
+	struct iatt      iatt = {0, };
+	int              reval = 0;
+
+	__glfs_entry_fs (fs);
+
+	subvol = glfs_active_subvol (fs);
+	if (!subvol) {
+		ret = -1;
+		errno = EIO;
+		goto out;
+	}
+retry:
+	ret = glfs_resolve (fs, subvol, path, &loc, &iatt, reval);
+
+	ESTALE_RETRY (ret, errno, reval, &loc, retry);
+
+	if (ret)
+		goto out;
+
+	ret = syncop_access (subvol, &loc, mode);
+
+	ESTALE_RETRY (ret, errno, reval, &loc, retry);
+out:
+	loc_wipe (&loc);
+
+	glfs_subvol_done (fs, subvol);
+
+	return ret;
+}
+
+
+int
+glfs_symlink (struct glfs *fs, const char *data, const char *path)
+{
+	int              ret = -1;
+	xlator_t        *subvol = NULL;
+	loc_t            loc = {0, };
+	struct iatt      iatt = {0, };
+	uuid_t           gfid;
+	dict_t          *xattr_req = NULL;
+	int              reval = 0;
+
+	__glfs_entry_fs (fs);
+
+	subvol = glfs_active_subvol (fs);
+	if (!subvol) {
+		ret = -1;
+		errno = EIO;
+		goto out;
+	}
+
+	xattr_req = dict_new ();
+	if (!xattr_req) {
+		ret = -1;
+		errno = ENOMEM;
+		goto out;
+	}
+
+	uuid_generate (gfid);
+	ret = dict_set_static_bin (xattr_req, "gfid-req", gfid, 16);
+	if (ret) {
+		ret = -1;
+		errno = ENOMEM;
+		goto out;
+	}
+retry:
+	ret = glfs_lresolve (fs, subvol, path, &loc, &iatt, reval);
+
+	ESTALE_RETRY (ret, errno, reval, &loc, retry);
+
+	if (loc.inode) {
+		errno = EEXIST;
+		ret = -1;
+		goto out;
+	}
+
+	if (ret == -1 && errno != ENOENT)
+		/* Any other type of error is fatal */
+		goto out;
+
+	if (ret == -1 && errno == ENOENT && !loc.parent)
+		/* The parent directory or an ancestor even
+		   higher does not exist
+		*/
+		goto out;
+
+	/* ret == -1 && errno == ENOENT */
+	loc.inode = inode_new (loc.parent->table);
+	if (!loc.inode) {
+		ret = -1;
+		errno = ENOMEM;
+		goto out;
+	}
+
+	ret = syncop_symlink (subvol, &loc, data, xattr_req, &iatt);
+
+	ESTALE_RETRY (ret, errno, reval, &loc, retry);
+
+	if (ret == 0)
+		ret = glfs_loc_link (&loc, &iatt);
+out:
+	loc_wipe (&loc);
+
+	if (xattr_req)
+		dict_unref (xattr_req);
+
+	glfs_subvol_done (fs, subvol);
+
+	return ret;
+}
+
+
+int
+glfs_readlink (struct glfs *fs, const char *path, char *buf, size_t bufsiz)
+{
+	int              ret = -1;
+	xlator_t        *subvol = NULL;
+	loc_t            loc = {0, };
+	struct iatt      iatt = {0, };
+	int              reval = 0;
+	char            *linkval = NULL;
+
+	__glfs_entry_fs (fs);
+
+	subvol = glfs_active_subvol (fs);
+	if (!subvol) {
+		ret = -1;
+		errno = EIO;
+		goto out;
+	}
+retry:
+	ret = glfs_lresolve (fs, subvol, path, &loc, &iatt, reval);
+
+	ESTALE_RETRY (ret, errno, reval, &loc, retry);
+
+	if (ret)
+		goto out;
+
+	if (iatt.ia_type != IA_IFLNK) {
+		ret = -1;
+		errno = EINVAL;
+		goto out;
+	}
+
+	ret = syncop_readlink (subvol, &loc, &linkval, bufsiz);
+	if (ret > 0) {
+		memcpy (buf, linkval, ret);
+		GF_FREE (linkval);
+	}
+
+	ESTALE_RETRY (ret, errno, reval, &loc, retry);
+out:
+	loc_wipe (&loc);
+
+	glfs_subvol_done (fs, subvol);
+
+	return ret;
+}
+
+
+int
+glfs_mknod (struct glfs *fs, const char *path, mode_t mode, dev_t dev)
+{
+	int              ret = -1;
+	xlator_t        *subvol = NULL;
+	loc_t            loc = {0, };
+	struct iatt      iatt = {0, };
+	uuid_t           gfid;
+	dict_t          *xattr_req = NULL;
+	int              reval = 0;
+
+	__glfs_entry_fs (fs);
+
+	subvol = glfs_active_subvol (fs);
+	if (!subvol) {
+		ret = -1;
+		errno = EIO;
+		goto out;
+	}
+
+	xattr_req = dict_new ();
+	if (!xattr_req) {
+		ret = -1;
+		errno = ENOMEM;
+		goto out;
+	}
+
+	uuid_generate (gfid);
+	ret = dict_set_static_bin (xattr_req, "gfid-req", gfid, 16);
+	if (ret) {
+		ret = -1;
+		errno = ENOMEM;
+		goto out;
+	}
+retry:
+	ret = glfs_lresolve (fs, subvol, path, &loc, &iatt, reval);
+
+	ESTALE_RETRY (ret, errno, reval, &loc, retry);
+
+	if (loc.inode) {
+		errno = EEXIST;
+		ret = -1;
+		goto out;
+	}
+
+	if (ret == -1 && errno != ENOENT)
+		/* Any other type of error is fatal */
+		goto out;
+
+	if (ret == -1 && errno == ENOENT && !loc.parent)
+		/* The parent directory or an ancestor even
+		   higher does not exist
+		*/
+		goto out;
+
+	/* ret == -1 && errno == ENOENT */
+	loc.inode = inode_new (loc.parent->table);
+	if (!loc.inode) {
+		ret = -1;
+		errno = ENOMEM;
+		goto out;
+	}
+
+	ret = syncop_mknod (subvol, &loc, mode, dev, xattr_req, &iatt);
+
+	ESTALE_RETRY (ret, errno, reval, &loc, retry);
+
+	if (ret == 0)
+		ret = glfs_loc_link (&loc, &iatt);
+out:
+	loc_wipe (&loc);
+
+	if (xattr_req)
+		dict_unref (xattr_req);
+
+	glfs_subvol_done (fs, subvol);
+
+	return ret;
+}
+
+
+int
+glfs_mkdir (struct glfs *fs, const char *path, mode_t mode)
+{
+	int              ret = -1;
+	xlator_t        *subvol = NULL;
+	loc_t            loc = {0, };
+	struct iatt      iatt = {0, };
+	uuid_t           gfid;
+	dict_t          *xattr_req = NULL;
+	int              reval = 0;
+
+	__glfs_entry_fs (fs);
+
+	subvol = glfs_active_subvol (fs);
+	if (!subvol) {
+		ret = -1;
+		errno = EIO;
+		goto out;
+	}
+
+	xattr_req = dict_new ();
+	if (!xattr_req) {
+		ret = -1;
+		errno = ENOMEM;
+		goto out;
+	}
+
+	uuid_generate (gfid);
+	ret = dict_set_static_bin (xattr_req, "gfid-req", gfid, 16);
+	if (ret) {
+		ret = -1;
+		errno = ENOMEM;
+		goto out;
+	}
+retry:
+	ret = glfs_lresolve (fs, subvol, path, &loc, &iatt, reval);
+
+	ESTALE_RETRY (ret, errno, reval, &loc, retry);
+
+	if (loc.inode) {
+		errno = EEXIST;
+		ret = -1;
+		goto out;
+	}
+
+	if (ret == -1 && errno != ENOENT)
+		/* Any other type of error is fatal */
+		goto out;
+
+	if (ret == -1 && errno == ENOENT && !loc.parent)
+		/* The parent directory or an ancestor even
+		   higher does not exist
+		*/
+		goto out;
+
+	/* ret == -1 && errno == ENOENT */
+	loc.inode = inode_new (loc.parent->table);
+	if (!loc.inode) {
+		ret = -1;
+		errno = ENOMEM;
+		goto out;
+	}
+
+	ret = syncop_mkdir (subvol, &loc, mode, xattr_req, &iatt);
+
+	ESTALE_RETRY (ret, errno, reval, &loc, retry);
+
+	if (ret == 0)
+		ret = glfs_loc_link (&loc, &iatt);
+out:
+	loc_wipe (&loc);
+
+	if (xattr_req)
+		dict_unref (xattr_req);
+
+	glfs_subvol_done (fs, subvol);
+
+	return ret;
+}
+
+
+int
+glfs_unlink (struct glfs *fs, const char *path)
+{
+	int              ret = -1;
+	xlator_t        *subvol = NULL;
+	loc_t            loc = {0, };
+	struct iatt      iatt = {0, };
+	int              reval = 0;
+
+	__glfs_entry_fs (fs);
+
+	subvol = glfs_active_subvol (fs);
+	if (!subvol) {
+		ret = -1;
+		errno = EIO;
+		goto out;
+	}
+retry:
+	ret = glfs_lresolve (fs, subvol, path, &loc, &iatt, reval);
+
+	ESTALE_RETRY (ret, errno, reval, &loc, retry);
+
+	if (ret)
+		goto out;
+
+	if (iatt.ia_type == IA_IFDIR) {
+		ret = -1;
+		errno = EISDIR;
+		goto out;
+	}
+
+	ret = syncop_unlink (subvol, &loc);
+
+	ESTALE_RETRY (ret, errno, reval, &loc, retry);
+
+	if (ret == 0)
+		ret = glfs_loc_unlink (&loc);
+out:
+	loc_wipe (&loc);
+
+	glfs_subvol_done (fs, subvol);
+
+	return ret;
+}
+
+
+int
+glfs_rmdir (struct glfs *fs, const char *path)
+{
+	int              ret = -1;
+	xlator_t        *subvol = NULL;
+	loc_t            loc = {0, };
+	struct iatt      iatt = {0, };
+	int              reval = 0;
+
+	__glfs_entry_fs (fs);
+
+	subvol = glfs_active_subvol (fs);
+	if (!subvol) {
+		ret = -1;
+		errno = EIO;
+		goto out;
+	}
+retry:
+	ret = glfs_lresolve (fs, subvol, path, &loc, &iatt, reval);
+
+	ESTALE_RETRY (ret, errno, reval, &loc, retry);
+
+	if (ret)
+		goto out;
+
+	if (iatt.ia_type != IA_IFDIR) {
+		ret = -1;
+		errno = ENOTDIR;
+		goto out;
+	}
+
+	ret = syncop_rmdir (subvol, &loc);
+
+	ESTALE_RETRY (ret, errno, reval, &loc, retry);
+
+	if (ret == 0)
+		ret = glfs_loc_unlink (&loc);
+out:
+	loc_wipe (&loc);
+
+	glfs_subvol_done (fs, subvol);
+
+	return ret;
+}
+
+
+int
+glfs_rename (struct glfs *fs, const char *oldpath, const char *newpath)
+{
+	int              ret = -1;
+	xlator_t        *subvol = NULL;
+	loc_t            oldloc = {0, };
+	loc_t            newloc = {0, };
+	struct iatt      oldiatt = {0, };
+	struct iatt      newiatt = {0, };
+	int              reval = 0;
+
+	__glfs_entry_fs (fs);
+
+	subvol = glfs_active_subvol (fs);
+	if (!subvol) {
+		ret = -1;
+		errno = EIO;
+		goto out;
+	}
+retry:
+	ret = glfs_lresolve (fs, subvol, oldpath, &oldloc, &oldiatt, reval);
+
+	ESTALE_RETRY (ret, errno, reval, &oldloc, retry);
+
+	if (ret)
+		goto out;
+retrynew:
+	ret = glfs_lresolve (fs, subvol, newpath, &newloc, &newiatt, reval);
+
+	ESTALE_RETRY (ret, errno, reval, &newloc, retrynew);
+
+	if (ret && errno != ENOENT && newloc.parent)
+		goto out;
+
+	if (newiatt.ia_type != IA_INVAL) {
+                if ((oldiatt.ia_type == IA_IFDIR) !=
+                    (newiatt.ia_type == IA_IFDIR)) {
+                       /* Either both old and new must be dirs,
+                        * or both must be non-dirs. Else, fail.
+                        */
+                       ret = -1;
+                       errno = EISDIR;
+                       goto out;
+                }
+        }
+
+	/* TODO: check if new or old is a prefix of the other, and fail EINVAL */
+
+	ret = syncop_rename (subvol, &oldloc, &newloc);
+
+	if (ret == -1 && errno == ESTALE) {
+		if (reval < DEFAULT_REVAL_COUNT) {
+			reval++;
+			loc_wipe (&oldloc);
+			loc_wipe (&newloc);
+			goto retry;
+		}
+	}
+
+	if (ret == 0)
+		inode_rename (oldloc.parent->table, oldloc.parent, oldloc.name,
+			      newloc.parent, newloc.name, oldloc.inode,
+			      &oldiatt);
+out:
+	loc_wipe (&oldloc);
+	loc_wipe (&newloc);
+
+	glfs_subvol_done (fs, subvol);
+
+	return ret;
+}
+
+
+int
+glfs_link (struct glfs *fs, const char *oldpath, const char *newpath)
+{
+	int              ret = -1;
+	xlator_t        *subvol = NULL;
+	loc_t            oldloc = {0, };
+	loc_t            newloc = {0, };
+	struct iatt      oldiatt = {0, };
+	struct iatt      newiatt = {0, };
+	int              reval = 0;
+
+	__glfs_entry_fs (fs);
+
+	subvol = glfs_active_subvol (fs);
+	if (!subvol) {
+		ret = -1;
+		errno = EIO;
+		goto out;
+	}
+retry:
+	ret = glfs_lresolve (fs, subvol, oldpath, &oldloc, &oldiatt, reval);
+
+	ESTALE_RETRY (ret, errno, reval, &oldloc, retry);
+
+	if (ret)
+		goto out;
+retrynew:
+	ret = glfs_lresolve (fs, subvol, newpath, &newloc, &newiatt, reval);
+
+	ESTALE_RETRY (ret, errno, reval, &newloc, retrynew);
+
+	if (ret == 0) {
+		ret = -1;
+		errno = EEXIST;
+		goto out;
+	}
+
+	if (oldiatt.ia_type == IA_IFDIR) {
+		ret = -1;
+		errno = EISDIR;
+		goto out;
+	}
+
+        /* Filling the inode of the hard link to be same as that of the
+           original file
+        */
+	if (newloc.inode) {
+		inode_unref (newloc.inode);
+		newloc.inode = NULL;
+	}
+        newloc.inode = inode_ref (oldloc.inode);
+
+	ret = syncop_link (subvol, &oldloc, &newloc);
+
+	if (ret == -1 && errno == ESTALE) {
+		loc_wipe (&oldloc);
+		loc_wipe (&newloc);
+		if (reval--)
+			goto retry;
+	}
+
+	if (ret == 0)
+		ret = glfs_loc_link (&newloc, &oldiatt);
+out:
+	loc_wipe (&oldloc);
+	loc_wipe (&newloc);
+
+	glfs_subvol_done (fs, subvol);
+
+	return ret;
+}
+
+
+struct glfs_fd *
+glfs_opendir (struct glfs *fs, const char *path)
+{
+	int              ret = -1;
+	struct glfs_fd  *glfd = NULL;
+	xlator_t        *subvol = NULL;
+	loc_t            loc = {0, };
+	struct iatt      iatt = {0, };
+	int              reval = 0;
+
+	__glfs_entry_fs (fs);
+
+	subvol = glfs_active_subvol (fs);
+	if (!subvol) {
+		ret = -1;
+		errno = EIO;
+		goto out;
+	}
+
+	glfd = glfs_fd_new (fs);
+	if (!glfd)
+		goto out;
+
+	INIT_LIST_HEAD (&glfd->entries);
+retry:
+	ret = glfs_resolve (fs, subvol, path, &loc, &iatt, reval);
+
+	ESTALE_RETRY (ret, errno, reval, &loc, retry);
+
+	if (ret)
+		goto out;
+
+	if (!IA_ISDIR (iatt.ia_type)) {
+		ret = -1;
+		errno = ENOTDIR;
+		goto out;
+	}
+
+	if (glfd->fd) {
+		/* Retry. Safe to touch glfd->fd as we
+		   still have not glfs_fd_bind() yet.
+		*/
+		fd_unref (glfd->fd);
+		glfd->fd = NULL;
+	}
+
+	glfd->fd = fd_create (loc.inode, getpid());
+	if (!glfd->fd) {
+		ret = -1;
+		errno = ENOMEM;
+		goto out;
+	}
+
+	ret = syncop_opendir (subvol, &loc, glfd->fd);
+
+	ESTALE_RETRY (ret, errno, reval, &loc, retry);
+out:
+	loc_wipe (&loc);
+
+	if (ret && glfd) {
+		glfs_fd_destroy (glfd);
+		glfd = NULL;
+	} else {
+		fd_bind (glfd->fd);
+		glfs_fd_bind (glfd);
+	}
+
+	glfs_subvol_done (fs, subvol);
+
+	return glfd;
+}
+
+
+int
+glfs_closedir (struct glfs_fd *glfd)
+{
+	__glfs_entry_fd (glfd);
+
+	gf_dirent_free (list_entry (&glfd->entries, gf_dirent_t, list));
+
+	glfs_fd_destroy (glfd);
+
+	return 0;
+}
+
+
+long
+glfs_telldir (struct glfs_fd *fd)
+{
+	return fd->offset;
+}
+
+
+void
+glfs_seekdir (struct glfs_fd *fd, long offset)
+{
+	gf_dirent_t *entry = NULL;
+	gf_dirent_t *tmp = NULL;
+
+	if (fd->offset == offset)
+		return;
+
+	fd->offset = offset;
+	fd->next = NULL;
+
+	list_for_each_entry_safe (entry, tmp, &fd->entries, list) {
+		if (entry->d_off != offset)
+			continue;
+
+		if (&tmp->list != &fd->entries) {
+			/* found! */
+			fd->next = tmp;
+			return;
+		}
+	}
+	/* could not find entry at requested offset in the cache.
+	   next readdir_r() will result in glfd_entry_refresh()
+	*/
+}
+
+int
+glfs_discard_async (struct glfs_fd *glfd, off_t offset, size_t len,
+		      glfs_io_cbk fn, void *data)
+{
+	struct glfs_io *gio = NULL;
+	int             ret = 0;
+
+	gio = GF_CALLOC (1, sizeof (*gio), glfs_mt_glfs_io_t);
+	if (!gio) {
+		errno = ENOMEM;
+		return -1;
+	}
+
+	gio->op     = GF_FOP_DISCARD;
+	gio->glfd   = glfd;
+	gio->offset = offset;
+	gio->count  = len;
+	gio->fn     = fn;
+	gio->data   = data;
+
+	ret = synctask_new (glfs_from_glfd (glfd)->ctx->env,
+			    glfs_io_async_task, glfs_io_async_cbk,
+			    NULL, gio);
+
+	if (ret) {
+		GF_FREE (gio->iov);
+		GF_FREE (gio);
+	}
+
+	return ret;
+}
+
+int
+glfs_zerofill_async (struct glfs_fd *glfd, off_t offset, size_t len,
+                      glfs_io_cbk fn, void *data)
+{
+        struct glfs_io *gio  = NULL;
+        int             ret  = 0;
+
+        gio = GF_CALLOC (1, sizeof (*gio), glfs_mt_glfs_io_t);
+        if (!gio) {
+                errno = ENOMEM;
+                return -1;
+        }
+
+        gio->op     = GF_FOP_ZEROFILL;
+        gio->glfd   = glfd;
+        gio->offset = offset;
+        gio->count  = len;
+        gio->fn     = fn;
+        gio->data   = data;
+
+        ret = synctask_new (glfs_from_glfd (glfd)->ctx->env,
+                            glfs_io_async_task, glfs_io_async_cbk,
+                            NULL, gio);
+
+        if (ret) {
+                GF_FREE (gio->iov);
+                GF_FREE (gio);
+        }
+
+        return ret;
+}
+
+
+void
+gf_dirent_to_dirent (gf_dirent_t *gf_dirent, struct dirent *dirent)
+{
+	dirent->d_ino = gf_dirent->d_ino;
+
+#ifdef _DIRENT_HAVE_D_OFF
+	dirent->d_off = gf_dirent->d_off;
+#endif
+
+#ifdef _DIRENT_HAVE_D_TYPE
+	dirent->d_type = gf_dirent->d_type;
+#endif
+
+#ifdef _DIRENT_HAVE_D_NAMLEN
+	dirent->d_namlen = strlen (gf_dirent->d_name);
+#endif
+
+	strncpy (dirent->d_name, gf_dirent->d_name, GF_NAME_MAX + 1);
+}
+
+
+int
+glfd_entry_refresh (struct glfs_fd *glfd, int plus)
+{
+	xlator_t        *subvol = NULL;
+	gf_dirent_t      entries;
+	gf_dirent_t      old;
+	int              ret = -1;
+	fd_t            *fd = NULL;
+
+	subvol = glfs_active_subvol (glfd->fs);
+	if (!subvol) {
+		ret = -1;
+		errno = EIO;
+		goto out;
+	}
+
+	fd = glfs_resolve_fd (glfd->fs, subvol, glfd);
+	if (!fd) {
+		ret = -1;
+		errno = EBADFD;
+		goto out;
+	}
+
+	if (fd->inode->ia_type != IA_IFDIR) {
+		ret = -1;
+		errno = EBADF;
+		goto out;
+	}
+
+	INIT_LIST_HEAD (&entries.list);
+	INIT_LIST_HEAD (&old.list);
+
+	if (plus)
+		ret = syncop_readdirp (subvol, fd, 131072, glfd->offset,
+				       NULL, &entries);
+	else
+		ret = syncop_readdir (subvol, fd, 131072, glfd->offset,
+				      &entries);
+	if (ret >= 0) {
+		if (plus)
+			gf_link_inodes_from_dirent (THIS, fd->inode, &entries);
+
+		list_splice_init (&glfd->entries, &old.list);
+		list_splice_init (&entries.list, &glfd->entries);
+
+		/* spurious errno is dangerous for glfd_entry_next() */
+		errno = 0;
+	}
+
+	if (ret > 0)
+		glfd->next = list_entry (glfd->entries.next, gf_dirent_t, list);
+
+	gf_dirent_free (&old);
+out:
+	if (fd)
+		fd_unref (fd);
+
+	glfs_subvol_done (glfd->fs, subvol);
+
+	return ret;
+}
+
+
+gf_dirent_t *
+glfd_entry_next (struct glfs_fd *glfd, int plus)
+{
+	gf_dirent_t     *entry = NULL;
+	int              ret = -1;
+
+	if (!glfd->offset || !glfd->next) {
+		ret = glfd_entry_refresh (glfd, plus);
+		if (ret < 0)
+			return NULL;
+	}
+
+	entry = glfd->next;
+	if (!entry)
+		return NULL;
+
+	if (&entry->next->list == &glfd->entries)
+		glfd->next = NULL;
+	else
+		glfd->next = entry->next;
+
+	glfd->offset = entry->d_off;
+
+	return entry;
+}
+
+
+static struct dirent *
+glfs_readdirbuf_get (struct glfs_fd *glfd)
+{
+        struct dirent *buf = NULL;
+
+        LOCK (&glfd->fd->lock);
+        {
+                buf = glfd->readdirbuf;
+                if (buf) {
+                        memset (buf, 0, READDIRBUF_SIZE);
+                        goto unlock;
+                }
+
+                buf = GF_CALLOC (1, READDIRBUF_SIZE, glfs_mt_readdirbuf_t);
+                if (!buf) {
+                        errno = ENOMEM;
+                        goto unlock;
+                }
+
+                glfd->readdirbuf = buf;
+        }
+unlock:
+        UNLOCK (&glfd->fd->lock);
+
+        return buf;
+}
+
+
+int
+glfs_readdirplus_r (struct glfs_fd *glfd, struct stat *stat, struct dirent *ext,
+		    struct dirent **res)
+{
+	int              ret = 0;
+	gf_dirent_t     *entry = NULL;
+	struct dirent   *buf = NULL;
+
+	__glfs_entry_fd (glfd);
+
+	errno = 0;
+
+	if (ext)
+		buf = ext;
+	else
+		buf = glfs_readdirbuf_get (glfd);
+
+	if (!buf) {
+		errno = ENOMEM;
+		return -1;
+	}
+
+	entry = glfd_entry_next (glfd, !!stat);
+	if (errno)
+		ret = -1;
+
+	if (res) {
+		if (entry)
+			*res = buf;
+		else
+			*res = NULL;
+	}
+
+	if (entry) {
+		gf_dirent_to_dirent (entry, buf);
+		if (stat)
+			glfs_iatt_to_stat (glfd->fs, &entry->d_stat, stat);
+	}
+
+	return ret;
+}
+
+
+int
+glfs_readdir_r (struct glfs_fd *glfd, struct dirent *buf, struct dirent **res)
+{
+	return glfs_readdirplus_r (glfd, 0, buf, res);
+}
+
+
+struct dirent *
+glfs_readdirplus (struct glfs_fd *glfd, struct stat *stat)
+{
+        struct dirent *res = NULL;
+        int ret = -1;
+
+        ret = glfs_readdirplus_r (glfd, stat, NULL, &res);
+        if (ret)
+                return NULL;
+
+        return res;
+}
+
+
+
+struct dirent *
+glfs_readdir (struct glfs_fd *glfd)
+{
+        return glfs_readdirplus (glfd, NULL);
+}
+
+
+int
+glfs_statvfs (struct glfs *fs, const char *path, struct statvfs *buf)
+{
+	int              ret = -1;
+	xlator_t        *subvol = NULL;
+	loc_t            loc = {0, };
+	struct iatt      iatt = {0, };
+	int              reval = 0;
+
+	__glfs_entry_fs (fs);
+
+	subvol = glfs_active_subvol (fs);
+	if (!subvol) {
+		ret = -1;
+		errno = EIO;
+		goto out;
+	}
+retry:
+	ret = glfs_resolve (fs, subvol, path, &loc, &iatt, reval);
+
+	ESTALE_RETRY (ret, errno, reval, &loc, retry);
+
+	if (ret)
+		goto out;
+
+	ret = syncop_statfs (subvol, &loc, buf);
+
+	ESTALE_RETRY (ret, errno, reval, &loc, retry);
+out:
+	loc_wipe (&loc);
+
+	glfs_subvol_done (fs, subvol);
+
+	return ret;
+}
+
+
+int
+glfs_setattr (struct glfs *fs, const char *path, struct iatt *iatt,
+	      int valid, int follow)
+{
+	int              ret = -1;
+	xlator_t        *subvol = NULL;
+	loc_t            loc = {0, };
+	struct iatt      riatt = {0, };
+	int              reval = 0;
+
+	__glfs_entry_fs (fs);
+
+	subvol = glfs_active_subvol (fs);
+	if (!subvol) {
+		ret = -1;
+		errno = EIO;
+		goto out;
+	}
+retry:
+	if (follow)
+		ret = glfs_resolve (fs, subvol, path, &loc, &riatt, reval);
+	else
+		ret = glfs_lresolve (fs, subvol, path, &loc, &riatt, reval);
+
+	ESTALE_RETRY (ret, errno, reval, &loc, retry);
+
+	if (ret)
+		goto out;
+
+	ret = syncop_setattr (subvol, &loc, iatt, valid, 0, 0);
+
+	ESTALE_RETRY (ret, errno, reval, &loc, retry);
+out:
+	loc_wipe (&loc);
+
+	glfs_subvol_done (fs, subvol);
+
+	return ret;
+}
+
+
+int
+glfs_fsetattr (struct glfs_fd *glfd, struct iatt *iatt, int valid)
+{
+	int              ret = -1;
+	xlator_t        *subvol = NULL;
+	fd_t            *fd = NULL;
+
+	__glfs_entry_fd (glfd);
+
+	subvol = glfs_active_subvol (glfd->fs);
+	if (!subvol) {
+		ret = -1;
+		errno = EIO;
+		goto out;
+	}
+
+	fd = glfs_resolve_fd (glfd->fs, subvol, glfd);
+	if (!fd) {
+		ret = -1;
+		errno = EBADFD;
+		goto out;
+	}
+
+	ret = syncop_fsetattr (subvol, fd, iatt, valid, 0, 0);
+out:
+	if (fd)
+		fd_unref (fd);
+
+	glfs_subvol_done (glfd->fs, subvol);
+
+	return ret;
+}
+
+
+int
+glfs_chmod (struct glfs *fs, const char *path, mode_t mode)
+{
+	int              ret = -1;
+	struct iatt      iatt = {0, };
+	int              valid = 0;
+
+	iatt.ia_prot = ia_prot_from_st_mode (mode);
+	valid = GF_SET_ATTR_MODE;
+
+	ret = glfs_setattr (fs, path, &iatt, valid, 1);
+
+	return ret;
+}
+
+
+int
+glfs_fchmod (struct glfs_fd *glfd, mode_t mode)
+{
+	int              ret = -1;
+	struct iatt      iatt = {0, };
+	int              valid = 0;
+
+	iatt.ia_prot = ia_prot_from_st_mode (mode);
+	valid = GF_SET_ATTR_MODE;
+
+	ret = glfs_fsetattr (glfd, &iatt, valid);
+
+	return ret;
+}
+
+
+int
+glfs_chown (struct glfs *fs, const char *path, uid_t uid, gid_t gid)
+{
+	int              ret = -1;
+	int              valid = 0;
+	struct iatt      iatt = {0, };
+
+	iatt.ia_uid = uid;
+	iatt.ia_gid = gid;
+	valid = GF_SET_ATTR_UID|GF_SET_ATTR_GID;
+
+	ret = glfs_setattr (fs, path, &iatt, valid, 1);
+
+	return ret;
+}
+
+
+int
+glfs_lchown (struct glfs *fs, const char *path, uid_t uid, gid_t gid)
+{
+	int              ret = -1;
+	int              valid = 0;
+	struct iatt      iatt = {0, };
+
+	iatt.ia_uid = uid;
+	iatt.ia_gid = gid;
+	valid = GF_SET_ATTR_UID|GF_SET_ATTR_GID;
+
+	ret = glfs_setattr (fs, path, &iatt, valid, 0);
+
+	return ret;
+}
+
+
+int
+glfs_fchown (struct glfs_fd *glfd, uid_t uid, gid_t gid)
+{
+	int              ret = -1;
+	int              valid = 0;
+	struct iatt      iatt = {0, };
+
+	iatt.ia_uid = uid;
+	iatt.ia_gid = gid;
+	valid = GF_SET_ATTR_UID|GF_SET_ATTR_GID;
+
+	ret = glfs_fsetattr (glfd, &iatt, valid);
+
+	return ret;
+}
+
+
+int
+glfs_utimens (struct glfs *fs, const char *path, struct timespec times[2])
+{
+	int              ret = -1;
+	int              valid = 0;
+	struct iatt      iatt = {0, };
+
+	iatt.ia_atime = times[0].tv_sec;
+	iatt.ia_atime_nsec = times[0].tv_nsec;
+	iatt.ia_mtime = times[1].tv_sec;
+	iatt.ia_mtime_nsec = times[1].tv_nsec;
+
+	valid = GF_SET_ATTR_ATIME|GF_SET_ATTR_MTIME;
+
+	ret = glfs_setattr (fs, path, &iatt, valid, 1);
+
+	return ret;
+}
+
+
+int
+glfs_lutimens (struct glfs *fs, const char *path, struct timespec times[2])
+{
+	int              ret = -1;
+	int              valid = 0;
+	struct iatt      iatt = {0, };
+
+	iatt.ia_atime = times[0].tv_sec;
+	iatt.ia_atime_nsec = times[0].tv_nsec;
+	iatt.ia_mtime = times[1].tv_sec;
+	iatt.ia_mtime_nsec = times[1].tv_nsec;
+
+	valid = GF_SET_ATTR_ATIME|GF_SET_ATTR_MTIME;
+
+	ret = glfs_setattr (fs, path, &iatt, valid, 0);
+
+	return ret;
+}
+
+
+int
+glfs_futimens (struct glfs_fd *glfd, struct timespec times[2])
+{
+	int              ret = -1;
+	int              valid = 0;
+	struct iatt      iatt = {0, };
+
+	iatt.ia_atime = times[0].tv_sec;
+	iatt.ia_atime_nsec = times[0].tv_nsec;
+	iatt.ia_mtime = times[1].tv_sec;
+	iatt.ia_mtime_nsec = times[1].tv_nsec;
+
+	valid = GF_SET_ATTR_ATIME|GF_SET_ATTR_MTIME;
+
+	ret = glfs_fsetattr (glfd, &iatt, valid);
+
+	return ret;
+}
+
+
+int
+glfs_getxattr_process (void *value, size_t size, dict_t *xattr,
+		       const char *name)
+{
+	data_t *data = NULL;
+	int     ret = -1;
+
+	data = dict_get (xattr, (char *)name);
+	if (!data) {
+		errno = ENODATA;
+		ret = -1;
+		goto out;
+	}
+
+	ret = data->len;
+	if (!value || !size)
+		goto out;
+
+	if (size < ret) {
+		ret = -1;
+		errno = ERANGE;
+		goto out;
+	}
+
+	memcpy (value, data->data, ret);
+out:
+	if (xattr)
+		dict_unref (xattr);
+	return ret;
+}
+
+
+ssize_t
+glfs_getxattr_common (struct glfs *fs, const char *path, const char *name,
+		      void *value, size_t size, int follow)
+{
+	int              ret = -1;
+	xlator_t        *subvol = NULL;
+	loc_t            loc = {0, };
+	struct iatt      iatt = {0, };
+	dict_t          *xattr = NULL;
+	int              reval = 0;
+
+	__glfs_entry_fs (fs);
+
+	subvol = glfs_active_subvol (fs);
+	if (!subvol) {
+		ret = -1;
+		errno = EIO;
+		goto out;
+	}
+retry:
+	if (follow)
+		ret = glfs_resolve (fs, subvol, path, &loc, &iatt, reval);
+	else
+		ret = glfs_lresolve (fs, subvol, path, &loc, &iatt, reval);
+
+	ESTALE_RETRY (ret, errno, reval, &loc, retry);
+
+	if (ret)
+		goto out;
+
+	ret = syncop_getxattr (subvol, &loc, &xattr, name);
+
+	ESTALE_RETRY (ret, errno, reval, &loc, retry);
+
+	if (ret)
+		goto out;
+
+	ret = glfs_getxattr_process (value, size, xattr, name);
+out:
+	loc_wipe (&loc);
+
+	glfs_subvol_done (fs, subvol);
+
+	return ret;
+}
+
+
+ssize_t
+glfs_getxattr (struct glfs *fs, const char *path, const char *name,
+	       void *value, size_t size)
+{
+	return glfs_getxattr_common (fs, path, name, value, size, 1);
+}
+
+
+ssize_t
+glfs_lgetxattr (struct glfs *fs, const char *path, const char *name,
+		void *value, size_t size)
+{
+	return glfs_getxattr_common (fs, path, name, value, size, 0);
+}
+
+
+ssize_t
+glfs_fgetxattr (struct glfs_fd *glfd, const char *name, void *value,
+		size_t size)
+{
+	int              ret = -1;
+	xlator_t        *subvol = NULL;
+	dict_t          *xattr = NULL;
+	fd_t            *fd = NULL;
+
+	__glfs_entry_fd (glfd);
+
+	subvol = glfs_active_subvol (glfd->fs);
+	if (!subvol) {
+		ret = -1;
+		errno = EIO;
+		goto out;
+	}
+
+	fd = glfs_resolve_fd (glfd->fs, subvol, glfd);
+	if (!fd) {
+		ret = -1;
+		errno = EBADFD;
+		goto out;
+	}
+
+	ret = syncop_fgetxattr (subvol, fd, &xattr, name);
+	if (ret)
+		goto out;
+
+	ret = glfs_getxattr_process (value, size, xattr, name);
+out:
+	if (fd)
+		fd_unref (fd);
+
+	glfs_subvol_done (glfd->fs, subvol);
+
+	return ret;
+}
+
+
+int
+glfs_listxattr_process (void *value, size_t size, dict_t *xattr)
+{
+	int     ret = -1;
+
+	ret = dict_keys_join (NULL, 0, xattr, NULL);
+
+	if (!value || !size)
+		goto out;
+
+	if (size < ret) {
+		ret = -1;
+		errno = ERANGE;
+		goto out;
+	}
+
+	dict_keys_join (value, size, xattr, NULL);
+out:
+	if (xattr)
+		dict_unref (xattr);
+	return ret;
+}
+
+
+ssize_t
+glfs_listxattr_common (struct glfs *fs, const char *path, void *value,
+		       size_t size, int follow)
+{
+	int              ret = -1;
+	xlator_t        *subvol = NULL;
+	loc_t            loc = {0, };
+	struct iatt      iatt = {0, };
+	dict_t          *xattr = NULL;
+	int              reval = 0;
+
+	__glfs_entry_fs (fs);
+
+	subvol = glfs_active_subvol (fs);
+	if (!subvol) {
+		ret = -1;
+		errno = EIO;
+		goto out;
+	}
+
+retry:
+	if (follow)
+		ret = glfs_resolve (fs, subvol, path, &loc, &iatt, reval);
+	else
+		ret = glfs_lresolve (fs, subvol, path, &loc, &iatt, reval);
+
+	ESTALE_RETRY (ret, errno, reval, &loc, retry);
+
+	if (ret)
+		goto out;
+
+	ret = syncop_getxattr (subvol, &loc, &xattr, NULL);
+
+	ESTALE_RETRY (ret, errno, reval, &loc, retry);
+
+	if (ret)
+		goto out;
+
+	ret = glfs_listxattr_process (value, size, xattr);
+out:
+	loc_wipe (&loc);
+
+	glfs_subvol_done (fs, subvol);
+
+	return ret;
+}
+
+
+ssize_t
+glfs_listxattr (struct glfs *fs, const char *path, void *value, size_t size)
+{
+	return glfs_listxattr_common (fs, path, value, size, 1);
+}
+
+
+ssize_t
+glfs_llistxattr (struct glfs *fs, const char *path, void *value, size_t size)
+{
+	return glfs_listxattr_common (fs, path, value, size, 0);
+}
+
+
+ssize_t
+glfs_flistxattr (struct glfs_fd *glfd, void *value, size_t size)
+{
+	int              ret = -1;
+	xlator_t        *subvol = NULL;
+	dict_t          *xattr = NULL;
+	fd_t            *fd = NULL;
+
+	__glfs_entry_fd (glfd);
+
+	subvol = glfs_active_subvol (glfd->fs);
+	if (!subvol) {
+		ret = -1;
+		errno = EIO;
+		goto out;
+	}
+
+	fd = glfs_resolve_fd (glfd->fs, subvol, glfd);
+	if (!fd) {
+		ret = -1;
+		errno = EBADFD;
+		goto out;
+	}
+
+	ret = syncop_fgetxattr (subvol, fd, &xattr, NULL);
+	if (ret)
+		goto out;
+
+	ret = glfs_listxattr_process (value, size, xattr);
+out:
+	if (fd)
+		fd_unref (fd);
+
+	glfs_subvol_done (glfd->fs, subvol);
+
+	return ret;
+}
+
+
+dict_t *
+dict_for_key_value (const char *name, const char *value, size_t size)
+{
+	dict_t *xattr = NULL;
+	int     ret = 0;
+
+	xattr = dict_new ();
+	if (!xattr)
+		return NULL;
+
+	ret = dict_set_static_bin (xattr, (char *)name, (void *)value, size);
+	if (ret) {
+		dict_destroy (xattr);
+		xattr = NULL;
+	}
+
+	return xattr;
+}
+
+
+int
+glfs_setxattr_common (struct glfs *fs, const char *path, const char *name,
+		      const void *value, size_t size, int flags, int follow)
+{
+	int              ret = -1;
+	xlator_t        *subvol = NULL;
+	loc_t            loc = {0, };
+	struct iatt      iatt = {0, };
+	dict_t          *xattr = NULL;
+	int              reval = 0;
+
+	__glfs_entry_fs (fs);
+
+	subvol = glfs_active_subvol (fs);
+	if (!subvol) {
+		ret = -1;
+		errno = EIO;
+		goto out;
+	}
+retry:
+	if (follow)
+		ret = glfs_resolve (fs, subvol, path, &loc, &iatt, reval);
+	else
+		ret = glfs_lresolve (fs, subvol, path, &loc, &iatt, reval);
+
+	ESTALE_RETRY (ret, errno, reval, &loc, retry);
+
+	if (ret)
+		goto out;
+
+	xattr = dict_for_key_value (name, value, size);
+	if (!xattr) {
+		ret = -1;
+		errno = ENOMEM;
+		goto out;
+	}
+
+	ret = syncop_setxattr (subvol, &loc, xattr, flags);
+
+	ESTALE_RETRY (ret, errno, reval, &loc, retry);
+
+out:
+	loc_wipe (&loc);
+	if (xattr)
+		dict_unref (xattr);
+
+	glfs_subvol_done (fs, subvol);
+
+	return ret;
+}
+
+
+int
+glfs_setxattr (struct glfs *fs, const char *path, const char *name,
+	       const void *value, size_t size, int flags)
+{
+	return glfs_setxattr_common (fs, path, name, value, size, flags, 1);
+}
+
+
+int
+glfs_lsetxattr (struct glfs *fs, const char *path, const char *name,
+		const void *value, size_t size, int flags)
+{
+	return glfs_setxattr_common (fs, path, name, value, size, flags, 0);
+}
+
+
+int
+glfs_fsetxattr (struct glfs_fd *glfd, const char *name, const void *value,
+		size_t size, int flags)
+{
+	int              ret = -1;
+	xlator_t        *subvol = NULL;
+	dict_t          *xattr = NULL;
+	fd_t            *fd = NULL;
+
+	__glfs_entry_fd (glfd);
+
+	subvol = glfs_active_subvol (glfd->fs);
+	if (!subvol) {
+		ret = -1;
+		errno = EIO;
+		goto out;
+	}
+
+	fd = glfs_resolve_fd (glfd->fs, subvol, glfd);
+	if (!fd) {
+		ret = -1;
+		errno = EBADFD;
+		goto out;
+	}
+
+	xattr = dict_for_key_value (name, value, size);
+	if (!xattr) {
+		ret = -1;
+		errno = ENOMEM;
+		goto out;
+	}
+
+	ret = syncop_fsetxattr (subvol, fd, xattr, flags);
+out:
+	if (xattr)
+		dict_unref (xattr);
+
+	if (fd)
+		fd_unref (fd);
+
+	glfs_subvol_done (glfd->fs, subvol);
+
+	return ret;
+}
+
+
+int
+glfs_removexattr_common (struct glfs *fs, const char *path, const char *name,
+			 int follow)
+{
+	int              ret = -1;
+	xlator_t        *subvol = NULL;
+	loc_t            loc = {0, };
+	struct iatt      iatt = {0, };
+	int              reval = 0;
+
+	__glfs_entry_fs (fs);
+
+	subvol = glfs_active_subvol (fs);
+	if (!subvol) {
+		ret = -1;
+		errno = EIO;
+		goto out;
+	}
+retry:
+	if (follow)
+		ret = glfs_resolve (fs, subvol, path, &loc, &iatt, reval);
+	else
+		ret = glfs_lresolve (fs, subvol, path, &loc, &iatt, reval);
+
+	ESTALE_RETRY (ret, errno, reval, &loc, retry);
+
+	if (ret)
+		goto out;
+
+	ret = syncop_removexattr (subvol, &loc, name);
+
+	ESTALE_RETRY (ret, errno, reval, &loc, retry);
+
+out:
+	loc_wipe (&loc);
+
+	glfs_subvol_done (fs, subvol);
+
+	return ret;
+}
+
+
+int
+glfs_removexattr (struct glfs *fs, const char *path, const char *name)
+{
+	return glfs_removexattr_common (fs, path, name, 1);
+}
+
+
+int
+glfs_lremovexattr (struct glfs *fs, const char *path, const char *name)
+{
+	return glfs_removexattr_common (fs, path, name, 0);
+}
+
+
+int
+glfs_fremovexattr (struct glfs_fd *glfd, const char *name)
+{
+	int              ret = -1;
+	xlator_t        *subvol = NULL;
+	fd_t            *fd = NULL;
+
+	__glfs_entry_fd (glfd);
+
+	subvol = glfs_active_subvol (glfd->fs);
+	if (!subvol) {
+		ret = -1;
+		errno = EIO;
+		goto out;
+	}
+
+	fd = glfs_resolve_fd (glfd->fs, subvol, glfd);
+	if (!fd) {
+		ret = -1;
+		errno = EBADFD;
+		goto out;
+	}
+
+	ret = syncop_fremovexattr (subvol, fd, name);
+out:
+	if (fd)
+		fd_unref (fd);
+
+	glfs_subvol_done (glfd->fs, subvol);
+
+	return ret;
+}
+
+
+int
+glfs_fallocate (struct glfs_fd *glfd, int keep_size, off_t offset, size_t len)
+{
+	int              ret = -1;
+	xlator_t        *subvol = NULL;
+	fd_t		*fd = NULL;
+
+	__glfs_entry_fd (glfd);
+
+	subvol = glfs_active_subvol (glfd->fs);
+	if (!subvol) {
+		ret = -1;
+		errno = EIO;
+		goto out;
+	}
+
+	fd = glfs_resolve_fd (glfd->fs, subvol, glfd);
+	if (!fd) {
+		ret = -1;
+		errno = EBADFD;
+		goto out;
+	}
+
+	ret = syncop_fallocate (subvol, fd, keep_size, offset, len);
+out:
+	if (fd)
+		fd_unref(fd);
+
+	glfs_subvol_done (glfd->fs, subvol);
+
+	return ret;
+}
+
+
+int
+glfs_discard (struct glfs_fd *glfd, off_t offset, size_t len)
+{
+	int              ret = -1;
+	xlator_t        *subvol = NULL;
+	fd_t		*fd = NULL;
+
+	__glfs_entry_fd (glfd);
+
+	subvol = glfs_active_subvol (glfd->fs);
+	if (!subvol) {
+		ret = -1;
+		errno = EIO;
+		goto out;
+	}
+
+	fd = glfs_resolve_fd (glfd->fs, subvol, glfd);
+	if (!fd) {
+		ret = -1;
+		errno = EBADFD;
+		goto out;
+	}
+
+	ret = syncop_discard (subvol, fd, offset, len);
+out:
+	if (fd)
+		fd_unref(fd);
+
+	glfs_subvol_done (glfd->fs, subvol);
+
+	return ret;
+}
+
+int
+glfs_zerofill (struct glfs_fd *glfd, off_t offset, size_t len)
+{
+        int               ret             = -1;
+        xlator_t         *subvol          = NULL;
+        fd_t             *fd              = NULL;
+
+        __glfs_entry_fd (glfd);
+
+        subvol = glfs_active_subvol (glfd->fs);
+        if (!subvol) {
+                errno = EIO;
+                goto out;
+        }
+
+        fd = glfs_resolve_fd (glfd->fs, subvol, glfd);
+        if (!fd) {
+                errno = EBADFD;
+                goto out;
+        }
+
+        ret = syncop_zerofill (subvol, fd, offset, len);
+out:
+        if (fd)
+                fd_unref(fd);
+
+        glfs_subvol_done (glfd->fs, subvol);
+
+        return ret;
+}
+
+int
+glfs_chdir (struct glfs *fs, const char *path)
+{
+	int              ret = -1;
+	xlator_t        *subvol = NULL;
+	loc_t            loc = {0, };
+	struct iatt      iatt = {0, };
+	int              reval = 0;
+
+	__glfs_entry_fs (fs);
+
+	subvol = glfs_active_subvol (fs);
+	if (!subvol) {
+		ret = -1;
+		errno = EIO;
+		goto out;
+	}
+retry:
+	ret = glfs_resolve (fs, subvol, path, &loc, &iatt, reval);
+
+	ESTALE_RETRY (ret, errno, reval, &loc, retry);
+
+	if (ret)
+		goto out;
+
+	if (!IA_ISDIR (iatt.ia_type)) {
+		ret = -1;
+		errno = ENOTDIR;
+		goto out;
+	}
+
+	glfs_cwd_set (fs, loc.inode);
+
+out:
+	loc_wipe (&loc);
+
+	glfs_subvol_done (fs, subvol);
+
+	return ret;
+}
+
+
+int
+glfs_fchdir (struct glfs_fd *glfd)
+{
+	int       ret = -1;
+	inode_t  *inode = NULL;
+	xlator_t *subvol = NULL;
+	fd_t     *fd = NULL;
+
+	__glfs_entry_fd (glfd);
+
+	subvol = glfs_active_subvol (glfd->fs);
+	if (!subvol) {
+		ret = -1;
+		errno = EIO;
+		goto out;
+	}
+
+	fd = glfs_resolve_fd (glfd->fs, subvol, glfd);
+	if (!fd) {
+		ret = -1;
+		errno = EBADFD;
+		goto out;
+	}
+
+	inode = fd->inode;
+
+	if (!IA_ISDIR (inode->ia_type)) {
+		ret = -1;
+		errno = ENOTDIR;
+		goto out;
+	}
+
+	glfs_cwd_set (glfd->fs, inode);
+	ret = 0;
+out:
+	if (fd)
+		fd_unref (fd);
+
+	glfs_subvol_done (glfd->fs, subvol);
+
+	return ret;
+}
+
+
+char *
+glfs_realpath (struct glfs *fs, const char *path, char *resolved_path)
+{
+	int              ret = -1;
+	char            *retpath = NULL;
+	char            *allocpath = NULL;
+	xlator_t        *subvol = NULL;
+	loc_t            loc = {0, };
+	struct iatt      iatt = {0, };
+	int              reval = 0;
+
+	__glfs_entry_fs (fs);
+
+	if (resolved_path)
+		retpath = resolved_path;
+	else
+		retpath = allocpath = malloc (PATH_MAX + 1);
+
+	if (!retpath) {
+		ret = -1;
+		errno = ENOMEM;
+		goto out;
+	}
+
+	subvol = glfs_active_subvol (fs);
+	if (!subvol) {
+		ret = -1;
+		errno = EIO;
+		goto out;
+	}
+retry:
+	ret = glfs_resolve (fs, subvol, path, &loc, &iatt, reval);
+
+	ESTALE_RETRY (ret, errno, reval, &loc, retry);
+
+	if (ret)
+		goto out;
+
+	if (loc.path) {
+		strncpy (retpath, loc.path, PATH_MAX);
+		retpath[PATH_MAX] = 0;
+	}
+
+out:
+	loc_wipe (&loc);
+
+	if (ret == -1) {
+		if (allocpath)
+			free (allocpath);
+		retpath = NULL;
+	}
+
+	glfs_subvol_done (fs, subvol);
+
+	return retpath;
+}
+
+
+char *
+glfs_getcwd (struct glfs *fs, char *buf, size_t n)
+{
+	int              ret = -1;
+	inode_t         *inode = NULL;
+	char            *path = NULL;
+
+	__glfs_entry_fs (fs);
+
+	if (!buf || n < 2) {
+		ret = -1;
+		errno = EINVAL;
+		goto out;
+	}
+
+	inode = glfs_cwd_get (fs);
+
+	if (!inode) {
+		strncpy (buf, "/", n);
+		ret = 0;
+		goto out;
+	}
+
+	ret = inode_path (inode, 0, &path);
+	if (n <= ret) {
+		ret = -1;
+		errno = ERANGE;
+		goto out;
+	}
+
+	strncpy (buf, path, n);
+	ret = 0;
+out:
+	GF_FREE (path);
+
+	if (inode)
+		inode_unref (inode);
+
+	if (ret < 0)
+		return NULL;
+
+	return buf;
+}
+
+
+static void
+gf_flock_to_flock (struct gf_flock *gf_flock, struct flock *flock)
+{
+	flock->l_type   = gf_flock->l_type;
+	flock->l_whence = gf_flock->l_whence;
+	flock->l_start  = gf_flock->l_start;
+	flock->l_len    = gf_flock->l_len;
+	flock->l_pid    = gf_flock->l_pid;
+}
+
+
+static void
+gf_flock_from_flock (struct gf_flock *gf_flock, struct flock *flock)
+{
+	gf_flock->l_type   = flock->l_type;
+	gf_flock->l_whence = flock->l_whence;
+	gf_flock->l_start  = flock->l_start;
+	gf_flock->l_len    = flock->l_len;
+	gf_flock->l_pid    = flock->l_pid;
+}
+
+
+int
+glfs_posix_lock (struct glfs_fd *glfd, int cmd, struct flock *flock)
+{
+	int              ret = -1;
+	xlator_t        *subvol = NULL;
+	struct gf_flock  gf_flock = {0, };
+	struct gf_flock  saved_flock = {0, };
+	fd_t            *fd = NULL;
+
+	__glfs_entry_fd (glfd);
+
+	subvol = glfs_active_subvol (glfd->fs);
+	if (!subvol) {
+		ret = -1;
+		errno = EIO;
+		goto out;
+	}
+
+	fd = glfs_resolve_fd (glfd->fs, subvol, glfd);
+	if (!fd) {
+		ret = -1;
+		errno = EBADFD;
+		goto out;
+	}
+
+	gf_flock_from_flock (&gf_flock, flock);
+	gf_flock_from_flock (&saved_flock, flock);
+	ret = syncop_lk (subvol, fd, cmd, &gf_flock);
+	gf_flock_to_flock (&gf_flock, flock);
+
+	if (ret == 0 && (cmd == F_SETLK || cmd == F_SETLKW))
+		fd_lk_insert_and_merge (fd, cmd, &saved_flock);
+out:
+	if (fd)
+		fd_unref (fd);
+
+	glfs_subvol_done (glfd->fs, subvol);
+
+	return ret;
+}
+
+
+struct glfs_fd *
+glfs_dup (struct glfs_fd *glfd)
+{
+	xlator_t  *subvol = NULL;
+	fd_t      *fd = NULL;
+	glfs_fd_t *dupfd = NULL;
+	struct glfs *fs = NULL;
+
+	__glfs_entry_fd (glfd);
+
+	fs = glfd->fs;
+	subvol = glfs_active_subvol (fs);
+	if (!subvol) {
+		errno = EIO;
+		goto out;
+	}
+
+	fd = glfs_resolve_fd (fs, subvol, glfd);
+	if (!fd) {
+		errno = EBADFD;
+		goto out;
+	}
+
+	dupfd = glfs_fd_new (fs);
+	if (!dupfd) {
+		errno = ENOMEM;
+		goto out;
+	}
+
+	dupfd->fd = fd_ref (fd);
+out:
+	if (fd)
+		fd_unref (fd);
+	if (dupfd)
+		glfs_fd_bind (dupfd);
+
+	glfs_subvol_done (fs, subvol);
+
+	return dupfd;
+}
diff --git a/api/src/glfs-handleops.c b/api/src/glfs-handleops.c
new file mode 100644
index 000000000..9c707a619
--- /dev/null
+++ b/api/src/glfs-handleops.c
@@ -0,0 +1,1278 @@
+/*
+ *  Copyright (c) 2013 Red Hat, Inc. <http://www.redhat.com>
+ *  This file is part of GlusterFS.
+ *
+ *  This file is licensed to you under your choice of the GNU Lesser
+ *  General Public License, version 3 or any later version (LGPLv3 or
+ *  later), or the GNU General Public License, version 2 (GPLv2), in all
+ *  cases as published by the Free Software Foundation.
+ */
+
+
+#include "glfs-internal.h"
+#include "glfs-mem-types.h"
+#include "syncop.h"
+#include "glfs.h"
+#include "glfs-handles.h"
+
+static void
+glfs_iatt_from_stat (struct stat *stat, int valid, struct iatt *iatt,
+		     int *glvalid)
+{
+	/* validate in args */
+	if ((stat == NULL) || (iatt == NULL) || (glvalid == NULL)) {
+		errno = EINVAL;
+		return;
+	}
+
+	*glvalid = 0;
+
+	if (valid & GFAPI_SET_ATTR_MODE) {
+		iatt->ia_prot = ia_prot_from_st_mode (stat->st_mode);
+		*glvalid |= GF_SET_ATTR_MODE;
+	}
+
+	if (valid & GFAPI_SET_ATTR_UID) {
+		iatt->ia_uid = stat->st_uid;
+		*glvalid |= GF_SET_ATTR_UID;
+	}
+
+	if (valid & GFAPI_SET_ATTR_GID) {
+		iatt->ia_gid = stat->st_gid;
+		*glvalid |= GF_SET_ATTR_GID;
+	}
+
+	if (valid & GFAPI_SET_ATTR_ATIME) {
+		iatt->ia_atime = stat->st_atime;
+		iatt->ia_atime_nsec = ST_ATIM_NSEC (stat);
+		*glvalid |= GF_SET_ATTR_ATIME;
+	}
+
+	if (valid & GFAPI_SET_ATTR_MTIME) {
+		iatt->ia_mtime = stat->st_mtime;
+		iatt->ia_mtime_nsec = ST_MTIM_NSEC (stat);
+		*glvalid |= GF_SET_ATTR_MTIME;
+	}
+
+	return;
+}
+
+struct glfs_object *
+glfs_h_lookupat (struct glfs *fs, struct glfs_object *parent,
+		 const char *path, struct stat *stat)
+{
+	int                      ret = 0;
+	xlator_t                *subvol = NULL;
+	inode_t                 *inode = NULL;
+	struct iatt              iatt = {0, };
+	struct glfs_object      *object = NULL;
+	loc_t                    loc = {0, };
+
+	/* validate in args */
+	if ((fs == NULL) || (path == NULL)) {
+		errno = EINVAL;
+		return NULL;
+	}
+
+	__glfs_entry_fs (fs);
+
+	/* get the active volume */
+	subvol = glfs_active_subvol (fs);
+	if (!subvol) {
+		errno = EIO;
+		goto out;
+	}
+
+	/* get/refresh the in arg objects inode in correlation to the xlator */
+	if (parent) {
+		inode = glfs_resolve_inode (fs, subvol, parent);
+		if (!inode) {
+			errno = ESTALE;
+			goto out;
+		}
+	}
+
+	/* fop/op */
+	ret = glfs_resolve_at (fs, subvol, inode, path, &loc, &iatt,
+			       0 /*TODO: links? */, 0);
+
+	/* populate out args */
+	if (!ret) {
+		if (stat)
+			glfs_iatt_to_stat (fs, &iatt, stat);
+
+		ret = glfs_create_object (&loc, &object);
+	}
+
+out:
+	loc_wipe (&loc);
+
+	if (inode)
+		inode_unref (inode);
+
+	glfs_subvol_done (fs, subvol);
+
+	return object;
+}
+
+int
+glfs_h_stat (struct glfs *fs, struct glfs_object *object, struct stat *stat)
+{
+	int              ret = -1;
+	xlator_t        *subvol = NULL;
+	inode_t         *inode = NULL;
+	loc_t            loc = {0, };
+	struct iatt      iatt = {0, };
+
+	/* validate in args */
+	if ((fs == NULL) || (object == NULL)) {
+		errno = EINVAL;
+		return -1;
+	}
+
+	__glfs_entry_fs (fs);
+
+	/* get the active volume */
+	subvol = glfs_active_subvol (fs);
+	if (!subvol) {
+		ret = -1;
+		errno = EIO;
+		goto out;
+	}
+
+	/* get/refresh the in arg objects inode in correlation to the xlator */
+	inode = glfs_resolve_inode (fs, subvol, object);
+	if (!inode) {
+		errno = ESTALE;
+		goto out;
+	}
+
+	/* populate loc */
+	GLFS_LOC_FILL_INODE (inode, loc, out);
+
+	/* fop/op */
+	ret = syncop_stat (subvol, &loc, &iatt);
+
+	/* populate out args */
+	if (!ret && stat) {
+		glfs_iatt_to_stat (fs, &iatt, stat);
+	}
+out:
+	loc_wipe (&loc);
+
+	if (inode)
+		inode_unref (inode);
+
+	glfs_subvol_done (fs, subvol);
+
+	return ret;
+}
+
+int
+glfs_h_getattrs (struct glfs *fs, struct glfs_object *object, struct stat *stat)
+{
+	int                      ret = 0;
+	xlator_t                *subvol = NULL;
+	inode_t                 *inode = NULL;
+	struct iatt              iatt = {0, };
+
+	/* validate in args */
+	if ((fs == NULL) || (object == NULL)) {
+		errno = EINVAL;
+		return -1;
+	}
+
+	__glfs_entry_fs (fs);
+
+	/* get the active volume */
+	subvol = glfs_active_subvol (fs);
+	if (!subvol) {
+		ret = -1;
+		errno = EIO;
+		goto out;
+	}
+
+	/* get/refresh the in arg objects inode in correlation to the xlator */
+	inode = glfs_resolve_inode (fs, subvol, object);
+	if (!inode) {
+		errno = ESTALE;
+		goto out;
+	}
+
+	/* fop/op */
+	ret = glfs_resolve_base (fs, subvol, inode, &iatt);
+
+	/* populate out args */
+	if (!ret && stat) {
+		glfs_iatt_to_stat (fs, &iatt, stat);
+	}
+
+out:
+	if (inode)
+		inode_unref (inode);
+
+	glfs_subvol_done (fs, subvol);
+
+	return ret;
+}
+
+int
+glfs_h_setattrs (struct glfs *fs, struct glfs_object *object, struct stat *stat,
+		 int valid)
+{
+	int              ret = -1;
+	xlator_t        *subvol = NULL;
+	inode_t         *inode = NULL;
+	loc_t            loc = {0, };
+	struct iatt      iatt = {0, };
+	int              glvalid = 0;
+
+	/* validate in args */
+	if ((fs == NULL) || (object == NULL) || (stat == NULL)) {
+		errno = EINVAL;
+		return -1;
+	}
+
+	__glfs_entry_fs (fs);
+
+	/* get the active volume */
+	subvol = glfs_active_subvol (fs);
+	if (!subvol) {
+		ret = -1;
+		errno = EIO;
+		goto out;
+	}
+
+	/* get/refresh the in arg objects inode in correlation to the xlator */
+	inode = glfs_resolve_inode (fs, subvol, object);
+	if (!inode) {
+		errno = ESTALE;
+		goto out;
+	}
+
+	/* map valid masks from in args */
+	glfs_iatt_from_stat (stat, valid, &iatt, &glvalid);
+
+	/* populate loc */
+	GLFS_LOC_FILL_INODE (inode, loc, out);
+
+	/* fop/op */
+	ret = syncop_setattr (subvol, &loc, &iatt, glvalid, 0, 0);
+out:
+	loc_wipe (&loc);
+
+	if (inode)
+		inode_unref (inode);
+
+	glfs_subvol_done (fs, subvol);
+
+	return ret;
+}
+
+struct glfs_fd *
+glfs_h_open (struct glfs *fs, struct glfs_object *object, int flags)
+{
+	int              ret = -1;
+	struct glfs_fd  *glfd = NULL;
+	xlator_t        *subvol = NULL;
+	inode_t         *inode = NULL;
+	loc_t            loc = {0, };
+
+	/* validate in args */
+	if ((fs == NULL) || (object == NULL)) {
+		errno = EINVAL;
+		return NULL;
+	}
+
+	__glfs_entry_fs (fs);
+
+	/* get the active volume */
+	subvol = glfs_active_subvol (fs);
+	if (!subvol) {
+		errno = EIO;
+		goto out;
+	}
+
+	/* get/refresh the in arg objects inode in correlation to the xlator */
+	inode = glfs_resolve_inode (fs, subvol, object);
+	if (!inode) {
+		errno = ESTALE;
+		goto out;
+	}
+
+	/* check types to open */
+	if (IA_ISDIR (inode->ia_type)) {
+		ret = -1;
+		errno = EISDIR;
+		goto out;
+	}
+
+	if (!IA_ISREG (inode->ia_type)) {
+		ret = -1;
+		errno = EINVAL;
+		goto out;
+	}
+
+	glfd = glfs_fd_new (fs);
+	if (!glfd) {
+		errno = ENOMEM;
+		goto out;
+	}
+
+	glfd->fd = fd_create (inode, getpid());
+	if (!glfd->fd) {
+		ret = -1;
+		errno = ENOMEM;
+		goto out;
+	}
+
+	/* populate loc */
+	GLFS_LOC_FILL_INODE (inode, loc, out);
+
+	/* fop/op */
+	ret = syncop_open (subvol, &loc, flags, glfd->fd);
+
+out:
+	loc_wipe (&loc);
+
+	if (inode)
+		inode_unref (inode);
+
+	if (ret && glfd) {
+		glfs_fd_destroy (glfd);
+		glfd = NULL;
+	} else {
+		glfd->fd->flags = flags;
+		fd_bind (glfd->fd);
+		glfs_fd_bind (glfd);
+	}
+
+	glfs_subvol_done (fs, subvol);
+
+	return glfd;
+}
+
+struct glfs_object *
+glfs_h_creat (struct glfs *fs, struct glfs_object *parent, const char *path,
+	      int flags, mode_t mode, struct stat *stat)
+{
+	int                 ret = -1;
+	struct glfs_fd     *glfd = NULL;
+	xlator_t           *subvol = NULL;
+	inode_t            *inode = NULL;
+	loc_t               loc = {0, };
+	struct iatt         iatt = {0, };
+	uuid_t              gfid;
+	dict_t             *xattr_req = NULL;
+	struct glfs_object *object = NULL;
+
+	/* validate in args */
+	if ((fs == NULL) || (parent == NULL) || (path == NULL)) {
+		errno = EINVAL;
+		return NULL;
+	}
+
+	__glfs_entry_fs (fs);
+
+	/* get the active volume */
+	subvol = glfs_active_subvol (fs);
+	if (!subvol) {
+		ret = -1;
+		errno = EIO;
+		goto out;
+	}
+
+	/* get/refresh the in arg objects inode in correlation to the xlator */
+	inode = glfs_resolve_inode (fs, subvol, parent);
+	if (!inode) {
+		errno = ESTALE;
+		goto out;
+	}
+
+	xattr_req = dict_new ();
+	if (!xattr_req) {
+		ret = -1;
+		errno = ENOMEM;
+		goto out;
+	}
+
+	uuid_generate (gfid);
+	ret = dict_set_static_bin (xattr_req, "gfid-req", gfid, 16);
+	if (ret) {
+		ret = -1;
+		errno = ENOMEM;
+		goto out;
+	}
+
+	GLFS_LOC_FILL_PINODE (inode, loc, ret, errno, out, path);
+
+	glfd = glfs_fd_new (fs);
+	if (!glfd)
+		goto out;
+
+	glfd->fd = fd_create (loc.inode, getpid());
+	if (!glfd->fd) {
+		ret = -1;
+		errno = ENOMEM;
+		goto out;
+	}
+
+	/* fop/op */
+	ret = syncop_create (subvol, &loc, flags, mode, glfd->fd,
+			     xattr_req, &iatt);
+
+	/* populate out args */
+	if (ret == 0) {
+		/* TODO: If the inode existed in the cache (say file already
+		   exists), then the glfs_loc_link will not update the
+		   loc.inode, as a result we will have a 0000 GFID that we
+		   would copy out to the object, this needs to be fixed.
+		*/
+		ret = glfs_loc_link (&loc, &iatt);
+		if (ret != 0) {
+			goto out;
+		}
+
+		if (stat)
+			glfs_iatt_to_stat (fs, &iatt, stat);
+
+		ret = glfs_create_object (&loc, &object);
+	}
+
+out:
+	if (ret && object != NULL) {
+		glfs_h_close (object);
+		object = NULL;
+	}
+
+	loc_wipe(&loc);
+
+	if (inode)
+		inode_unref (inode);
+
+	if (xattr_req)
+		dict_unref (xattr_req);
+
+	if (glfd) {
+		glfs_fd_destroy (glfd);
+		glfd = NULL;
+	}
+
+	glfs_subvol_done (fs, subvol);
+
+	return object;
+}
+
+struct glfs_object *
+glfs_h_mkdir (struct glfs *fs, struct glfs_object *parent, const char *path,
+	      mode_t mode, struct stat *stat)
+{
+	int                 ret = -1;
+	xlator_t           *subvol = NULL;
+	inode_t            *inode = NULL;
+	loc_t               loc = {0, };
+	struct iatt         iatt = {0, };
+	uuid_t              gfid;
+	dict_t             *xattr_req = NULL;
+	struct glfs_object *object = NULL;
+
+	/* validate in args */
+	if ((fs == NULL) || (parent == NULL) || (path == NULL)) {
+		errno = EINVAL;
+		return NULL;
+	}
+
+	__glfs_entry_fs (fs);
+
+	/* get the active volume */
+	subvol = glfs_active_subvol (fs);
+	if (!subvol) {
+		ret = -1;
+		errno = EIO;
+		goto out;
+	}
+
+	/* get/refresh the in arg objects inode in correlation to the xlator */
+	inode = glfs_resolve_inode (fs, subvol, parent);
+	if (!inode) {
+		errno = ESTALE;
+		goto out;
+	}
+
+	xattr_req = dict_new ();
+	if (!xattr_req) {
+		ret = -1;
+		errno = ENOMEM;
+		goto out;
+	}
+
+	uuid_generate (gfid);
+	ret = dict_set_static_bin (xattr_req, "gfid-req", gfid, 16);
+	if (ret) {
+		ret = -1;
+		errno = ENOMEM;
+		goto out;
+	}
+
+	GLFS_LOC_FILL_PINODE (inode, loc, ret, errno, out, path);
+
+	/* fop/op */
+	ret = syncop_mkdir (subvol, &loc, mode, xattr_req, &iatt);
+
+	/* populate out args */
+	if ( ret == 0 )  {
+		ret = glfs_loc_link (&loc, &iatt);
+		if (ret != 0) {
+			goto out;
+		}
+
+		if (stat)
+			glfs_iatt_to_stat (fs, &iatt, stat);
+
+		ret = glfs_create_object (&loc, &object);
+	}
+
+out:
+	if (ret && object != NULL) {
+		glfs_h_close (object);
+		object = NULL;
+	}
+
+	loc_wipe(&loc);
+
+	if (inode)
+		inode_unref (inode);
+
+	if (xattr_req)
+		dict_unref (xattr_req);
+
+	glfs_subvol_done (fs, subvol);
+
+	return object;
+}
+
+struct glfs_object *
+glfs_h_mknod (struct glfs *fs, struct glfs_object *parent, const char *path,
+	      mode_t mode, dev_t dev, struct stat *stat)
+{
+	int                 ret = -1;
+	xlator_t           *subvol = NULL;
+	inode_t            *inode = NULL;
+	loc_t               loc = {0, };
+	struct iatt         iatt = {0, };
+	uuid_t              gfid;
+	dict_t             *xattr_req = NULL;
+	struct glfs_object *object = NULL;
+
+	/* validate in args */
+	if ((fs == NULL) || (parent == NULL) || (path == NULL)) {
+		errno = EINVAL;
+		return NULL;
+	}
+
+	__glfs_entry_fs (fs);
+
+	/* get the active volume */
+	subvol = glfs_active_subvol (fs);
+	if (!subvol) {
+		ret = -1;
+		errno = EIO;
+		goto out;
+	}
+
+	/* get/refresh the in arg objects inode in correlation to the xlator */
+	inode = glfs_resolve_inode (fs, subvol, parent);
+	if (!inode) {
+		errno = ESTALE;
+		goto out;
+	}
+
+	xattr_req = dict_new ();
+	if (!xattr_req) {
+		ret = -1;
+		errno = ENOMEM;
+		goto out;
+	}
+
+	uuid_generate (gfid);
+	ret = dict_set_static_bin (xattr_req, "gfid-req", gfid, 16);
+	if (ret) {
+		ret = -1;
+		errno = ENOMEM;
+		goto out;
+	}
+
+	GLFS_LOC_FILL_PINODE (inode, loc, ret, errno, out, path);
+
+	/* fop/op */
+	ret = syncop_mknod (subvol, &loc, mode, dev, xattr_req, &iatt);
+
+	/* populate out args */
+	if (ret == 0) {
+		ret = glfs_loc_link (&loc, &iatt);
+		if (ret != 0) {
+			goto out;
+		}
+
+		if (stat)
+			glfs_iatt_to_stat (fs, &iatt, stat);
+
+		ret = glfs_create_object (&loc, &object);
+	}
+out:
+	if (ret && object != NULL) {
+		glfs_h_close (object);
+		object = NULL;
+	}
+
+	loc_wipe(&loc);
+
+	if (inode)
+		inode_unref (inode);
+
+	if (xattr_req)
+		dict_unref (xattr_req);
+
+	glfs_subvol_done (fs, subvol);
+
+	return object;
+}
+
+int
+glfs_h_unlink (struct glfs *fs, struct glfs_object *parent, const char *path)
+{
+	int                 ret = -1;
+	xlator_t           *subvol = NULL;
+	inode_t            *inode = NULL;
+	loc_t               loc = {0, };
+
+	/* validate in args */
+	if ((fs == NULL) || (parent == NULL) || (path == NULL)) {
+		errno = EINVAL;
+		return -1;
+	}
+
+	__glfs_entry_fs (fs);
+
+	/* get the active volume */
+	subvol = glfs_active_subvol (fs);
+	if ( !subvol ) {
+		ret = -1;
+		errno = EIO;
+		goto out;
+	}
+
+	/* get/refresh the in arg objects inode in correlation to the xlator */
+	inode = glfs_resolve_inode (fs, subvol, parent);
+	if (!inode) {
+		errno = ESTALE;
+		goto out;
+	}
+
+	ret = glfs_resolve_at (fs, subvol, inode, path, &loc, NULL, 0 , 0);
+	if (ret != 0) {
+		goto out;
+	}
+
+	if (!IA_ISDIR(loc.inode->ia_type)) {
+		ret = syncop_unlink (subvol, &loc);
+		if (ret != 0) {
+			goto out;
+		}
+	} else {
+		ret = syncop_rmdir (subvol, &loc);
+		if (ret != 0) {
+			goto out;
+		}
+	}
+
+	if (ret == 0)
+		ret = glfs_loc_unlink (&loc);
+
+out:
+	loc_wipe (&loc);
+
+	if (inode)
+		inode_unref (inode);
+
+	glfs_subvol_done (fs, subvol);
+
+	return ret;
+}
+
+struct glfs_fd *
+glfs_h_opendir (struct glfs *fs, struct glfs_object *object)
+{
+	int              ret = -1;
+	struct glfs_fd  *glfd = NULL;
+	xlator_t        *subvol = NULL;
+	inode_t         *inode = NULL;
+	loc_t            loc = {0, };
+
+	/* validate in args */
+	if ((fs == NULL) || (object == NULL)) {
+		errno = EINVAL;
+		return NULL;
+	}
+
+	__glfs_entry_fs (fs);
+
+	/* get the active volume */
+	subvol = glfs_active_subvol (fs);
+	if (!subvol) {
+		ret = -1;
+		errno = EIO;
+		goto out;
+	}
+
+	/* get/refresh the in arg objects inode in correlation to the xlator */
+	inode = glfs_resolve_inode (fs, subvol, object);
+	if (!inode) {
+		errno = ESTALE;
+		goto out;
+	}
+
+	if (!IA_ISDIR (inode->ia_type)) {
+		ret = -1;
+		errno = ENOTDIR;
+		goto out;
+	}
+
+	glfd = glfs_fd_new (fs);
+	if (!glfd)
+		goto out;
+
+	INIT_LIST_HEAD (&glfd->entries);
+
+	glfd->fd = fd_create (inode, getpid());
+	if (!glfd->fd) {
+		ret = -1;
+		errno = ENOMEM;
+		goto out;
+	}
+
+	GLFS_LOC_FILL_INODE (inode, loc, out);
+
+	/* fop/op */
+	ret = syncop_opendir (subvol, &loc, glfd->fd);
+
+out:
+	loc_wipe (&loc);
+
+	if (inode)
+		inode_unref (inode);
+
+	if (ret && glfd) {
+		glfs_fd_destroy (glfd);
+		glfd = NULL;
+	} else {
+		fd_bind (glfd->fd);
+		glfs_fd_bind (glfd);
+	}
+
+	glfs_subvol_done (fs, subvol);
+
+	return glfd;
+}
+
+ssize_t
+glfs_h_extract_handle (struct glfs_object *object, unsigned char *handle,
+		       int len)
+{
+	ssize_t ret = -1;
+
+	/* validate in args */
+	if (object == NULL) {
+		errno = EINVAL;
+		goto out;
+	}
+
+	if (!handle || !len) {
+		ret = GFAPI_HANDLE_LENGTH;
+		goto out;
+	}
+
+	if (len < GFAPI_HANDLE_LENGTH)
+	{
+		errno = ERANGE;
+		goto out;
+	}
+
+	memcpy (handle, object->gfid, GFAPI_HANDLE_LENGTH);
+
+	ret = GFAPI_HANDLE_LENGTH;
+
+out:
+	return ret;
+}
+
+struct glfs_object *
+glfs_h_create_from_handle (struct glfs *fs, unsigned char *handle, int len,
+			   struct stat *stat)
+{
+	loc_t               loc = {0, };
+	int                 ret = -1;
+	struct iatt         iatt = {0, };
+	inode_t            *newinode = NULL;
+	xlator_t           *subvol = NULL;
+	struct glfs_object *object = NULL;
+
+	/* validate in args */
+	if ((fs == NULL) || (handle == NULL) || (len != GFAPI_HANDLE_LENGTH)) {
+		errno = EINVAL;
+		return NULL;
+	}
+
+	__glfs_entry_fs (fs);
+
+	/* get the active volume */
+	subvol = glfs_active_subvol (fs);
+	if (!subvol) {
+		errno = EIO;
+		goto out;
+	}
+
+	memcpy (loc.gfid, handle, GFAPI_HANDLE_LENGTH);
+
+	newinode = inode_find (subvol->itable, loc.gfid);
+	if (newinode)
+		loc.inode = newinode;
+	else {
+		loc.inode = inode_new (subvol->itable);
+		if (!loc.inode) {
+			errno = ENOMEM;
+			goto out;
+		}
+	}
+
+	ret = syncop_lookup (subvol, &loc, 0, &iatt, 0, 0);
+	if (ret) {
+		gf_log (subvol->name, GF_LOG_WARNING,
+			"inode refresh of %s failed: %s",
+			uuid_utoa (loc.gfid), strerror (errno));
+		goto out;
+	}
+
+	newinode = inode_link (loc.inode, 0, 0, &iatt);
+	if (newinode)
+		inode_lookup (newinode);
+	else {
+		gf_log (subvol->name, GF_LOG_WARNING,
+			"inode linking of %s failed: %s",
+			uuid_utoa (loc.gfid), strerror (errno));
+		errno = EINVAL;
+		goto out;
+	}
+
+	/* populate stat */
+	if (stat)
+		glfs_iatt_to_stat (fs, &iatt, stat);
+
+	object = GF_CALLOC (1, sizeof(struct glfs_object),
+			    glfs_mt_glfs_object_t);
+	if (object == NULL) {
+		errno = ENOMEM;
+		ret = -1;
+		goto out;
+	}
+
+	/* populate the return object */
+	object->inode = newinode;
+	uuid_copy (object->gfid, object->inode->gfid);
+
+out:
+	/* TODO: Check where the inode ref is being held? */
+	loc_wipe (&loc);
+
+	glfs_subvol_done (fs, subvol);
+
+	return object;
+}
+
+int
+glfs_h_close (struct glfs_object *object)
+{
+	/* Release the held reference */
+	inode_unref (object->inode);
+	GF_FREE (object);
+
+	return 0;
+}
+
+int
+glfs_h_truncate (struct glfs *fs, struct glfs_object *object, off_t offset)
+{
+	loc_t               loc = {0, };
+	int                 ret = -1;
+	xlator_t           *subvol = NULL;
+	inode_t            *inode = NULL;
+
+	/* validate in args */
+	if ((fs == NULL) || (object == NULL)) {
+		errno = EINVAL;
+		return -1;
+	}
+
+	__glfs_entry_fs (fs);
+
+	/* get the active volume */
+	subvol = glfs_active_subvol (fs);
+	if (!subvol) {
+		ret = -1;
+		errno = EIO;
+		goto out;
+	}
+
+	/* get/refresh the in arg objects inode in correlation to the xlator */
+	inode = glfs_resolve_inode (fs, subvol, object);
+	if (!inode) {
+		errno = ESTALE;
+		goto out;
+	}
+
+	GLFS_LOC_FILL_INODE (inode, loc, out);
+
+	/* fop/op */
+	ret = syncop_truncate (subvol, &loc, (off_t)offset);
+
+	/* populate out args */
+	if (ret == 0)
+		ret = glfs_loc_unlink (&loc);
+
+out:
+	loc_wipe (&loc);
+
+	if (inode)
+		inode_unref (inode);
+
+	glfs_subvol_done (fs, subvol);
+
+	return ret;
+}
+
+struct glfs_object *
+glfs_h_symlink (struct glfs *fs, struct glfs_object *parent, const char *name,
+		const char *data, struct stat *stat)
+{
+	int                 ret = -1;
+	xlator_t           *subvol = NULL;
+	inode_t            *inode = NULL;
+	loc_t               loc = {0, };
+	struct iatt         iatt = {0, };
+	uuid_t              gfid;
+	dict_t             *xattr_req = NULL;
+	struct glfs_object *object = NULL;
+
+	/* validate in args */
+	if ((fs == NULL) || (parent == NULL) || (name == NULL) ||
+		(data == NULL)) {
+		errno = EINVAL;
+		return NULL;
+	}
+
+	__glfs_entry_fs (fs);
+
+	/* get the active volume */
+	subvol = glfs_active_subvol (fs);
+	if (!subvol) {
+		ret = -1;
+		errno = EIO;
+		goto out;
+	}
+
+	/* get/refresh the in arg objects inode in correlation to the xlator */
+	inode = glfs_resolve_inode (fs, subvol, parent);
+	if (!inode) {
+		errno = ESTALE;
+		goto out;
+	}
+
+	xattr_req = dict_new ();
+	if (!xattr_req) {
+		ret = -1;
+		errno = ENOMEM;
+		goto out;
+	}
+
+	uuid_generate (gfid);
+	ret = dict_set_static_bin (xattr_req, "gfid-req", gfid, 16);
+	if (ret) {
+		ret = -1;
+		errno = ENOMEM;
+		goto out;
+	}
+
+	GLFS_LOC_FILL_PINODE (inode, loc, ret, errno, out, name);
+
+	/* fop/op */
+	ret = syncop_symlink (subvol, &loc, data, xattr_req, &iatt);
+
+	/* populate out args */
+	if (ret == 0) {
+		/* TODO: If the inode existed in the cache (say file already
+		 * exists), then the glfs_loc_link will not update the
+		 * loc.inode, as a result we will have a 0000 GFID that we
+		 * would copy out to the object, this needs to be fixed.
+		 */
+		ret = glfs_loc_link (&loc, &iatt);
+		if (ret != 0) {
+			goto out;
+		}
+
+		if (stat)
+			glfs_iatt_to_stat (fs, &iatt, stat);
+
+		ret = glfs_create_object (&loc, &object);
+	}
+
+out:
+	if (ret && object != NULL) {
+		glfs_h_close (object);
+		object = NULL;
+	}
+
+	loc_wipe(&loc);
+
+	if (inode)
+		inode_unref (inode);
+
+	if (xattr_req)
+		dict_unref (xattr_req);
+
+	glfs_subvol_done (fs, subvol);
+
+	return object;
+}
+
+int
+glfs_h_readlink (struct glfs *fs, struct glfs_object *object, char *buf,
+		 size_t bufsiz)
+{
+	loc_t               loc = {0, };
+	int                 ret = -1;
+	xlator_t           *subvol = NULL;
+	inode_t            *inode = NULL;
+	char               *linkval = NULL;
+
+	/* validate in args */
+	if ((fs == NULL) || (object == NULL) || (buf == NULL)) {
+		errno = EINVAL;
+		return -1;
+	}
+
+	__glfs_entry_fs (fs);
+
+	/* get the active volume */
+	subvol = glfs_active_subvol (fs);
+	if (!subvol) {
+		ret = -1;
+		errno = EIO;
+		goto out;
+	}
+
+	/* get/refresh the in arg objects inode in correlation to the xlator */
+	inode = glfs_resolve_inode (fs, subvol, object);
+	if (!inode) {
+		errno = ESTALE;
+		goto out;
+	}
+
+	GLFS_LOC_FILL_INODE (inode, loc, out);
+
+	/* fop/op */
+	ret = syncop_readlink (subvol, &loc, &linkval, bufsiz);
+
+	/* populate out args */
+	if (ret > 0)
+		memcpy (buf, linkval, ret);
+
+out:
+	loc_wipe (&loc);
+
+	if (inode)
+		inode_unref (inode);
+
+	if (linkval)
+		GF_FREE (linkval);
+
+	glfs_subvol_done (fs, subvol);
+
+	return ret;
+}
+
+int
+glfs_h_link (struct glfs *fs, struct glfs_object *linksrc,
+	     struct glfs_object *parent, const char *name)
+{
+	int                 ret = -1;
+	xlator_t           *subvol = NULL;
+	inode_t            *inode = NULL;
+	inode_t            *pinode = NULL;
+	loc_t               oldloc = {0, };
+	loc_t               newloc = {0, };
+
+	/* validate in args */
+	if ((fs == NULL) || (linksrc == NULL) || (parent == NULL) ||
+		(name == NULL)) {
+		errno = EINVAL;
+		return -1;
+	}
+
+	__glfs_entry_fs (fs);
+
+	/* get the active volume */
+	subvol = glfs_active_subvol (fs);
+	if (!subvol) {
+		ret = -1;
+		errno = EIO;
+		goto out;
+	}
+
+	/* get/refresh the in arg objects inode in correlation to the xlator */
+	inode = glfs_resolve_inode (fs, subvol, linksrc);
+	if (!inode) {
+		errno = ESTALE;
+		goto out;
+	}
+
+	if (inode->ia_type == IA_IFDIR) {
+		ret = -1;
+		errno = EISDIR;
+		goto out;
+	}
+
+	GLFS_LOC_FILL_INODE (inode, oldloc, out);
+
+	/* get/refresh the in arg objects inode in correlation to the xlator */
+	pinode = glfs_resolve_inode (fs, subvol, parent);
+	if (!pinode) {
+		errno = ESTALE;
+		goto out;
+	}
+
+	/* setup newloc based on parent */
+	newloc.parent = inode_ref (pinode);
+	newloc.name = name;
+	ret = glfs_loc_touchup (&newloc);
+	if (ret != 0) {
+		errno = EINVAL;
+		goto out;
+	}
+
+	/* Filling the inode of the hard link to be same as that of the
+	 * original file
+	 */
+	newloc.inode = inode_ref (inode);
+
+	/* fop/op */
+	ret = syncop_link (subvol, &oldloc, &newloc);
+
+	if (ret == 0)
+		/* TODO: No iatt to pass as there has been no lookup */
+		ret = glfs_loc_link (&newloc, NULL);
+out:
+	loc_wipe (&oldloc);
+	loc_wipe (&newloc);
+
+	if (inode)
+		inode_unref (inode);
+
+	if (pinode)
+		inode_unref (pinode);
+
+	glfs_subvol_done (fs, subvol);
+
+	return ret;
+}
+
+int
+glfs_h_rename (struct glfs *fs, struct glfs_object *olddir, const char *oldname,
+	       struct glfs_object *newdir, const char *newname)
+{
+	int                 ret = -1;
+	xlator_t           *subvol = NULL;
+	inode_t            *oldpinode = NULL;
+	inode_t            *newpinode = NULL;
+	loc_t               oldloc = {0, };
+	loc_t               newloc = {0, };
+	struct iatt         oldiatt = {0, };
+	struct iatt         newiatt = {0, };
+
+	/* validate in args */
+	if ((fs == NULL) || (olddir == NULL) || (oldname == NULL) ||
+		(newdir == NULL) || (newname == NULL)) {
+		errno = EINVAL;
+		return -1;
+	}
+
+	__glfs_entry_fs (fs);
+
+	/* get the active volume */
+	subvol = glfs_active_subvol (fs);
+	if ( !subvol ) {
+		ret = -1;
+		errno = EIO;
+		goto out;
+	}
+
+	/* get/refresh the in arg objects inode in correlation to the xlator */
+	oldpinode = glfs_resolve_inode (fs, subvol, olddir);
+	if (!oldpinode) {
+		errno = ESTALE;
+		goto out;
+	}
+
+	ret = glfs_resolve_at (fs, subvol, oldpinode, oldname, &oldloc,
+			       &oldiatt, 0 , 0);
+	if (ret != 0) {
+		goto out;
+	}
+
+	/* get/refresh the in arg objects inode in correlation to the xlator */
+	newpinode = glfs_resolve_inode (fs, subvol, newdir);
+	if (!newpinode) {
+		errno = ESTALE;
+		goto out;
+	}
+
+	ret = glfs_resolve_at (fs, subvol, newpinode, newname, &newloc,
+			       &newiatt, 0, 0);
+
+	if (ret && errno != ENOENT && newloc.parent)
+		goto out;
+
+	if (newiatt.ia_type != IA_INVAL) {
+		if ((oldiatt.ia_type == IA_IFDIR) !=
+			(newiatt.ia_type == IA_IFDIR)) {
+			/* Either both old and new must be dirs,
+			 * or both must be non-dirs. Else, fail.
+			 */
+			ret = -1;
+			errno = EISDIR;
+			goto out;
+		}
+	}
+
+	/* TODO: check if new or old is a prefix of the other, and fail EINVAL */
+
+	ret = syncop_rename (subvol, &oldloc, &newloc);
+
+	if (ret == 0)
+		inode_rename (oldloc.parent->table, oldloc.parent, oldloc.name,
+			      newloc.parent, newloc.name, oldloc.inode,
+			      &oldiatt);
+
+out:
+	loc_wipe (&oldloc);
+	loc_wipe (&newloc);
+
+	if (oldpinode)
+		inode_unref (oldpinode);
+
+	if (newpinode)
+		inode_unref (newpinode);
+
+	glfs_subvol_done (fs, subvol);
+
+	return ret;
+}
diff --git a/api/src/glfs-handles.h b/api/src/glfs-handles.h
new file mode 100644
index 000000000..437f2cbc8
--- /dev/null
+++ b/api/src/glfs-handles.h
@@ -0,0 +1,143 @@
+/*
+  Copyright (c) 2013 Red Hat, Inc. <http://www.redhat.com>
+  This file is part of GlusterFS.
+
+  This file is licensed to you under your choice of the GNU Lesser
+  General Public License, version 3 or any later version (LGPLv3 or
+  later), or the GNU General Public License, version 2 (GPLv2), in all
+  cases as published by the Free Software Foundation.
+*/
+
+#ifndef _GLFS_HANDLES_H
+#define _GLFS_HANDLES_H
+
+#include "glfs.h"
+
+/* GLFS OBJECT BASED OPERATIONS
+ *
+ * The following APIs are introduced to provide an API framework that can work
+ * with gluster objects (files and directories), instead of absolute paths.
+ *
+ * The following API set can be related to the POSIX *at interfaces (like
+ * openat (2)). The intention of these APIs is to be able to operate based
+ * on parent object and looking up or creating child objects within, OR to be
+ * used on the actual object thus looked up or created, and retrieve information
+ * regarding the same.
+ *
+ * The APIs also provide for generating an opaque invariant handle to the
+ * object, that can later be used to lookup the object, instead of the regular
+ * glfs_h_* variants. The APIs that provide this behaviour are,
+ * glfs_h_extract_handle and glfs_h_create_from_handle.
+ *
+ * The object handles can be transitioned to fd based operations as supported
+ * by glfs.h calls, using the glfs_h_open call. This provides a way to move
+ * from objects to fd's akin to moving from path to fd for required operations.
+ *
+ * NOTE: The opaque invariant handle is the GFID of the object in reality, but
+ * maintained as an opaque data value, for potential internal changes to the
+ * same without impacting the caller.
+ *
+ * NOTE: Currently looking up an object can create multiple object handles to
+ * the same, i.e distinct glfs_object *. Hence each such looked up or received
+ * handle from other calls, would need to be closed. In the future, for a given
+ * object these pointers would be the same, and an ease of use API to forget all
+ * instances of this bject would be provided (instead of a per lookup close).
+ * This should not change the APIs in their current form.
+ *
+ */
+
+/* Values for valid falgs to be used when using XXXsetattr, to set multiple
+ attribute values passed via the related stat structure.
+ */
+#define GFAPI_SET_ATTR_MODE  0x1
+#define GFAPI_SET_ATTR_UID   0x2
+#define GFAPI_SET_ATTR_GID   0x4
+#define GFAPI_SET_ATTR_SIZE  0x8
+#define GFAPI_SET_ATTR_ATIME 0x10
+#define GFAPI_SET_ATTR_MTIME 0x20
+
+/* Handle length for object handles returned from glfs_h_extract_handle or
+ * glfs_h_create_from_handle */
+#define GFAPI_HANDLE_LENGTH 16
+
+__BEGIN_DECLS
+
+/*
+ * Notes:
+ *
+ * The file object handle. One per looked up, created file/directory
+ *
+ * This had been introduced to facilitate gfid/inode based gfapi
+ * - a requirement introduced by nfs-ganesha
+ */
+struct glfs_object;
+typedef struct glfs_object glfs_object_t;
+
+/* Handle based operations */
+/* Operations that generate handles */
+struct glfs_object *glfs_h_lookupat (struct glfs *fs,
+				     struct glfs_object *parent,
+				     const char *path, struct stat *stat);
+
+struct glfs_object *glfs_h_creat (struct glfs *fs, struct glfs_object *parent,
+				  const char *path, int flags, mode_t mode,
+				  struct stat *sb);
+
+struct glfs_object *glfs_h_mkdir (struct glfs *fs, struct glfs_object *parent,
+				  const char *path, mode_t flags,
+				  struct stat *sb);
+
+struct glfs_object *glfs_h_mknod (struct glfs *fs, struct glfs_object *parent,
+				  const char *path, mode_t mode, dev_t dev,
+				  struct stat *sb);
+
+struct glfs_object *glfs_h_symlink (struct glfs *fs, struct glfs_object *parent,
+				    const char *name, const char *data,
+				    struct stat *stat);
+
+/* Operations on the actual objects */
+int glfs_h_unlink (struct glfs *fs, struct glfs_object *parent,
+		   const char *path);
+
+int glfs_h_close (struct glfs_object *object);
+
+int glfs_caller_specific_init (void *uid_caller_key, void *gid_caller_key,
+			       void *future);
+
+int glfs_h_truncate (struct glfs *fs, struct glfs_object *object, off_t offset);
+
+int glfs_h_stat(struct glfs *fs, struct glfs_object *object, struct stat *stat);
+
+int glfs_h_getattrs (struct glfs *fs, struct glfs_object *object,
+		     struct stat *stat);
+
+int glfs_h_setattrs (struct glfs *fs, struct glfs_object *object,
+		     struct stat *sb, int valid);
+
+int glfs_h_readlink (struct glfs *fs, struct glfs_object *object, char *buf,
+		     size_t bufsiz);
+
+int glfs_h_link (struct glfs *fs, struct glfs_object *linktgt,
+		 struct glfs_object *parent, const char *name);
+
+int glfs_h_rename (struct glfs *fs, struct glfs_object *olddir,
+		   const char *oldname, struct glfs_object *newdir,
+		   const char *newname);
+
+/* Operations enabling opaque invariant handle to object transitions */
+ssize_t glfs_h_extract_handle (struct glfs_object *object,
+			       unsigned char *handle, int len);
+
+struct glfs_object *glfs_h_create_from_handle (struct glfs *fs,
+					       unsigned char *handle, int len,
+					       struct stat *stat);
+
+/* Operations enabling object handles to fd transitions */
+struct glfs_fd *glfs_h_opendir (struct glfs *fs, struct glfs_object *object);
+
+struct glfs_fd *glfs_h_open (struct glfs *fs, struct glfs_object *object,
+			     int flags);
+
+__END_DECLS
+
+#endif /* !_GLFS_HANDLES_H */
\ No newline at end of file
diff --git a/api/src/glfs-internal.h b/api/src/glfs-internal.h
new file mode 100644
index 000000000..ec1d5579d
--- /dev/null
+++ b/api/src/glfs-internal.h
@@ -0,0 +1,200 @@
+/*
+  Copyright (c) 2012 Red Hat, Inc. <http://www.redhat.com>
+  This file is part of GlusterFS.
+
+  This file is licensed to you under your choice of the GNU Lesser
+  General Public License, version 3 or any later version (LGPLv3 or
+  later), or the GNU General Public License, version 2 (GPLv2), in all
+  cases as published by the Free Software Foundation.
+*/
+
+
+#ifndef _GLFS_INTERNAL_H
+#define _GLFS_INTERNAL_H
+
+#include "xlator.h"
+
+#define GLFS_SYMLINK_MAX_FOLLOW 2048
+
+#define DEFAULT_REVAL_COUNT 1
+
+#define ESTALE_RETRY(ret,errno,reval,loc,label) do {	\
+	if (ret == -1 && errno == ESTALE) {	        \
+		if (reval < DEFAULT_REVAL_COUNT) {	\
+			reval++;			\
+			loc_wipe (loc);			\
+			goto label;			\
+		}					\
+	}						\
+	} while (0)
+
+#define GLFS_LOC_FILL_INODE(oinode, loc, label) do {   \
+	loc.inode = inode_ref (oinode);                \
+	uuid_copy (loc.gfid, oinode->gfid);            \
+	ret = glfs_loc_touchup (&loc);                 \
+	if (ret != 0) {                                \
+		errno = EINVAL;                        \
+		goto label;                            \
+	}                                              \
+	} while (0)
+
+#define GLFS_LOC_FILL_PINODE(pinode, loc, ret, errno, label, path) do {   \
+	loc.inode = inode_new (pinode->table);                            \
+	if (!loc.inode) {                                                 \
+		ret = -1;                                                 \
+		errno = ENOMEM;                                           \
+		goto label;                                               \
+	}                                                                 \
+	loc.parent = inode_ref (pinode);                                  \
+	loc.name = path;                                                  \
+	ret = glfs_loc_touchup (&loc);                                    \
+	if (ret != 0) {                                                   \
+		errno = EINVAL;                                           \
+		goto label;                                               \
+	}                                                                 \
+	} while (0)
+
+struct glfs;
+
+typedef int (*glfs_init_cbk) (struct glfs *fs, int ret);
+
+struct glfs {
+	char               *volname;
+
+	glusterfs_ctx_t    *ctx;
+
+	pthread_t           poller;
+
+	glfs_init_cbk       init_cbk;
+	pthread_mutex_t     mutex;
+	pthread_cond_t      cond;
+	int                 init;
+	int                 ret;
+	int                 err;
+
+	xlator_t           *active_subvol;
+	xlator_t           *next_subvol;
+	xlator_t           *old_subvol;
+
+	char               *oldvolfile;
+	ssize_t             oldvollen;
+
+	inode_t            *cwd;
+
+	uint32_t            dev_id; /* Used to fill st_dev in struct stat */
+
+	struct list_head    openfds;
+
+	gf_boolean_t        migration_in_progress;
+};
+
+struct glfs_fd {
+	struct list_head   openfds;
+	struct glfs       *fs;
+	off_t              offset;
+	fd_t              *fd; /* Currently guared by @fs->mutex. TODO: per-glfd lock */
+	struct list_head   entries;
+	gf_dirent_t       *next;
+	struct dirent     *readdirbuf;
+};
+
+/* glfs object handle introduced for the alternate gfapi implementation based
+   on glfs handles/gfid/inode
+*/
+struct glfs_object {
+        inode_t         *inode;
+        uuid_t          gfid;
+};
+
+#define DEFAULT_EVENT_POOL_SIZE           16384
+#define GF_MEMPOOL_COUNT_OF_DICT_T        4096
+#define GF_MEMPOOL_COUNT_OF_DATA_T        (GF_MEMPOOL_COUNT_OF_DICT_T * 4)
+#define GF_MEMPOOL_COUNT_OF_DATA_PAIR_T   (GF_MEMPOOL_COUNT_OF_DICT_T * 4)
+
+int glfs_mgmt_init (struct glfs *fs);
+void glfs_init_done (struct glfs *fs, int ret);
+int glfs_process_volfp (struct glfs *fs, FILE *fp);
+int glfs_resolve (struct glfs *fs, xlator_t *subvol, const char *path, loc_t *loc,
+		  struct iatt *iatt, int reval);
+int glfs_lresolve (struct glfs *fs, xlator_t *subvol, const char *path, loc_t *loc,
+		   struct iatt *iatt, int reval);
+fd_t *glfs_resolve_fd (struct glfs *fs, xlator_t *subvol, struct glfs_fd *glfd);
+
+fd_t *__glfs_migrate_fd (struct glfs *fs, xlator_t *subvol, struct glfs_fd *glfd);
+
+int glfs_first_lookup (xlator_t *subvol);
+
+static inline void
+__glfs_entry_fs (struct glfs *fs)
+{
+	THIS = fs->ctx->master;
+}
+
+
+static inline void
+__glfs_entry_fd (struct glfs_fd *fd)
+{
+	THIS = fd->fd->inode->table->xl->ctx->master;
+}
+
+
+/*
+  By default all lock attempts from user context must
+  use glfs_lock() and glfs_unlock(). This allows
+  for a safe implementation of graph migration where
+  we can give up the mutex during syncop calls so
+  that bottom up calls (particularly CHILD_UP notify)
+  can do a mutex_lock() on @glfs without deadlocking
+  the filesystem
+*/
+static inline int
+glfs_lock (struct glfs *fs)
+{
+	pthread_mutex_lock (&fs->mutex);
+
+	while (!fs->init)
+		pthread_cond_wait (&fs->cond, &fs->mutex);
+
+	while (fs->migration_in_progress)
+		pthread_cond_wait (&fs->cond, &fs->mutex);
+
+	return 0;
+}
+
+
+static inline void
+glfs_unlock (struct glfs *fs)
+{
+	pthread_mutex_unlock (&fs->mutex);
+}
+
+
+void glfs_fd_destroy (struct glfs_fd *glfd);
+
+struct glfs_fd *glfs_fd_new (struct glfs *fs);
+void glfs_fd_bind (struct glfs_fd *glfd);
+
+xlator_t * glfs_active_subvol (struct glfs *fs);
+xlator_t * __glfs_active_subvol (struct glfs *fs);
+void glfs_subvol_done (struct glfs *fs, xlator_t *subvol);
+
+inode_t * glfs_refresh_inode (xlator_t *subvol, inode_t *inode);
+
+inode_t *glfs_cwd_get (struct glfs *fs);
+int glfs_cwd_set (struct glfs *fs, inode_t *inode);
+inode_t *glfs_resolve_inode (struct glfs *fs, xlator_t *subvol,
+			     struct glfs_object *object);
+int glfs_create_object (loc_t *loc, struct glfs_object **retobject);
+int __glfs_cwd_set (struct glfs *fs, inode_t *inode);
+
+int glfs_resolve_base (struct glfs *fs, xlator_t *subvol, inode_t *inode,
+		       struct iatt *iatt);
+int glfs_resolve_at (struct glfs *fs, xlator_t *subvol, inode_t *at,
+                     const char *origpath, loc_t *loc, struct iatt *iatt,
+                     int follow, int reval);
+int glfs_loc_touchup (loc_t *loc);
+void glfs_iatt_to_stat (struct glfs *fs, struct iatt *iatt, struct stat *stat);
+int glfs_loc_link (loc_t *loc, struct iatt *iatt);
+int glfs_loc_unlink (loc_t *loc);
+
+#endif /* !_GLFS_INTERNAL_H */
diff --git a/api/src/glfs-master.c b/api/src/glfs-master.c
new file mode 100644
index 000000000..c02534c18
--- /dev/null
+++ b/api/src/glfs-master.c
@@ -0,0 +1,154 @@
+/*
+  Copyright (c) 2012 Red Hat, Inc. <http://www.redhat.com>
+  This file is part of GlusterFS.
+
+  This file is licensed to you under your choice of the GNU Lesser
+  General Public License, version 3 or any later version (LGPLv3 or
+  later), or the GNU General Public License, version 2 (GPLv2), in all
+  cases as published by the Free Software Foundation.
+*/
+
+#include <unistd.h>
+#include <string.h>
+#include <stdlib.h>
+#include <stdio.h>
+#include <inttypes.h>
+#include <limits.h>
+
+#ifndef _CONFIG_H
+#define _CONFIG_H
+#include "config.h"
+#endif
+
+#include "xlator.h"
+#include "glusterfs.h"
+
+#include "glfs-internal.h"
+#include "glfs-mem-types.h"
+
+
+int
+glfs_graph_setup (struct glfs *fs, glusterfs_graph_t *graph)
+{
+	xlator_t      *new_subvol = NULL;
+	xlator_t      *old_subvol = NULL;
+	inode_table_t *itable = NULL;
+	int            ret = -1;
+
+	new_subvol = graph->top;
+
+	/* This is called in a bottom-up context, it should specifically
+	   NOT be glfs_lock()
+	*/
+	pthread_mutex_lock (&fs->mutex);
+	{
+		if (new_subvol->switched ||
+		    new_subvol == fs->active_subvol ||
+		    new_subvol == fs->next_subvol) {
+			/* Spurious CHILD_UP event on old graph */
+			ret = 0;
+			goto unlock;
+		}
+
+		if (!new_subvol->itable) {
+			itable = inode_table_new (131072, new_subvol);
+			if (!itable) {
+				errno = ENOMEM;
+				ret = -1;
+				goto unlock;
+			}
+
+			new_subvol->itable = itable;
+		}
+
+		old_subvol = fs->next_subvol;
+		fs->next_subvol = new_subvol;
+		fs->next_subvol->winds++; /* first ref */
+		ret = 0;
+	}
+unlock:
+	pthread_mutex_unlock (&fs->mutex);
+
+	if (old_subvol)
+		/* wasn't picked up so far, skip */
+		glfs_subvol_done (fs, old_subvol);
+
+	return ret;
+}
+
+
+int
+notify (xlator_t *this, int event, void *data, ...)
+{
+	glusterfs_graph_t   *graph = NULL;
+	struct glfs	    *fs = NULL;
+
+	graph = data;
+	fs = this->private;
+
+	switch (event) {
+	case GF_EVENT_GRAPH_NEW:
+		gf_log (this->name, GF_LOG_INFO, "New graph %s (%d) coming up",
+			uuid_utoa ((unsigned char *)graph->graph_uuid),
+			graph->id);
+		break;
+	case GF_EVENT_CHILD_UP:
+		glfs_graph_setup (fs, graph);
+		glfs_init_done (fs, 0);
+		break;
+	case GF_EVENT_CHILD_DOWN:
+		glfs_graph_setup (fs, graph);
+		glfs_init_done (fs, 1);
+		break;
+	case GF_EVENT_CHILD_CONNECTING:
+		break;
+	default:
+		gf_log (this->name, GF_LOG_DEBUG,
+			"got notify event %d", event);
+		break;
+	}
+
+	return 0;
+}
+
+
+int
+mem_acct_init (xlator_t *this)
+{
+	int ret = -1;
+
+	if (!this)
+		return ret;
+
+	ret = xlator_mem_acct_init (this, glfs_mt_end + 1);
+	if (ret) {
+		gf_log (this->name, GF_LOG_ERROR, "Failed to initialise "
+                        "memory accounting");
+		return ret;
+	}
+
+	return 0;
+}
+
+
+int
+init (xlator_t *this)
+{
+	return 0;
+}
+
+
+void
+fini (xlator_t *this)
+{
+
+}
+
+
+struct xlator_dumpops dumpops;
+
+
+struct xlator_fops fops;
+
+
+struct xlator_cbks cbks;
diff --git a/api/src/glfs-mem-types.h b/api/src/glfs-mem-types.h
new file mode 100644
index 000000000..3301b3da5
--- /dev/null
+++ b/api/src/glfs-mem-types.h
@@ -0,0 +1,32 @@
+/*
+  Copyright (c) 2012 Red Hat, Inc. <http://www.redhat.com>
+  This file is part of GlusterFS.
+
+  This file is licensed to you under your choice of the GNU Lesser
+  General Public License, version 3 or any later version (LGPLv3 or
+  later), or the GNU General Public License, version 2 (GPLv2), in all
+  cases as published by the Free Software Foundation.
+*/
+
+#ifndef _GLFS_MEM_TYPES_H
+#define _GLFS_MEM_TYPES_H
+
+#include "mem-types.h"
+
+#define GF_MEM_TYPE_START (gf_common_mt_end + 1)
+
+enum glfs_mem_types_ {
+        glfs_mt_glfs_t = GF_MEM_TYPE_START,
+        glfs_mt_call_pool_t,
+        glfs_mt_xlator_t,
+	glfs_mt_glfs_fd_t,
+	glfs_mt_glfs_io_t,
+	glfs_mt_volfile_t,
+	glfs_mt_xlator_cmdline_option_t,
+	glfs_mt_glfs_object_t,
+	glfs_mt_readdirbuf_t,
+	glfs_mt_end
+
+};
+#endif
+
diff --git a/api/src/glfs-mgmt.c b/api/src/glfs-mgmt.c
new file mode 100644
index 000000000..6843e9cb3
--- /dev/null
+++ b/api/src/glfs-mgmt.c
@@ -0,0 +1,543 @@
+/*
+  Copyright (c) 2012 Red Hat, Inc. <http://www.redhat.com>
+  This file is part of GlusterFS.
+
+  This file is licensed to you under your choice of the GNU Lesser
+  General Public License, version 3 or any later version (LGPLv3 or
+  later), or the GNU General Public License, version 2 (GPLv2), in all
+  cases as published by the Free Software Foundation.
+*/
+
+
+#include <stdio.h>
+#include <sys/types.h>
+#include <sys/wait.h>
+#include <stdlib.h>
+#include <signal.h>
+#include <pthread.h>
+
+#ifndef _CONFIG_H
+#define _CONFIG_H
+#include "config.h"
+#endif /* _CONFIG_H */
+
+#include "glusterfs.h"
+#include "stack.h"
+#include "dict.h"
+#include "event.h"
+#include "defaults.h"
+
+#include "rpc-clnt.h"
+#include "protocol-common.h"
+#include "glusterfs3.h"
+#include "portmap-xdr.h"
+#include "xdr-generic.h"
+
+#include "syncop.h"
+#include "xlator.h"
+
+#include "glfs-internal.h"
+#include "glfs-mem-types.h"
+
+
+int glfs_volfile_fetch (struct glfs *fs);
+
+int
+glfs_process_volfp (struct glfs *fs, FILE *fp)
+{
+	glusterfs_graph_t  *graph = NULL;
+	int		    ret = -1;
+	xlator_t	   *trav = NULL;
+	glusterfs_ctx_t	   *ctx = NULL;
+
+	ctx = fs->ctx;
+	graph = glusterfs_graph_construct (fp);
+	if (!graph) {
+		gf_log ("glfs", GF_LOG_ERROR, "failed to construct the graph");
+		goto out;
+	}
+
+	for (trav = graph->first; trav; trav = trav->next) {
+		if (strcmp (trav->type, "mount/fuse") == 0) {
+			gf_log ("glfs", GF_LOG_ERROR,
+				"fuse xlator cannot be specified "
+				"in volume file");
+			goto out;
+		}
+	}
+
+	ret = glusterfs_graph_prepare (graph, ctx);
+	if (ret) {
+		glusterfs_graph_destroy (graph);
+		goto out;
+	}
+
+	ret = glusterfs_graph_activate (graph, ctx);
+
+	if (ret) {
+		glusterfs_graph_destroy (graph);
+		goto out;
+	}
+
+	ret = 0;
+out:
+	if (fp)
+		fclose (fp);
+
+	if (!ctx->active) {
+		ret = -1;
+	}
+
+	return ret;
+}
+
+
+int
+mgmt_cbk_spec (struct rpc_clnt *rpc, void *mydata, void *data)
+{
+	struct glfs *fs = NULL;
+	xlator_t    *this = NULL;
+
+	this = mydata;
+	fs = this->private;
+
+	glfs_volfile_fetch (fs);
+
+	return 0;
+}
+
+
+int
+mgmt_cbk_event (struct rpc_clnt *rpc, void *mydata, void *data)
+{
+	return 0;
+}
+
+
+rpcclnt_cb_actor_t mgmt_cbk_actors[] = {
+	[GF_CBK_FETCHSPEC] = {"FETCHSPEC", GF_CBK_FETCHSPEC, mgmt_cbk_spec },
+	[GF_CBK_EVENT_NOTIFY] = {"EVENTNOTIFY", GF_CBK_EVENT_NOTIFY,
+				 mgmt_cbk_event},
+};
+
+
+struct rpcclnt_cb_program mgmt_cbk_prog = {
+	.progname  = "GlusterFS Callback",
+	.prognum   = GLUSTER_CBK_PROGRAM,
+	.progver   = GLUSTER_CBK_VERSION,
+	.actors	   = mgmt_cbk_actors,
+	.numactors = GF_CBK_MAXVALUE,
+};
+
+char *clnt_handshake_procs[GF_HNDSK_MAXVALUE] = {
+	[GF_HNDSK_NULL]		= "NULL",
+	[GF_HNDSK_SETVOLUME]	= "SETVOLUME",
+	[GF_HNDSK_GETSPEC]	= "GETSPEC",
+	[GF_HNDSK_PING]		= "PING",
+	[GF_HNDSK_EVENT_NOTIFY] = "EVENTNOTIFY",
+};
+
+rpc_clnt_prog_t clnt_handshake_prog = {
+	.progname  = "GlusterFS Handshake",
+	.prognum   = GLUSTER_HNDSK_PROGRAM,
+	.progver   = GLUSTER_HNDSK_VERSION,
+	.procnames = clnt_handshake_procs,
+};
+
+
+int
+mgmt_submit_request (void *req, call_frame_t *frame,
+		     glusterfs_ctx_t *ctx,
+		     rpc_clnt_prog_t *prog, int procnum,
+		     fop_cbk_fn_t cbkfn, xdrproc_t xdrproc)
+{
+	int			ret	    = -1;
+	int			count	   = 0;
+	struct iovec		iov	    = {0, };
+	struct iobuf		*iobuf = NULL;
+	struct iobref		*iobref = NULL;
+	ssize_t			xdr_size = 0;
+
+	iobref = iobref_new ();
+	if (!iobref) {
+		goto out;
+	}
+
+	if (req) {
+		xdr_size = xdr_sizeof (xdrproc, req);
+
+		iobuf = iobuf_get2 (ctx->iobuf_pool, xdr_size);
+		if (!iobuf) {
+			goto out;
+		};
+
+		iobref_add (iobref, iobuf);
+
+		iov.iov_base = iobuf->ptr;
+		iov.iov_len  = iobuf_pagesize (iobuf);
+
+		/* Create the xdr payload */
+		ret = xdr_serialize_generic (iov, req, xdrproc);
+		if (ret == -1) {
+			gf_log (THIS->name, GF_LOG_WARNING,
+				"failed to create XDR payload");
+			goto out;
+		}
+		iov.iov_len = ret;
+		count = 1;
+	}
+
+	/* Send the msg */
+	ret = rpc_clnt_submit (ctx->mgmt, prog, procnum, cbkfn,
+			       &iov, count,
+			       NULL, 0, iobref, frame, NULL, 0, NULL, 0, NULL);
+
+out:
+	if (iobref)
+		iobref_unref (iobref);
+
+	if (iobuf)
+		iobuf_unref (iobuf);
+	return ret;
+}
+
+
+static int
+glusterfs_oldvolfile_update (struct glfs *fs, char *volfile, ssize_t size)
+{
+	int ret = -1;
+
+	fs->oldvollen = size;
+	if (!fs->oldvolfile) {
+		fs->oldvolfile = GF_CALLOC (1, size+1, glfs_mt_volfile_t);
+	} else {
+		fs->oldvolfile = GF_REALLOC (fs->oldvolfile, size+1);
+	}
+
+	if (!fs->oldvolfile) {
+		fs->oldvollen = 0;
+	} else {
+		memcpy (fs->oldvolfile, volfile, size);
+		fs->oldvollen = size;
+		ret = 0;
+	}
+
+	return ret;
+}
+
+
+int
+mgmt_getspec_cbk (struct rpc_req *req, struct iovec *iov, int count,
+		  void *myframe)
+{
+	gf_getspec_rsp		 rsp   = {0,};
+	call_frame_t		*frame = NULL;
+	glusterfs_ctx_t		*ctx = NULL;
+	int			 ret   = 0;
+	ssize_t			 size = 0;
+	FILE			*tmpfp = NULL;
+	int                      need_retry = 0;
+	struct glfs		*fs = NULL;
+
+	frame = myframe;
+	ctx = frame->this->ctx;
+	fs = ((xlator_t *)ctx->master)->private;
+
+	if (-1 == req->rpc_status) {
+		ret = -1;
+		need_retry = 1;
+		goto out;
+	}
+
+	ret = xdr_to_generic (*iov, &rsp, (xdrproc_t)xdr_gf_getspec_rsp);
+	if (ret < 0) {
+		gf_log (frame->this->name, GF_LOG_ERROR, "XDR decoding error");
+		ret   = -1;
+		goto out;
+	}
+
+	if (-1 == rsp.op_ret) {
+		gf_log (frame->this->name, GF_LOG_ERROR,
+			"failed to get the 'volume file' from server");
+		ret = -1;
+                errno = rsp.op_errno;
+		goto out;
+	}
+
+	ret = 0;
+	size = rsp.op_ret;
+
+	if ((size == fs->oldvollen) &&
+	    (memcmp (fs->oldvolfile, rsp.spec, size) == 0)) {
+		gf_log (frame->this->name, GF_LOG_INFO,
+			"No change in volfile, continuing");
+		goto out;
+	}
+
+	tmpfp = tmpfile ();
+	if (!tmpfp) {
+		ret = -1;
+		goto out;
+	}
+
+	fwrite (rsp.spec, size, 1, tmpfp);
+	fflush (tmpfp);
+	if (ferror (tmpfp)) {
+		ret = -1;
+		goto out;
+	}
+
+	/*  Check if only options have changed. No need to reload the
+	*  volfile if topology hasn't changed.
+	*  glusterfs_volfile_reconfigure returns 3 possible return states
+	*  return 0	     =======> reconfiguration of options has succeeded
+	*  return 1	     =======> the graph has to be reconstructed and all the xlators should be inited
+	*  return -1(or -ve) =======> Some Internal Error occurred during the operation
+	*/
+
+	ret = glusterfs_volfile_reconfigure (fs->oldvollen, tmpfp, fs->ctx,
+                                             fs->oldvolfile);
+	if (ret == 0) {
+		gf_log ("glusterfsd-mgmt", GF_LOG_DEBUG,
+			"No need to re-load volfile, reconfigure done");
+		ret = glusterfs_oldvolfile_update (fs, rsp.spec, size);
+		goto out;
+	}
+
+	if (ret < 0) {
+		gf_log ("glusterfsd-mgmt", GF_LOG_DEBUG,
+			"Reconfigure failed !!");
+		goto out;
+	}
+
+	ret = glfs_process_volfp (fs, tmpfp);
+	/* tmpfp closed */
+	tmpfp = NULL;
+	if (ret)
+		goto out;
+
+	ret = glusterfs_oldvolfile_update (fs, rsp.spec, size);
+out:
+	STACK_DESTROY (frame->root);
+
+	if (rsp.spec)
+		free (rsp.spec);
+
+        // Stop if server is running at an unsupported op-version
+        if (ENOTSUP == ret) {
+                gf_log ("mgmt", GF_LOG_ERROR, "Server is operating at an "
+                        "op-version which is not supported");
+                errno = ENOTSUP;
+                glfs_init_done (fs, -1);
+        }
+
+	if (ret && ctx && !ctx->active) {
+		/* Do it only for the first time */
+		/* Failed to get the volume file, something wrong,
+		   restart the process */
+		gf_log ("glfs-mgmt", GF_LOG_ERROR,
+			"failed to fetch volume file (key:%s)",
+			ctx->cmd_args.volfile_id);
+		if (!need_retry) {
+                        if (!errno)
+                                errno = EINVAL;
+			glfs_init_done (fs, -1);
+                }
+	}
+
+	if (tmpfp)
+		fclose (tmpfp);
+
+	return 0;
+}
+
+
+int
+glfs_volfile_fetch (struct glfs *fs)
+{
+	cmd_args_t	 *cmd_args = NULL;
+	gf_getspec_req	  req = {0, };
+	int		  ret = 0;
+	call_frame_t	 *frame = NULL;
+	glusterfs_ctx_t	 *ctx = NULL;
+        dict_t           *dict = NULL;
+
+	ctx = fs->ctx;
+	cmd_args = &ctx->cmd_args;
+
+	frame = create_frame (THIS, ctx->pool);
+
+	req.key = cmd_args->volfile_id;
+	req.flags = 0;
+
+        dict = dict_new ();
+        if (!dict) {
+                ret = -1;
+                goto out;
+        }
+
+        // Set the supported min and max op-versions, so glusterd can make a
+        // decision
+        ret = dict_set_int32 (dict, "min-op-version", GD_OP_VERSION_MIN);
+        if (ret) {
+                gf_log (THIS->name, GF_LOG_ERROR, "Failed to set min-op-version"
+                        " in request dict");
+                goto out;
+        }
+
+        ret = dict_set_int32 (dict, "max-op-version", GD_OP_VERSION_MAX);
+        if (ret) {
+                gf_log (THIS->name, GF_LOG_ERROR, "Failed to set max-op-version"
+                        " in request dict");
+                goto out;
+        }
+
+        ret = dict_allocate_and_serialize (dict, &req.xdata.xdata_val,
+                                           &req.xdata.xdata_len);
+        if (ret < 0) {
+                gf_log (THIS->name, GF_LOG_ERROR,
+                        "Failed to serialize dictionary");
+                goto out;
+        }
+
+	ret = mgmt_submit_request (&req, frame, ctx, &clnt_handshake_prog,
+				   GF_HNDSK_GETSPEC, mgmt_getspec_cbk,
+				   (xdrproc_t)xdr_gf_getspec_req);
+out:
+        return ret;
+}
+
+
+static int
+mgmt_rpc_notify (struct rpc_clnt *rpc, void *mydata, rpc_clnt_event_t event,
+		 void *data)
+{
+	xlator_t	*this = NULL;
+	cmd_args_t	*cmd_args = NULL;
+	glusterfs_ctx_t *ctx = NULL;
+	struct glfs	 *fs = NULL;
+	int		 ret = 0;
+
+	this = mydata;
+	ctx = this->ctx;
+	fs = ((xlator_t *)ctx->master)->private;
+	cmd_args = &ctx->cmd_args;
+
+	switch (event) {
+	case RPC_CLNT_DISCONNECT:
+		if (!ctx->active) {
+			cmd_args->max_connect_attempts--;
+			gf_log ("glfs-mgmt", GF_LOG_ERROR,
+				"failed to connect with remote-host: %s",
+				strerror (errno));
+			gf_log ("glfs-mgmt", GF_LOG_INFO,
+				"%d connect attempts left",
+				cmd_args->max_connect_attempts);
+			if (0 >= cmd_args->max_connect_attempts) {
+                                errno = ENOTCONN;
+				glfs_init_done (fs, -1);
+                        }
+		}
+		break;
+	case RPC_CLNT_CONNECT:
+		rpc_clnt_set_connected (&((struct rpc_clnt*)ctx->mgmt)->conn);
+
+		ret = glfs_volfile_fetch (fs);
+		if (ret && ctx && (ctx->active == NULL)) {
+			/* Do it only for the first time */
+			/* Exit the process.. there are some wrong options */
+			gf_log ("glfs-mgmt", GF_LOG_ERROR,
+				"failed to fetch volume file (key:%s)",
+				ctx->cmd_args.volfile_id);
+                        errno = EINVAL;
+			glfs_init_done (fs, -1);
+		}
+
+		break;
+	default:
+		break;
+	}
+
+	return 0;
+}
+
+
+int
+glusterfs_mgmt_notify (int32_t op, void *data, ...)
+{
+	int ret = 0;
+
+	switch (op)
+	{
+		case GF_EN_DEFRAG_STATUS:
+			break;
+
+		default:
+			break;
+	}
+
+	return ret;
+}
+
+
+int
+glfs_mgmt_init (struct glfs *fs)
+{
+	cmd_args_t		*cmd_args = NULL;
+	struct rpc_clnt		*rpc = NULL;
+	dict_t			*options = NULL;
+	int			ret = -1;
+	int			port = GF_DEFAULT_BASE_PORT;
+	char			*host = NULL;
+	glusterfs_ctx_t		*ctx = NULL;
+
+	ctx = fs->ctx;
+	cmd_args = &ctx->cmd_args;
+
+	if (ctx->mgmt)
+		return 0;
+
+	if (cmd_args->volfile_server_port)
+		port = cmd_args->volfile_server_port;
+
+	host = "localhost";
+	if (cmd_args->volfile_server)
+		host = cmd_args->volfile_server;
+
+	ret = rpc_transport_inet_options_build (&options, host, port);
+	if (ret)
+		goto out;
+
+	rpc = rpc_clnt_new (options, THIS->ctx, THIS->name, 8);
+	if (!rpc) {
+		ret = -1;
+		gf_log (THIS->name, GF_LOG_WARNING,
+			"failed to create rpc clnt");
+		goto out;
+	}
+
+	ret = rpc_clnt_register_notify (rpc, mgmt_rpc_notify, THIS);
+	if (ret) {
+		gf_log (THIS->name, GF_LOG_WARNING,
+			"failed to register notify function");
+		goto out;
+	}
+
+	ret = rpcclnt_cbk_program_register (rpc, &mgmt_cbk_prog, THIS);
+	if (ret) {
+		gf_log (THIS->name, GF_LOG_WARNING,
+			"failed to register callback function");
+		goto out;
+	}
+
+	ctx->notify = glusterfs_mgmt_notify;
+
+	/* This value should be set before doing the 'rpc_clnt_start()' as
+	   the notify function uses this variable */
+	ctx->mgmt = rpc;
+
+	ret = rpc_clnt_start (rpc);
+out:
+	return ret;
+}
+
diff --git a/api/src/glfs-resolve.c b/api/src/glfs-resolve.c
new file mode 100644
index 000000000..4ca2eb6fc
--- /dev/null
+++ b/api/src/glfs-resolve.c
@@ -0,0 +1,969 @@
+/*
+  Copyright (c) 2012 Red Hat, Inc. <http://www.redhat.com>
+  This file is part of GlusterFS.
+
+  This file is licensed to you under your choice of the GNU Lesser
+  General Public License, version 3 or any later version (LGPLv3 or
+  later), or the GNU General Public License, version 2 (GPLv2), in all
+  cases as published by the Free Software Foundation.
+*/
+
+
+#include <unistd.h>
+#include <string.h>
+#include <stdlib.h>
+#include <stdio.h>
+#include <inttypes.h>
+#include <limits.h>
+
+#ifndef _CONFIG_H
+#define _CONFIG_H
+#include "config.h"
+#endif
+
+#include "glusterfs.h"
+#include "logging.h"
+#include "stack.h"
+#include "event.h"
+#include "glfs-mem-types.h"
+#include "common-utils.h"
+#include "syncop.h"
+#include "call-stub.h"
+
+#include "glfs-internal.h"
+
+#define graphid_str(subvol) (uuid_utoa((unsigned char *)subvol->graph->graph_uuid))
+
+
+int
+glfs_first_lookup_safe (xlator_t *subvol)
+{
+	loc_t  loc = {0, };
+	int    ret = -1;
+
+	loc.inode = subvol->itable->root;
+	memset (loc.gfid, 0, 16);
+	loc.gfid[15] = 1;
+	loc.path = "/";
+	loc.name = "";
+
+	ret = syncop_lookup (subvol, &loc, 0, 0, 0, 0);
+
+	gf_log (subvol->name, GF_LOG_DEBUG, "first lookup complete %d", ret);
+
+	return ret;
+}
+
+
+int
+__glfs_first_lookup (struct glfs *fs, xlator_t *subvol)
+{
+	int ret = -1;
+
+	fs->migration_in_progress = 1;
+	pthread_mutex_unlock (&fs->mutex);
+	{
+		ret = glfs_first_lookup_safe (subvol);
+	}
+	pthread_mutex_lock (&fs->mutex);
+	fs->migration_in_progress = 0;
+	pthread_cond_broadcast (&fs->cond);
+
+	return ret;
+}
+
+
+inode_t *
+glfs_refresh_inode_safe (xlator_t *subvol, inode_t *oldinode)
+{
+	loc_t        loc = {0, };
+	int          ret = -1;
+	struct iatt  iatt = {0, };
+	inode_t     *newinode = NULL;
+
+
+	if (!oldinode)
+		return NULL;
+
+	if (oldinode->table->xl == subvol)
+		return inode_ref (oldinode);
+
+	newinode = inode_find (subvol->itable, oldinode->gfid);
+	if (newinode)
+		return newinode;
+
+	uuid_copy (loc.gfid, oldinode->gfid);
+	loc.inode = inode_new (subvol->itable);
+	if (!loc.inode)
+		return NULL;
+
+	ret = syncop_lookup (subvol, &loc, 0, &iatt, 0, 0);
+
+	if (ret) {
+		gf_log (subvol->name, GF_LOG_WARNING,
+			"inode refresh of %s failed: %s",
+			uuid_utoa (oldinode->gfid), strerror (errno));
+		loc_wipe (&loc);
+		return NULL;
+	}
+
+	newinode = inode_link (loc.inode, 0, 0, &iatt);
+	if (newinode)
+		inode_lookup (newinode);
+
+	loc_wipe (&loc);
+
+	return newinode;
+}
+
+
+inode_t *
+__glfs_refresh_inode (struct glfs *fs, xlator_t *subvol, inode_t *inode)
+{
+	inode_t *newinode = NULL;
+
+	fs->migration_in_progress = 1;
+	pthread_mutex_unlock (&fs->mutex);
+	{
+		newinode = glfs_refresh_inode_safe (subvol, inode);
+	}
+	pthread_mutex_lock (&fs->mutex);
+	fs->migration_in_progress = 0;
+	pthread_cond_broadcast (&fs->cond);
+
+	return newinode;
+}
+
+int
+glfs_loc_touchup (loc_t *loc)
+{
+	char *path = NULL;
+	int   ret = -1;
+	char *bn = NULL;
+
+	if (loc->parent)
+		ret = inode_path (loc->parent, loc->name, &path);
+	else
+		ret = inode_path (loc->inode, 0, &path);
+
+	loc->path = path;
+
+	if (ret < 0 || !path) {
+		ret = -1;
+		errno = ENOMEM;
+		goto out;
+	}
+
+	bn = strrchr (path, '/');
+	if (bn)
+		bn++;
+	loc->name = bn;
+	ret = 0;
+out:
+	return ret;
+}
+
+
+int
+glfs_resolve_symlink (struct glfs *fs, xlator_t *subvol, inode_t *inode,
+		      char **lpath)
+{
+	loc_t  loc = {0, };
+	char  *path = NULL;
+	char  *rpath = NULL;
+	int    ret = -1;
+
+	loc.inode = inode_ref (inode);
+	uuid_copy (loc.gfid, inode->gfid);
+	ret = inode_path (inode, NULL, &rpath);
+	if (ret < 0)
+		goto out;
+	loc.path = rpath;
+
+	ret = syncop_readlink (subvol, &loc, &path, 4096);
+
+	if (ret < 0)
+		goto out;
+
+	if (lpath)
+		*lpath = path;
+out:
+	loc_wipe (&loc);
+	return ret;
+}
+
+
+int
+glfs_resolve_base (struct glfs *fs, xlator_t *subvol, inode_t *inode,
+		   struct iatt *iatt)
+{
+	loc_t       loc = {0, };
+	int         ret = -1;
+	char       *path = NULL;
+
+	loc.inode = inode_ref (inode);
+	uuid_copy (loc.gfid, inode->gfid);
+
+	ret = inode_path (loc.inode, NULL, &path);
+	loc.path = path;
+	if (ret < 0)
+		goto out;
+
+	ret = syncop_lookup (subvol, &loc, NULL, iatt, NULL, NULL);
+out:
+	loc_wipe (&loc);
+
+	return ret;
+}
+
+
+inode_t *
+glfs_resolve_component (struct glfs *fs, xlator_t *subvol, inode_t *parent,
+			const char *component, struct iatt *iatt,
+			int force_lookup)
+{
+	loc_t        loc = {0, };
+	inode_t     *inode = NULL;
+	int          reval = 0;
+	int          ret = -1;
+	int          glret = -1;
+	struct iatt  ciatt = {0, };
+	uuid_t       gfid;
+	dict_t      *xattr_req = NULL;
+
+	loc.name = component;
+
+	loc.parent = inode_ref (parent);
+	uuid_copy (loc.pargfid, parent->gfid);
+
+
+	if (strcmp (component, ".") == 0)
+		loc.inode = inode_ref (parent);
+	else if (strcmp (component, "..") == 0)
+		loc.inode = inode_parent (parent, 0, 0);
+	else
+		loc.inode = inode_grep (parent->table, parent, component);
+
+	if (loc.inode) {
+		uuid_copy (loc.gfid, loc.inode->gfid);
+		reval = 1;
+
+		if (!force_lookup) {
+			inode = inode_ref (loc.inode);
+			ciatt.ia_type = inode->ia_type;
+			goto found;
+		}
+	} else {
+		uuid_generate (gfid);
+		loc.inode = inode_new (parent->table);
+	}
+
+	if (!loc.inode)
+		goto out;
+
+	glret = glfs_loc_touchup (&loc);
+	if (glret < 0) {
+		ret = -1;
+		goto out;
+	}
+
+	ret = syncop_lookup (subvol, &loc, NULL, &ciatt, NULL, NULL);
+	if (ret && reval) {
+		inode_unref (loc.inode);
+		loc.inode = inode_new (parent->table);
+		if (!loc.inode) {
+			errno = ENOMEM;
+			goto out;
+		}
+
+		xattr_req = dict_new ();
+		if (!xattr_req) {
+			errno = ENOMEM;
+			goto out;
+		}
+
+		uuid_generate (gfid);
+
+		ret = dict_set_static_bin (xattr_req, "gfid-req", gfid, 16);
+		if (ret) {
+			errno = ENOMEM;
+			goto out;
+		}
+
+		ret = syncop_lookup (subvol, &loc, xattr_req, &ciatt,
+				     NULL, NULL);
+	}
+	if (ret)
+		goto out;
+
+	inode = inode_link (loc.inode, loc.parent, component, &ciatt);
+found:
+	if (inode)
+		inode_lookup (inode);
+	if (iatt)
+		*iatt = ciatt;
+out:
+	if (xattr_req)
+		dict_unref (xattr_req);
+
+	loc_wipe (&loc);
+
+	return inode;
+}
+
+
+int
+glfs_resolve_at (struct glfs *fs, xlator_t *subvol, inode_t *at,
+		 const char *origpath, loc_t *loc, struct iatt *iatt,
+		 int follow, int reval)
+{
+	inode_t    *inode = NULL;
+	inode_t    *parent = NULL;
+	char       *saveptr = NULL;
+	char       *path = NULL;
+	char       *component = NULL;
+	char       *next_component = NULL;
+	int         ret = -1;
+	struct iatt ciatt = {0, };
+
+	path = gf_strdup (origpath);
+	if (!path) {
+		errno = ENOMEM;
+		return -1;
+	}
+
+	parent = NULL;
+	if (at && path[0] != '/') {
+		/* A relative resolution of a path which starts with '/'
+		   is equal to an absolute path resolution.
+		*/
+		inode = inode_ref (at);
+	} else {
+		inode = inode_ref (subvol->itable->root);
+
+		if (strcmp (path, "/") == 0)
+			glfs_resolve_base (fs, subvol, inode, &ciatt);
+	}
+
+	for (component = strtok_r (path, "/", &saveptr);
+	     component; component = next_component) {
+
+		next_component = strtok_r (NULL, "/", &saveptr);
+
+		if (parent)
+			inode_unref (parent);
+
+		parent = inode;
+
+		inode = glfs_resolve_component (fs, subvol, parent,
+						component, &ciatt,
+						/* force hard lookup on the last
+						   component, as the caller
+						   wants proper iatt filled
+						*/
+						(reval || (!next_component &&
+						iatt)));
+		if (!inode)
+			break;
+
+		if (IA_ISLNK (ciatt.ia_type) && (next_component || follow)) {
+			/* If the component is not the last piece,
+			   then following it is necessary even if
+			   not requested by the caller
+			*/
+			char *lpath = NULL;
+			loc_t sym_loc = {0,};
+
+			if (follow > GLFS_SYMLINK_MAX_FOLLOW) {
+				errno = ELOOP;
+				ret = -1;
+				if (inode) {
+					inode_unref (inode);
+					inode = NULL;
+				}
+				break;
+			}
+
+			ret = glfs_resolve_symlink (fs, subvol, inode, &lpath);
+			inode_unref (inode);
+			inode = NULL;
+			if (ret < 0)
+				break;
+
+			ret = glfs_resolve_at (fs, subvol, parent, lpath,
+					       &sym_loc,
+					       /* followed iatt becomes the
+						  component iatt
+					       */
+					       &ciatt,
+					       /* always recurisvely follow while
+						  following symlink
+					       */
+					       follow + 1, reval);
+			if (ret == 0)
+				inode = inode_ref (sym_loc.inode);
+			loc_wipe (&sym_loc);
+			GF_FREE (lpath);
+		}
+
+		if (!next_component)
+			break;
+
+		if (!IA_ISDIR (ciatt.ia_type)) {
+			/* next_component exists and this component is
+			   not a directory
+			*/
+			inode_unref (inode);
+			inode = NULL;
+			ret = -1;
+			errno = ENOTDIR;
+			break;
+		}
+	}
+
+	if (parent && next_component)
+		/* resolution failed mid-way */
+		goto out;
+
+	/* At this point, all components up to the last parent directory
+	   have been resolved successfully (@parent). Resolution of basename
+	   might have failed (@inode) if at all.
+	*/
+
+	loc->parent = parent;
+	if (parent) {
+		uuid_copy (loc->pargfid, parent->gfid);
+		loc->name = component;
+	}
+
+	loc->inode = inode;
+	if (inode) {
+		uuid_copy (loc->gfid, inode->gfid);
+		if (iatt)
+			*iatt = ciatt;
+		ret = 0;
+	}
+
+	glfs_loc_touchup (loc);
+out:
+	GF_FREE (path);
+
+	/* do NOT loc_wipe here as only last component might be missing */
+
+	return ret;
+}
+
+
+int
+glfs_resolve_path (struct glfs *fs, xlator_t *subvol, const char *origpath,
+		   loc_t *loc, struct iatt *iatt, int follow, int reval)
+{
+	int ret = -1;
+	inode_t *cwd = NULL;
+
+	if (origpath[0] == '/')
+		return glfs_resolve_at (fs, subvol, NULL, origpath, loc, iatt,
+					follow, reval);
+
+	cwd = glfs_cwd_get (fs);
+
+	ret = glfs_resolve_at (fs, subvol, cwd, origpath, loc, iatt,
+			       follow, reval);
+	if (cwd)
+		inode_unref (cwd);
+
+	return ret;
+}
+
+
+int
+glfs_resolve (struct glfs *fs, xlator_t *subvol, const char *origpath,
+	      loc_t *loc, struct iatt *iatt, int reval)
+{
+	int ret = -1;
+
+	ret = glfs_resolve_path (fs, subvol, origpath, loc, iatt, 1, reval);
+
+	return ret;
+}
+
+
+int
+glfs_lresolve (struct glfs *fs, xlator_t *subvol, const char *origpath,
+	       loc_t *loc, struct iatt *iatt, int reval)
+{
+	int ret = -1;
+
+	ret = glfs_resolve_path (fs, subvol, origpath, loc, iatt, 0, reval);
+
+	return ret;
+}
+
+
+int
+glfs_migrate_fd_locks_safe (struct glfs *fs, xlator_t *oldsubvol, fd_t *oldfd,
+			    xlator_t *newsubvol, fd_t *newfd)
+{
+	dict_t *lockinfo = NULL;
+	int ret = 0;
+	char uuid1[64];
+
+	if (!oldfd->lk_ctx || fd_lk_ctx_empty (oldfd->lk_ctx))
+		return 0;
+
+	newfd->lk_ctx = fd_lk_ctx_ref (oldfd->lk_ctx);
+
+	ret = syncop_fgetxattr (oldsubvol, oldfd, &lockinfo,
+				GF_XATTR_LOCKINFO_KEY);
+	if (ret < 0) {
+		gf_log (fs->volname, GF_LOG_WARNING,
+			"fgetxattr (%s) failed (%s) on graph %s (%d)",
+			uuid_utoa_r (oldfd->inode->gfid, uuid1),
+			strerror (errno),
+			graphid_str (oldsubvol), oldsubvol->graph->id);
+		goto out;
+	}
+
+	if (!dict_get (lockinfo, GF_XATTR_LOCKINFO_KEY)) {
+		gf_log (fs->volname, GF_LOG_WARNING,
+			"missing lokinfo key (%s) on graph %s (%d)",
+			uuid_utoa_r (oldfd->inode->gfid, uuid1),
+			graphid_str (oldsubvol), oldsubvol->graph->id);
+		goto out;
+	}
+
+	ret = syncop_fsetxattr (newsubvol, newfd, lockinfo, 0);
+	if (ret < 0) {
+		gf_log (fs->volname, GF_LOG_WARNING,
+			"fsetxattr (%s) failed (%s) on graph %s (%d)",
+			uuid_utoa_r (newfd->inode->gfid, uuid1),
+			strerror (errno),
+			graphid_str (newsubvol), newsubvol->graph->id);
+		goto out;
+	}
+out:
+	if (lockinfo)
+		dict_unref (lockinfo);
+	return ret;
+}
+
+
+fd_t *
+glfs_migrate_fd_safe (struct glfs *fs, xlator_t *newsubvol, fd_t *oldfd)
+{
+	fd_t *newfd = NULL;
+	inode_t *oldinode = NULL;
+	inode_t *newinode = NULL;
+	xlator_t *oldsubvol = NULL;
+	int ret = -1;
+	loc_t loc = {0, };
+	char uuid1[64];
+
+
+	oldinode = oldfd->inode;
+	oldsubvol = oldinode->table->xl;
+
+	if (oldsubvol == newsubvol)
+		return fd_ref (oldfd);
+
+	if (!oldsubvol->switched) {
+		ret = syncop_fsync (oldsubvol, oldfd, 0);
+		if (ret) {
+			gf_log (fs->volname, GF_LOG_WARNING,
+				"fsync() failed (%s) on %s graph %s (%d)",
+				strerror (errno),
+				uuid_utoa_r (oldfd->inode->gfid, uuid1),
+				graphid_str (oldsubvol), oldsubvol->graph->id);
+		}
+	}
+
+	newinode = glfs_refresh_inode_safe (newsubvol, oldinode);
+	if (!newinode) {
+		gf_log (fs->volname, GF_LOG_WARNING,
+			"inode (%s) refresh failed (%s) on graph %s (%d)",
+			uuid_utoa_r (oldinode->gfid, uuid1),
+			strerror (errno),
+			graphid_str (newsubvol), newsubvol->graph->id);
+		goto out;
+	}
+
+	newfd = fd_create (newinode, getpid());
+	if (!newfd) {
+		gf_log (fs->volname, GF_LOG_WARNING,
+			"fd_create (%s) failed (%s) on graph %s (%d)",
+			uuid_utoa_r (newinode->gfid, uuid1),
+			strerror (errno),
+			graphid_str (newsubvol), newsubvol->graph->id);
+		goto out;
+	}
+
+	loc.inode = inode_ref (newinode);
+
+        ret = inode_path (oldfd->inode, NULL, (char **)&loc.path);
+        if (ret < 0) {
+                gf_log (fs->volname, GF_LOG_INFO, "inode_path failed");
+                goto out;
+        }
+
+        uuid_copy (loc.gfid, oldinode->gfid);
+
+
+	if (IA_ISDIR (oldinode->ia_type))
+		ret = syncop_opendir (newsubvol, &loc, newfd);
+	else
+		ret = syncop_open (newsubvol, &loc,
+				   oldfd->flags & ~(O_TRUNC|O_EXCL|O_CREAT),
+				   newfd);
+	loc_wipe (&loc);
+
+	if (ret) {
+		gf_log (fs->volname, GF_LOG_WARNING,
+			"syncop_open%s (%s) failed (%s) on graph %s (%d)",
+			IA_ISDIR (oldinode->ia_type) ? "dir" : "",
+			uuid_utoa_r (newinode->gfid, uuid1),
+			strerror (errno),
+			graphid_str (newsubvol), newsubvol->graph->id);
+		goto out;
+	}
+
+	ret = glfs_migrate_fd_locks_safe (fs, oldsubvol, oldfd, newsubvol,
+					  newfd);
+
+	if (ret) {
+		gf_log (fs->volname, GF_LOG_WARNING,
+			"lock migration (%s) failed (%s) on graph %s (%d)",
+			uuid_utoa_r (newinode->gfid, uuid1),
+			strerror (errno),
+			graphid_str (newsubvol), newsubvol->graph->id);
+		goto out;
+	}
+
+        newfd->flags = oldfd->flags;
+	fd_bind (newfd);
+out:
+	if (newinode)
+		inode_unref (newinode);
+
+	if (ret) {
+		fd_unref (newfd);
+		newfd = NULL;
+	}
+
+	return newfd;
+}
+
+
+fd_t *
+__glfs_migrate_fd (struct glfs *fs, xlator_t *newsubvol, struct glfs_fd *glfd)
+{
+	fd_t *oldfd = NULL;
+	fd_t *newfd = NULL;
+
+	oldfd = glfd->fd;
+
+	fs->migration_in_progress = 1;
+	pthread_mutex_unlock (&fs->mutex);
+	{
+		newfd = glfs_migrate_fd_safe (fs, newsubvol, oldfd);
+	}
+	pthread_mutex_lock (&fs->mutex);
+	fs->migration_in_progress = 0;
+	pthread_cond_broadcast (&fs->cond);
+
+	return newfd;
+}
+
+
+fd_t *
+__glfs_resolve_fd (struct glfs *fs, xlator_t *subvol, struct glfs_fd *glfd)
+{
+	fd_t *fd = NULL;
+
+	if (glfd->fd->inode->table->xl == subvol)
+		return fd_ref (glfd->fd);
+
+	fd = __glfs_migrate_fd (fs, subvol, glfd);
+	if (!fd)
+		return NULL;
+
+	if (subvol == fs->active_subvol) {
+		fd_unref (glfd->fd);
+		glfd->fd = fd_ref (fd);
+	}
+
+	return fd;
+}
+
+
+fd_t *
+glfs_resolve_fd (struct glfs *fs, xlator_t *subvol, struct glfs_fd *glfd)
+{
+	fd_t *fd = NULL;
+
+	glfs_lock (fs);
+	{
+		fd = __glfs_resolve_fd (fs, subvol, glfd);
+	}
+	glfs_unlock (fs);
+
+	return fd;
+}
+
+
+void
+__glfs_migrate_openfds (struct glfs *fs, xlator_t *subvol)
+{
+	struct glfs_fd *glfd = NULL;
+	fd_t *fd = NULL;
+
+	list_for_each_entry (glfd, &fs->openfds, openfds) {
+		if (uuid_is_null (glfd->fd->inode->gfid)) {
+			gf_log (fs->volname, GF_LOG_INFO,
+				"skipping openfd %p/%p in graph %s (%d)",
+				glfd, glfd->fd,	graphid_str(subvol),
+				subvol->graph->id);
+			/* create in progress, defer */
+			continue;
+		}
+
+		fd = __glfs_migrate_fd (fs, subvol, glfd);
+		if (fd) {
+			fd_unref (glfd->fd);
+			glfd->fd = fd;
+		}
+	}
+}
+
+
+xlator_t *
+__glfs_active_subvol (struct glfs *fs)
+{
+	xlator_t      *new_subvol = NULL;
+	int            ret = -1;
+	inode_t       *new_cwd = NULL;
+
+	if (!fs->next_subvol)
+		return fs->active_subvol;
+
+	new_subvol = fs->next_subvol;
+
+	ret = __glfs_first_lookup (fs, new_subvol);
+	if (ret) {
+		gf_log (fs->volname, GF_LOG_INFO,
+			"first lookup on graph %s (%d) failed (%s)",
+			graphid_str (new_subvol), new_subvol->graph->id,
+			strerror (errno));
+		return NULL;
+	}
+
+	if (fs->cwd) {
+		new_cwd = __glfs_refresh_inode (fs, new_subvol, fs->cwd);
+
+		if (!new_cwd) {
+			char buf1[64];
+			gf_log (fs->volname, GF_LOG_INFO,
+				"cwd refresh of %s graph %s (%d) failed (%s)",
+				uuid_utoa_r (fs->cwd->gfid, buf1),
+				graphid_str (new_subvol),
+				new_subvol->graph->id, strerror (errno));
+			return NULL;
+		}
+	}
+
+	__glfs_migrate_openfds (fs, new_subvol);
+
+	/* switching @active_subvol and @cwd
+	   should be atomic
+	*/
+	fs->old_subvol = fs->active_subvol;
+	fs->active_subvol = fs->next_subvol;
+	fs->next_subvol = NULL;
+
+	if (new_cwd) {
+		__glfs_cwd_set (fs, new_cwd);
+		inode_unref (new_cwd);
+	}
+
+	gf_log (fs->volname, GF_LOG_INFO, "switched to graph %s (%d)",
+		graphid_str (new_subvol), new_subvol->graph->id);
+
+	return new_subvol;
+}
+
+xlator_t *
+glfs_active_subvol (struct glfs *fs)
+{
+	xlator_t      *subvol = NULL;
+	xlator_t      *old_subvol = NULL;
+
+	glfs_lock (fs);
+	{
+		subvol = __glfs_active_subvol (fs);
+
+		if (subvol)
+			subvol->winds++;
+
+		if (fs->old_subvol) {
+			old_subvol = fs->old_subvol;
+			fs->old_subvol = NULL;
+			old_subvol->switched = 1;
+		}
+	}
+	glfs_unlock (fs);
+
+	if (old_subvol)
+		glfs_subvol_done (fs, old_subvol);
+
+	return subvol;
+}
+
+
+void
+glfs_subvol_done (struct glfs *fs, xlator_t *subvol)
+{
+	int ref = 0;
+	xlator_t *active_subvol = NULL;
+
+	if (!subvol)
+		return;
+
+	glfs_lock (fs);
+	{
+		ref = (--subvol->winds);
+		active_subvol = fs->active_subvol;
+	}
+	glfs_unlock (fs);
+
+	if (ref == 0) {
+		assert (subvol != active_subvol);
+		xlator_notify (subvol, GF_EVENT_PARENT_DOWN, subvol, NULL);
+	}
+}
+
+
+int
+__glfs_cwd_set (struct glfs *fs, inode_t *inode)
+{
+	if (inode->table->xl != fs->active_subvol) {
+		inode = __glfs_refresh_inode (fs, fs->active_subvol, inode);
+		if (!inode)
+			return -1;
+	} else {
+		inode_ref (inode);
+	}
+
+	if (fs->cwd)
+		inode_unref (fs->cwd);
+
+	fs->cwd = inode;
+
+	return 0;
+}
+
+
+int
+glfs_cwd_set (struct glfs *fs, inode_t *inode)
+{
+	int ret = 0;
+
+	glfs_lock (fs);
+	{
+		ret = __glfs_cwd_set (fs, inode);
+	}
+	glfs_unlock (fs);
+
+	return ret;
+}
+
+
+inode_t *
+__glfs_cwd_get (struct glfs *fs)
+{
+	inode_t *cwd = NULL;
+
+	if (!fs->cwd)
+		return NULL;
+
+	if (fs->cwd->table->xl == fs->active_subvol) {
+		cwd = inode_ref (fs->cwd);
+		return cwd;
+	}
+
+	cwd = __glfs_refresh_inode (fs, fs->active_subvol, fs->cwd);
+
+	return cwd;
+}
+
+inode_t *
+glfs_cwd_get (struct glfs *fs)
+{
+	inode_t *cwd = NULL;
+
+	glfs_lock (fs);
+	{
+		cwd = __glfs_cwd_get (fs);
+	}
+	glfs_unlock (fs);
+
+	return cwd;
+}
+
+inode_t *
+__glfs_resolve_inode (struct glfs *fs, xlator_t *subvol,
+		    struct glfs_object *object)
+{
+	inode_t *inode = NULL;
+
+	if (object->inode->table->xl == subvol)
+		return inode_ref (object->inode);
+
+	inode = __glfs_refresh_inode (fs, fs->active_subvol,
+					object->inode);
+	if (!inode)
+		return NULL;
+
+	if (subvol == fs->active_subvol) {
+		inode_unref (object->inode);
+		object->inode = inode_ref (inode);
+	}
+
+	return inode;
+}
+
+inode_t *
+glfs_resolve_inode (struct glfs *fs, xlator_t *subvol,
+		    struct glfs_object *object)
+{
+	inode_t *inode = NULL;
+
+	glfs_lock (fs);
+	{
+		inode = __glfs_resolve_inode(fs, subvol, object);
+	}
+	glfs_unlock (fs);
+
+	return inode;
+}
+
+int
+glfs_create_object (loc_t *loc, struct glfs_object **retobject)
+{
+	struct glfs_object *object = NULL;
+
+	object = GF_CALLOC (1, sizeof(struct glfs_object),
+			    glfs_mt_glfs_object_t);
+	if (object == NULL) {
+		errno = ENOMEM;
+		return -1;
+	}
+
+	object->inode = loc->inode;
+	uuid_copy (object->gfid, object->inode->gfid);
+
+	/* we hold the reference */
+	loc->inode = NULL;
+
+	*retobject = object;
+
+	return 0;
+}
diff --git a/api/src/glfs.c b/api/src/glfs.c
new file mode 100644
index 000000000..29ed47c0c
--- /dev/null
+++ b/api/src/glfs.c
@@ -0,0 +1,673 @@
+/*
+  Copyright (c) 2012 Red Hat, Inc. <http://www.redhat.com>
+  This file is part of GlusterFS.
+
+  This file is licensed to you under your choice of the GNU Lesser
+  General Public License, version 3 or any later version (LGPLv3 or
+  later), or the GNU General Public License, version 2 (GPLv2), in all
+  cases as published by the Free Software Foundation.
+*/
+
+
+/*
+  TODO:
+  - merge locks in glfs_posix_lock for lock self-healing
+  - set proper pid/lk_owner to call frames (currently buried in syncop)
+  - fix logging.c/h to store logfp and loglevel in glusterfs_ctx_t and
+    reach it via THIS.
+  - update syncop functions to accept/return xdata. ???
+  - protocol/client to reconnect immediately after portmap disconnect.
+  - handle SEEK_END failure in _lseek()
+  - handle umask (per filesystem?)
+  - make itables LRU based
+  - 0-copy for readv/writev
+  - reconcile the open/creat mess
+*/
+
+#include <unistd.h>
+#include <string.h>
+#include <stdlib.h>
+#include <stdio.h>
+#include <inttypes.h>
+#include <sys/types.h>
+#include <unistd.h>
+#include <limits.h>
+
+#ifndef _CONFIG_H
+#define _CONFIG_H
+#include "config.h"
+#endif
+
+#include "glusterfs.h"
+#include "logging.h"
+#include "stack.h"
+#include "event.h"
+#include "glfs-mem-types.h"
+#include "common-utils.h"
+#include "syncop.h"
+#include "call-stub.h"
+
+#include "glfs.h"
+#include "glfs-internal.h"
+#include "hashfn.h"
+#include "rpc-clnt.h"
+
+
+static gf_boolean_t
+vol_assigned (cmd_args_t *args)
+{
+	return args->volfile || args->volfile_server;
+}
+
+
+static int
+glusterfs_ctx_defaults_init (glusterfs_ctx_t *ctx)
+{
+	call_pool_t   *pool = NULL;
+	int	       ret = -1;
+
+	xlator_mem_acct_init (THIS, glfs_mt_end + 1);
+
+	ctx->process_uuid = generate_glusterfs_ctx_id ();
+	if (!ctx->process_uuid) {
+		goto err;
+	}
+
+	ctx->page_size	= 128 * GF_UNIT_KB;
+
+	ctx->iobuf_pool = iobuf_pool_new ();
+	if (!ctx->iobuf_pool) {
+		goto err;
+	}
+
+	ctx->event_pool = event_pool_new (DEFAULT_EVENT_POOL_SIZE);
+	if (!ctx->event_pool) {
+		goto err;
+	}
+
+	ctx->env = syncenv_new (0, 0, 0);
+	if (!ctx->env) {
+		goto err;
+	}
+
+	pool = GF_CALLOC (1, sizeof (call_pool_t),
+			  glfs_mt_call_pool_t);
+	if (!pool) {
+		goto err;
+	}
+
+	/* frame_mem_pool size 112 * 4k */
+	pool->frame_mem_pool = mem_pool_new (call_frame_t, 4096);
+	if (!pool->frame_mem_pool) {
+		goto err;
+	}
+	/* stack_mem_pool size 256 * 1024 */
+	pool->stack_mem_pool = mem_pool_new (call_stack_t, 1024);
+	if (!pool->stack_mem_pool) {
+		goto err;
+	}
+
+	ctx->stub_mem_pool = mem_pool_new (call_stub_t, 1024);
+	if (!ctx->stub_mem_pool) {
+		goto err;
+	}
+
+	ctx->dict_pool = mem_pool_new (dict_t, GF_MEMPOOL_COUNT_OF_DICT_T);
+	if (!ctx->dict_pool)
+		goto err;
+
+	ctx->dict_pair_pool = mem_pool_new (data_pair_t,
+					    GF_MEMPOOL_COUNT_OF_DATA_PAIR_T);
+	if (!ctx->dict_pair_pool)
+		goto err;
+
+	ctx->dict_data_pool = mem_pool_new (data_t, GF_MEMPOOL_COUNT_OF_DATA_T);
+	if (!ctx->dict_data_pool)
+		goto err;
+
+	INIT_LIST_HEAD (&pool->all_frames);
+	INIT_LIST_HEAD (&ctx->cmd_args.xlator_options);
+	LOCK_INIT (&pool->lock);
+	ctx->pool = pool;
+
+	pthread_mutex_init (&(ctx->lock), NULL);
+
+	ret = 0;
+err:
+	if (ret && pool) {
+		if (pool->frame_mem_pool)
+			mem_pool_destroy (pool->frame_mem_pool);
+		if (pool->stack_mem_pool)
+			mem_pool_destroy (pool->stack_mem_pool);
+		GF_FREE (pool);
+	}
+
+	if (ret && ctx) {
+		if (ctx->stub_mem_pool)
+			mem_pool_destroy (ctx->stub_mem_pool);
+		if (ctx->dict_pool)
+			mem_pool_destroy (ctx->dict_pool);
+		if (ctx->dict_data_pool)
+			mem_pool_destroy (ctx->dict_data_pool);
+		if (ctx->dict_pair_pool)
+			mem_pool_destroy (ctx->dict_pair_pool);
+	}
+
+	return ret;
+}
+
+
+static int
+create_master (struct glfs *fs)
+{
+	int		 ret = 0;
+	xlator_t	*master = NULL;
+
+	master = GF_CALLOC (1, sizeof (*master),
+			    glfs_mt_xlator_t);
+	if (!master)
+		goto err;
+
+	master->name = gf_strdup ("gfapi");
+	if (!master->name)
+		goto err;
+
+	if (xlator_set_type (master, "mount/api") == -1) {
+		gf_log ("glfs", GF_LOG_ERROR,
+			"master xlator for %s initialization failed",
+			fs->volname);
+		goto err;
+	}
+
+	master->ctx	 = fs->ctx;
+	master->private	 = fs;
+	master->options	 = get_new_dict ();
+	if (!master->options)
+		goto err;
+
+
+	ret = xlator_init (master);
+	if (ret) {
+		gf_log ("glfs", GF_LOG_ERROR,
+			"failed to initialize gfapi translator");
+		goto err;
+	}
+
+	fs->ctx->master = master;
+	THIS = master;
+
+	return 0;
+
+err:
+	if (master) {
+		xlator_destroy (master);
+	}
+
+	return -1;
+}
+
+
+static FILE *
+get_volfp (struct glfs *fs)
+{
+	int	     ret = 0;
+	cmd_args_t  *cmd_args = NULL;
+	FILE	    *specfp = NULL;
+	struct stat  statbuf;
+
+	cmd_args = &fs->ctx->cmd_args;
+
+	ret = lstat (cmd_args->volfile, &statbuf);
+	if (ret == -1) {
+		gf_log ("glfs", GF_LOG_ERROR,
+			"%s: %s", cmd_args->volfile, strerror (errno));
+		return NULL;
+	}
+
+	if ((specfp = fopen (cmd_args->volfile, "r")) == NULL) {
+		gf_log ("glfs", GF_LOG_ERROR,
+			"volume file %s: %s",
+			cmd_args->volfile,
+			strerror (errno));
+		return NULL;
+	}
+
+	gf_log ("glfs", GF_LOG_DEBUG,
+		"loading volume file %s", cmd_args->volfile);
+
+	return specfp;
+}
+
+
+int
+glfs_volumes_init (struct glfs *fs)
+{
+	FILE		   *fp = NULL;
+	cmd_args_t	   *cmd_args = NULL;
+	int		    ret = 0;
+
+	cmd_args = &fs->ctx->cmd_args;
+
+	if (!vol_assigned (cmd_args))
+		return -1;
+
+	if (cmd_args->volfile_server) {
+		ret = glfs_mgmt_init (fs);
+		goto out;
+	}
+
+	fp = get_volfp (fs);
+
+	if (!fp) {
+		gf_log ("glfs", GF_LOG_ERROR,
+			"Cannot reach volume specification file");
+		ret = -1;
+		goto out;
+	}
+
+	ret = glfs_process_volfp (fs, fp);
+	if (ret)
+		goto out;
+
+out:
+	return ret;
+}
+
+
+///////////////////////////////////////////////////////////////////////////////
+
+
+int
+glfs_set_xlator_option (struct glfs *fs, const char *xlator, const char *key,
+			const char *value)
+{
+	xlator_cmdline_option_t *option = NULL;
+
+	option = GF_CALLOC (1, sizeof (*option),
+			    glfs_mt_xlator_cmdline_option_t);
+	if (!option)
+		goto enomem;
+
+	INIT_LIST_HEAD (&option->cmd_args);
+
+	option->volume = gf_strdup (xlator);
+	if (!option->volume)
+		goto enomem;
+	option->key = gf_strdup (key);
+	if (!option->key)
+		goto enomem;
+	option->value = gf_strdup (value);
+	if (!option->value)
+		goto enomem;
+
+	list_add (&option->cmd_args, &fs->ctx->cmd_args.xlator_options);
+
+	return 0;
+enomem:
+	errno = ENOMEM;
+
+	if (!option)
+		return -1;
+
+	GF_FREE (option->volume);
+	GF_FREE (option->key);
+	GF_FREE (option->value);
+	GF_FREE (option);
+
+	return -1;
+}
+
+int glfs_setfsuid (uid_t fsuid)
+{
+	return syncopctx_setfsuid (&fsuid);
+}
+
+int glfs_setfsgid (gid_t fsgid)
+{
+	return syncopctx_setfsgid (&fsgid);
+}
+
+int glfs_setfsgroups (size_t size, const gid_t *list)
+{
+	return syncopctx_setfsgroups(size, list);
+}
+
+struct glfs *
+glfs_from_glfd (struct glfs_fd *glfd)
+{
+	return glfd->fs;
+}
+
+
+struct glfs_fd *
+glfs_fd_new (struct glfs *fs)
+{
+	struct glfs_fd  *glfd = NULL;
+
+	glfd = GF_CALLOC (1, sizeof (*glfd), glfs_mt_glfs_fd_t);
+	if (!glfd)
+		return NULL;
+
+	glfd->fs = fs;
+
+	INIT_LIST_HEAD (&glfd->openfds);
+
+	return glfd;
+}
+
+
+void
+glfs_fd_bind (struct glfs_fd *glfd)
+{
+	struct glfs *fs = NULL;
+
+	fs = glfd->fs;
+
+	glfs_lock (fs);
+	{
+		list_add_tail (&glfd->openfds, &fs->openfds);
+	}
+	glfs_unlock (fs);
+}
+
+void
+glfs_fd_destroy (struct glfs_fd *glfd)
+{
+	if (!glfd)
+		return;
+
+	glfs_lock (glfd->fs);
+	{
+		list_del_init (&glfd->openfds);
+	}
+	glfs_unlock (glfd->fs);
+
+	if (glfd->fd)
+		fd_unref (glfd->fd);
+
+	GF_FREE (glfd->readdirbuf);
+
+	GF_FREE (glfd);
+}
+
+
+static void *
+glfs_poller (void *data)
+{
+	struct glfs  *fs = NULL;
+
+	fs = data;
+
+	event_dispatch (fs->ctx->event_pool);
+
+	return NULL;
+}
+
+
+struct glfs *
+glfs_new (const char *volname)
+{
+	struct glfs     *fs = NULL;
+	int              ret = -1;
+	glusterfs_ctx_t *ctx = NULL;
+
+	ctx = glusterfs_ctx_new ();
+	if (!ctx) {
+		return NULL;
+	}
+
+#ifdef DEBUG
+        gf_mem_acct_enable_set (ctx);
+#endif
+
+	/* first globals init, for gf_mem_acct_enable_set () */
+	ret = glusterfs_globals_init (ctx);
+	if (ret)
+		return NULL;
+
+	THIS->ctx = ctx;
+
+	/* then ctx_defaults_init, for xlator_mem_acct_init(THIS) */
+	ret = glusterfs_ctx_defaults_init (ctx);
+	if (ret)
+		return NULL;
+
+	fs = GF_CALLOC (1, sizeof (*fs), glfs_mt_glfs_t);
+	if (!fs)
+		return NULL;
+	fs->ctx = ctx;
+
+	glfs_set_logging (fs, "/dev/null", 0);
+
+	fs->ctx->cmd_args.volfile_id = gf_strdup (volname);
+
+	fs->volname = gf_strdup (volname);
+
+	pthread_mutex_init (&fs->mutex, NULL);
+	pthread_cond_init (&fs->cond, NULL);
+
+	INIT_LIST_HEAD (&fs->openfds);
+
+	return fs;
+}
+
+
+int
+glfs_set_volfile (struct glfs *fs, const char *volfile)
+{
+	cmd_args_t  *cmd_args = NULL;
+
+	cmd_args = &fs->ctx->cmd_args;
+
+	if (vol_assigned (cmd_args))
+		return -1;
+
+	cmd_args->volfile = gf_strdup (volfile);
+
+	return 0;
+}
+
+
+int
+glfs_set_volfile_server (struct glfs *fs, const char *transport,
+			 const char *host, int port)
+{
+	cmd_args_t  *cmd_args = NULL;
+
+	cmd_args = &fs->ctx->cmd_args;
+
+	if (vol_assigned (cmd_args))
+		return -1;
+
+	cmd_args->volfile_server = gf_strdup (host);
+	cmd_args->volfile_server_transport = gf_strdup (transport);
+	cmd_args->volfile_server_port = port;
+	cmd_args->max_connect_attempts = 2;
+
+	return 0;
+}
+
+
+int
+glfs_set_logging (struct glfs *fs, const char *logfile, int loglevel)
+{
+	int  ret = 0;
+        char *tmplog = NULL;
+
+        if (!logfile) {
+                ret = gf_set_log_file_path (&fs->ctx->cmd_args);
+                if (ret)
+                        goto out;
+                tmplog = fs->ctx->cmd_args.log_file;
+        } else {
+                tmplog = (char *)logfile;
+        }
+
+        ret = gf_log_init (fs->ctx, tmplog, NULL);
+        if (ret)
+                goto out;
+
+	if (loglevel >= 0)
+		gf_log_set_loglevel (loglevel);
+
+out:
+	return ret;
+}
+
+
+int
+glfs_init_wait (struct glfs *fs)
+{
+	int   ret = -1;
+
+	/* Always a top-down call, use glfs_lock() */
+	glfs_lock (fs);
+	{
+		while (!fs->init)
+			pthread_cond_wait (&fs->cond,
+					   &fs->mutex);
+		ret = fs->ret;
+		errno = fs->err;
+	}
+	glfs_unlock (fs);
+
+	return ret;
+}
+
+
+void
+glfs_init_done (struct glfs *fs, int ret)
+{
+	glfs_init_cbk init_cbk;
+
+	if (!fs) {
+		gf_log ("glfs", GF_LOG_ERROR,
+			"fs is NULL");
+		goto out;
+	}
+
+	init_cbk = fs->init_cbk;
+
+	/* Always a bottom-up call, use mutex_lock() */
+	pthread_mutex_lock (&fs->mutex);
+	{
+		fs->init = 1;
+		fs->ret = ret;
+		fs->err = errno;
+
+		if (!init_cbk)
+			pthread_cond_broadcast (&fs->cond);
+	}
+	pthread_mutex_unlock (&fs->mutex);
+
+	if (init_cbk)
+		init_cbk (fs, ret);
+out:
+	return;
+}
+
+
+int
+glfs_init_common (struct glfs *fs)
+{
+	int  ret = -1;
+
+	ret = create_master (fs);
+	if (ret)
+		return ret;
+
+	ret = gf_thread_create (&fs->poller, NULL, glfs_poller, fs);
+	if (ret)
+		return ret;
+
+	ret = glfs_volumes_init (fs);
+	if (ret)
+		return ret;
+
+	fs->dev_id = gf_dm_hashfn (fs->volname, strlen (fs->volname));
+	return ret;
+}
+
+
+int
+glfs_init_async (struct glfs *fs, glfs_init_cbk cbk)
+{
+	int  ret = -1;
+
+	fs->init_cbk = cbk;
+
+	ret = glfs_init_common (fs);
+
+	return ret;
+}
+
+
+int
+glfs_init (struct glfs *fs)
+{
+	int  ret = -1;
+
+	ret = glfs_init_common (fs);
+	if (ret)
+		return ret;
+
+	ret = glfs_init_wait (fs);
+
+	return ret;
+}
+
+
+int
+glfs_fini (struct glfs *fs)
+{
+        int             ret = -1;
+        int             countdown = 100;
+        xlator_t        *subvol = NULL;
+        glusterfs_ctx_t *ctx = NULL;
+        call_pool_t     *call_pool = NULL;
+
+        ctx = fs->ctx;
+
+        if (ctx->mgmt) {
+                rpc_clnt_disable (ctx->mgmt);
+                ctx->mgmt = NULL;
+        }
+
+        __glfs_entry_fs (fs);
+
+        call_pool = fs->ctx->pool;
+
+        while (countdown--) {
+                /* give some time for background frames to finish */
+                if (!call_pool->cnt)
+                        break;
+                usleep (100000);
+        }
+        /* leaked frames may exist, we ignore */
+
+        /*We deem glfs_fini as successful if there are no pending frames in the call
+         *pool*/
+        ret = (call_pool->cnt == 0)? 0: -1;
+
+        subvol = glfs_active_subvol (fs);
+        if (subvol) {
+                /* PARENT_DOWN within glfs_subvol_done() is issued only
+                   on graph switch (new graph should activiate and
+                   decrement the extra @winds count taken in glfs_graph_setup()
+
+                   Since we are explicitly destroying, PARENT_DOWN is necessary
+                */
+                xlator_notify (subvol, GF_EVENT_PARENT_DOWN, subvol, 0);
+                /* TBD: wait for CHILD_DOWN before exiting, in case of
+                   asynchronous cleanup like graceful socket disconnection
+                   in the future.
+                */
+        }
+
+        glfs_subvol_done (fs, subvol);
+
+        if (ctx->log.logfile)
+                fclose (ctx->log.logfile);
+
+        return ret;
+}
diff --git a/api/src/glfs.h b/api/src/glfs.h
new file mode 100644
index 000000000..18fda496e
--- /dev/null
+++ b/api/src/glfs.h
@@ -0,0 +1,581 @@
+/*
+  Copyright (c) 2012 Red Hat, Inc. <http://www.redhat.com>
+  This file is part of GlusterFS.
+
+  This file is licensed to you under your choice of the GNU Lesser
+  General Public License, version 3 or any later version (LGPLv3 or
+  later), or the GNU General Public License, version 2 (GPLv2), in all
+  cases as published by the Free Software Foundation.
+*/
+
+
+#ifndef _GLFS_H
+#define _GLFS_H
+
+/*
+  Enforce the following flags as libgfapi is built
+  with them, and we want programs linking against them to also
+  be built with these flags. This is necessary as it affects
+  some of the structures defined in libc headers (like struct stat)
+  and those definitions need to be consistently compiled in
+  both the library and the application.
+*/
+
+#ifndef _FILE_OFFSET_BITS
+#define _FILE_OFFSET_BITS 64
+#endif
+
+#ifndef __USE_FILE_OFFSET64
+#define __USE_FILE_OFFSET64
+#endif
+
+#ifndef _GNU_SOURCE
+#define _GNU_SOURCE
+#endif
+
+#include <sys/types.h>
+#include <sys/stat.h>
+#include <fcntl.h>
+#include <sys/uio.h>
+#include <unistd.h>
+#include <sys/cdefs.h>
+#include <dirent.h>
+#include <sys/statvfs.h>
+
+__BEGIN_DECLS
+
+/* The filesystem object. One object per 'virtual mount' */
+struct glfs;
+typedef struct glfs glfs_t;
+
+
+/*
+  SYNOPSIS
+
+  glfs_new: Create a new 'virtual mount' object.
+
+  DESCRIPTION
+
+  This is most likely the very first function you will use. This function
+  will create a new glfs_t (virtual mount) object in memory.
+
+  On this newly created glfs_t, you need to be either set a volfile path
+  (glfs_set_volfile) or a volfile server (glfs_set_volfile_server).
+
+  The glfs_t object needs to be initialized with glfs_init() before you
+  can start issuing file operations on it.
+
+  PARAMETERS
+
+  @volname: Name of the volume. This identifies the server-side volume and
+            the fetched volfile (equivalent of --volfile-id command line
+	    parameter to glusterfsd). When used with glfs_set_volfile() the
+	    @volname has no effect (except for appearing in log messages).
+
+  RETURN VALUES
+
+  NULL   : Out of memory condition.
+  Others : Pointer to the newly created glfs_t virtual mount object.
+
+*/
+
+glfs_t *glfs_new (const char *volname);
+
+
+/*
+  SYNOPSIS
+
+  glfs_set_volfile: Specify the path to the volume specification file.
+
+  DESCRIPTION
+
+  If you are using a static volume specification file (without dynamic
+  volume management abilities from the CLI), then specify the path to
+  the volume specification file.
+
+  This is incompatible with glfs_set_volfile_server().
+
+  PARAMETERS
+
+  @fs: The 'virtual mount' object to be configured with the volume
+       specification file.
+
+  @volfile: Path to the locally available volume specification file.
+
+  RETURN VALUES
+
+   0 : Success.
+  -1 : Failure. @errno will be set with the type of failure.
+
+*/
+
+int glfs_set_volfile (glfs_t *fs, const char *volfile);
+
+
+/*
+  SYNOPSIS
+
+  glfs_set_volfile_server: Specify the address of management server.
+
+  DESCRIPTION
+
+  This function specifies the address of the management server (glusterd)
+  to connect, and establish the volume configuration. The @volname
+  parameter passed to glfs_new() is the volume which will be virtually
+  mounted as the glfs_t object. All operations performed by the CLI at
+  the management server will automatically be reflected in the 'virtual
+  mount' object as it maintains a connection to glusterd and polls on
+  configuration change notifications.
+
+  This is incompatible with glfs_set_volfile().
+
+  PARAMETERS
+
+  @fs: The 'virtual mount' object to be configured with the volume
+       specification file.
+
+  @transport: String specifying the transport used to connect to the
+              management daemon. Specifying NULL will result in the usage
+	      of the default (tcp) transport type. Permitted values
+	      are those what you specify as transport-type in a volume
+	      specification file (e.g "tcp", "rdma", "unix".)
+
+  @host: String specifying the address of where to find the management
+         daemon. Depending on the transport type this would either be
+	 an FQDN (e.g: "storage01.company.com"), ASCII encoded IP
+	 address "192.168.22.1", or a UNIX domain socket path (e.g
+	 "/tmp/glusterd.socket".)
+
+  @port: The TCP port number where gluster management daemon is listening.
+         Specifying 0 uses the default port number GF_DEFAULT_BASE_PORT.
+	 This parameter is unused if you are using a UNIX domain socket.
+
+  RETURN VALUES
+
+   0 : Success.
+  -1 : Failure. @errno will be set with the type of failure.
+
+*/
+
+int glfs_set_volfile_server (glfs_t *fs, const char *transport,
+			     const char *host, int port);
+
+
+/*
+  SYNOPSIS
+
+  glfs_set_logging: Specify logging parameters.
+
+  DESCRIPTION
+
+  This function specifies logging parameters for the virtual mount.
+  Default log file is /dev/null.
+
+  PARAMETERS
+
+  @fs: The 'virtual mount' object to be configured with the logging parameters.
+
+  @logfile: The logfile to be used for logging. Will be created if it does not
+            already exist (provided system permissions allow). If NULL, a new
+            logfile will be created in default log directory associated with
+            the glusterfs installation.
+
+  @loglevel: Numerical value specifying the degree of verbosity. Higher the
+             value, more verbose the logging.
+
+  RETURN VALUES
+
+   0 : Success.
+  -1 : Failure. @errno will be set with the type of failure.
+
+*/
+
+int glfs_set_logging (glfs_t *fs, const char *logfile, int loglevel);
+
+
+/*
+  SYNOPSIS
+
+  glfs_init: Initialize the 'virtual mount'
+
+  DESCRIPTION
+
+  This function initializes the glfs_t object. This consists of many steps:
+  - Spawn a poll-loop thread.
+  - Establish connection to management daemon and receive volume specification.
+  - Construct translator graph and initialize graph.
+  - Wait for initialization (connecting to all bricks) to complete.
+
+  PARAMETERS
+
+  @fs: The 'virtual mount' object to be initialized.
+
+  RETURN VALUES
+
+   0 : Success.
+  -1 : Failure. @errno will be set with the type of failure.
+
+*/
+
+int glfs_init (glfs_t *fs);
+
+
+/*
+  SYNOPSIS
+
+  glfs_fini: Cleanup and destroy the 'virtual mount'
+
+  DESCRIPTION
+
+  This function attempts to gracefully destroy glfs_t object. An attempt is
+  made to wait for all background processing to complete before returning.
+
+  glfs_fini() must be called after all operations on glfs_t is finished.
+
+  IMPORTANT
+
+  IT IS NECESSARY TO CALL glfs_fini() ON ALL THE INITIALIZED glfs_t
+  OBJECTS BEFORE TERMINATING THE PROGRAM. THERE MAY BE CACHED AND
+  UNWRITTEN / INCOMPLETE OPERATIONS STILL IN PROGRESS EVEN THOUGH THE
+  API CALLS HAVE RETURNED. glfs_fini() WILL WAIT FOR BACKGROUND OPERATIONS
+  TO COMPLETE BEFORE RETURNING, THEREBY MAKING IT SAFE FOR THE PROGRAM TO
+  EXIT.
+
+  PARAMETERS
+
+  @fs: The 'virtual mount' object to be destroyed.
+
+  RETURN VALUES
+
+   0 : Success.
+*/
+
+int glfs_fini (glfs_t *fs);
+
+/*
+ * FILE OPERATION
+ *
+ * What follows are filesystem operations performed on the
+ * 'virtual mount'. The calls here are kept as close to
+ * the POSIX system calls as possible.
+ *
+ * Notes:
+ *
+ * - All paths specified, even if absolute, are relative to the
+ *   root of the virtual mount and not the system root (/).
+ *
+ */
+
+/* The file descriptor object. One per open file/directory. */
+
+struct glfs_fd;
+typedef struct glfs_fd glfs_fd_t;
+
+/*
+ * PER THREAD IDENTITY MODIFIERS
+ *
+ * The following operations enable to set a per thread identity context
+ * for the glfs APIs to perform operations as. The calls here are kept as close
+ * to POSIX equivalents as possible.
+ *
+ * NOTES:
+ *
+ *  - setgroups is a per thread setting, hence this is named as fsgroups to be
+ *    close in naming to the fs(u/g)id APIs
+ *  - Typical mode of operation is to set the IDs as required, with the
+ *    supplementary groups being optionally set, make the glfs call and post the
+ *    glfs operation set them back to eu/gid or uid/gid as appropriate to the
+ *    caller
+ *  - The groups once set, need to be unset by setting the size to 0 (in which
+ *    case the list argument is a do not care)
+ *  - Once a process for a thread of operation choses to set the IDs, all glfs
+ *    calls made from that thread would default to the IDs set for the thread.
+ *    As a result use these APIs with care and ensure that the set IDs are
+ *    reverted to global process defaults as required.
+ *
+ */
+int glfs_setfsuid (uid_t fsuid);
+int glfs_setfsgid (gid_t fsgid);
+int glfs_setfsgroups (size_t size, const gid_t *list);
+
+/*
+  SYNOPSIS
+
+  glfs_open: Open a file.
+
+  DESCRIPTION
+
+  This function opens a file on a virtual mount.
+
+  PARAMETERS
+
+  @fs: The 'virtual mount' object to be initialized.
+
+  @path: Path of the file within the virtual mount.
+
+  @flags: Open flags. See open(2). O_CREAT is not supported.
+          Use glfs_creat() for creating files.
+
+  RETURN VALUES
+
+  NULL   : Failure. @errno will be set with the type of failure.
+  Others : Pointer to the opened glfs_fd_t.
+
+ */
+
+glfs_fd_t *glfs_open (glfs_t *fs, const char *path, int flags);
+
+
+/*
+  SYNOPSIS
+
+  glfs_creat: Create a file.
+
+  DESCRIPTION
+
+  This function opens a file on a virtual mount.
+
+  PARAMETERS
+
+  @fs: The 'virtual mount' object to be initialized.
+
+  @path: Path of the file within the virtual mount.
+
+  @mode: Permission of the file to be created.
+
+  @flags: Create flags. See open(2). O_EXCL is supported.
+
+  RETURN VALUES
+
+  NULL   : Failure. @errno will be set with the type of failure.
+  Others : Pointer to the opened glfs_fd_t.
+
+ */
+
+glfs_fd_t *glfs_creat (glfs_t *fs, const char *path, int flags,
+		       mode_t mode);
+
+int glfs_close (glfs_fd_t *fd);
+
+glfs_t *glfs_from_glfd (glfs_fd_t *fd);
+
+int glfs_set_xlator_option (glfs_t *fs, const char *xlator, const char *key,
+			    const char *value);
+
+/*
+
+  glfs_io_cbk
+
+  The following is the function type definition of the callback
+  function pointer which has to be provided by the caller to the
+  *_async() versions of the IO calls.
+
+  The callback function is called on completion of the requested
+  IO, and the appropriate return value is returned in @ret.
+
+  In case of an error in completing the IO, @ret will be -1 and
+  @errno will be set with the appropriate error.
+
+  @ret will be same as the return value of the non _async() variant
+  of the particular call
+
+  @data is the same context pointer provided by the caller at the
+  time of issuing the async IO call. This can be used by the
+  caller to differentiate different instances of the async requests
+  in a common callback function.
+*/
+
+typedef void (*glfs_io_cbk) (glfs_fd_t *fd, ssize_t ret, void *data);
+
+// glfs_{read,write}[_async]
+
+ssize_t glfs_read (glfs_fd_t *fd, void *buf, size_t count, int flags);
+ssize_t glfs_write (glfs_fd_t *fd, const void *buf, size_t count, int flags);
+int glfs_read_async (glfs_fd_t *fd, void *buf, size_t count, int flags,
+		     glfs_io_cbk fn, void *data);
+int glfs_write_async (glfs_fd_t *fd, const void *buf, size_t count, int flags,
+		      glfs_io_cbk fn, void *data);
+
+// glfs_{read,write}v[_async]
+
+ssize_t glfs_readv (glfs_fd_t *fd, const struct iovec *iov, int iovcnt,
+		    int flags);
+ssize_t glfs_writev (glfs_fd_t *fd, const struct iovec *iov, int iovcnt,
+		     int flags);
+int glfs_readv_async (glfs_fd_t *fd, const struct iovec *iov, int count,
+		      int flags, glfs_io_cbk fn, void *data);
+int glfs_writev_async (glfs_fd_t *fd, const struct iovec *iov, int count,
+		       int flags, glfs_io_cbk fn, void *data);
+
+// glfs_p{read,write}[_async]
+
+ssize_t glfs_pread (glfs_fd_t *fd, void *buf, size_t count, off_t offset,
+		    int flags);
+ssize_t glfs_pwrite (glfs_fd_t *fd, const void *buf, size_t count,
+		     off_t offset, int flags);
+int glfs_pread_async (glfs_fd_t *fd, void *buf, size_t count, off_t offset,
+		      int flags, glfs_io_cbk fn, void *data);
+int glfs_pwrite_async (glfs_fd_t *fd, const void *buf, int count, off_t offset,
+		       int flags, glfs_io_cbk fn, void *data);
+
+// glfs_p{read,write}v[_async]
+
+ssize_t glfs_preadv (glfs_fd_t *fd, const struct iovec *iov, int iovcnt,
+		     off_t offset, int flags);
+ssize_t glfs_pwritev (glfs_fd_t *fd, const struct iovec *iov, int iovcnt,
+		      off_t offset, int flags);
+int glfs_preadv_async (glfs_fd_t *fd, const struct iovec *iov, int count,
+		       off_t offset, int flags, glfs_io_cbk fn, void *data);
+int glfs_pwritev_async (glfs_fd_t *fd, const struct iovec *iov, int count,
+			off_t offset, int flags, glfs_io_cbk fn, void *data);
+
+
+off_t glfs_lseek (glfs_fd_t *fd, off_t offset, int whence);
+
+int glfs_truncate (glfs_t *fs, const char *path, off_t length);
+
+int glfs_ftruncate (glfs_fd_t *fd, off_t length);
+int glfs_ftruncate_async (glfs_fd_t *fd, off_t length, glfs_io_cbk fn,
+			  void *data);
+
+int glfs_lstat (glfs_t *fs, const char *path, struct stat *buf);
+int glfs_stat (glfs_t *fs, const char *path, struct stat *buf);
+int glfs_fstat (glfs_fd_t *fd, struct stat *buf);
+
+int glfs_fsync (glfs_fd_t *fd);
+int glfs_fsync_async (glfs_fd_t *fd, glfs_io_cbk fn, void *data);
+
+int glfs_fdatasync (glfs_fd_t *fd);
+int glfs_fdatasync_async (glfs_fd_t *fd, glfs_io_cbk fn, void *data);
+
+int glfs_access (glfs_t *fs, const char *path, int mode);
+
+int glfs_symlink (glfs_t *fs, const char *oldpath, const char *newpath);
+
+int glfs_readlink (glfs_t *fs, const char *path, char *buf, size_t bufsiz);
+
+int glfs_mknod (glfs_t *fs, const char *path, mode_t mode, dev_t dev);
+
+int glfs_mkdir (glfs_t *fs, const char *path, mode_t mode);
+
+int glfs_unlink (glfs_t *fs, const char *path);
+
+int glfs_rmdir (glfs_t *fs, const char *path);
+
+int glfs_rename (glfs_t *fs, const char *oldpath, const char *newpath);
+
+int glfs_link (glfs_t *fs, const char *oldpath, const char *newpath);
+
+glfs_fd_t *glfs_opendir (glfs_t *fs, const char *path);
+
+/*
+ * @glfs_readdir_r and @glfs_readdirplus_r ARE thread safe AND re-entrant,
+ * but the interface has ambiguity about the size of @dirent to be allocated
+ * before calling the APIs. 512 byte buffer (for @dirent) is sufficient for
+ * all known systems which are tested againt glusterfs/gfapi, but may be
+ * insufficient in the future.
+ */
+
+int glfs_readdir_r (glfs_fd_t *fd, struct dirent *dirent,
+		    struct dirent **result);
+
+int glfs_readdirplus_r (glfs_fd_t *fd, struct stat *stat, struct dirent *dirent,
+			struct dirent **result);
+
+/*
+ * @glfs_readdir and @glfs_readdirplus are NEITHER thread safe NOR re-entrant
+ * when called on the same directory handle. However they ARE thread safe
+ * AND re-entrant when called on different directory handles (which may be
+ * referring to the same directory too.)
+ */
+
+struct dirent *glfs_readdir (glfs_fd_t *fd);
+
+struct dirent *glfs_readdirplus (glfs_fd_t *fd, struct stat *stat);
+
+long glfs_telldir (glfs_fd_t *fd);
+
+void glfs_seekdir (glfs_fd_t *fd, long offset);
+
+int glfs_closedir (glfs_fd_t *fd);
+
+int glfs_statvfs (glfs_t *fs, const char *path, struct statvfs *buf);
+
+int glfs_chmod (glfs_t *fs, const char *path, mode_t mode);
+
+int glfs_fchmod (glfs_fd_t *fd, mode_t mode);
+
+int glfs_chown (glfs_t *fs, const char *path, uid_t uid, gid_t gid);
+
+int glfs_lchown (glfs_t *fs, const char *path, uid_t uid, gid_t gid);
+
+int glfs_fchown (glfs_fd_t *fd, uid_t uid, gid_t gid);
+
+int glfs_utimens (glfs_t *fs, const char *path, struct timespec times[2]);
+
+int glfs_lutimens (glfs_t *fs, const char *path, struct timespec times[2]);
+
+int glfs_futimens (glfs_fd_t *fd, struct timespec times[2]);
+
+ssize_t glfs_getxattr (glfs_t *fs, const char *path, const char *name,
+		       void *value, size_t size);
+
+ssize_t glfs_lgetxattr (glfs_t *fs, const char *path, const char *name,
+			void *value, size_t size);
+
+ssize_t glfs_fgetxattr (glfs_fd_t *fd, const char *name,
+			void *value, size_t size);
+
+ssize_t glfs_listxattr (glfs_t *fs, const char *path, void *value, size_t size);
+
+ssize_t glfs_llistxattr (glfs_t *fs, const char *path, void *value,
+			 size_t size);
+
+ssize_t glfs_flistxattr (glfs_fd_t *fd, void *value, size_t size);
+
+int glfs_setxattr (glfs_t *fs, const char *path, const char *name,
+		   const void *value, size_t size, int flags);
+
+int glfs_lsetxattr (glfs_t *fs, const char *path, const char *name,
+		    const void *value, size_t size, int flags);
+
+int glfs_fsetxattr (glfs_fd_t *fd, const char *name,
+		    const void *value, size_t size, int flags);
+
+int glfs_removexattr (glfs_t *fs, const char *path, const char *name);
+
+int glfs_lremovexattr (glfs_t *fs, const char *path, const char *name);
+
+int glfs_fremovexattr (glfs_fd_t *fd, const char *name);
+
+int glfs_fallocate(glfs_fd_t *fd, int keep_size, off_t offset, size_t len);
+
+int glfs_discard(glfs_fd_t *fd, off_t offset, size_t len);
+
+
+int glfs_discard_async (glfs_fd_t *fd, off_t length, size_t lent,
+			glfs_io_cbk fn, void *data);
+
+int glfs_zerofill(glfs_fd_t *fd, off_t offset, size_t len);
+
+int glfs_zerofill_async (glfs_fd_t *fd, off_t length, size_t len,
+                        glfs_io_cbk fn, void *data);
+
+char *glfs_getcwd (glfs_t *fs, char *buf, size_t size);
+
+int glfs_chdir (glfs_t *fs, const char *path);
+
+int glfs_fchdir (glfs_fd_t *fd);
+
+char *glfs_realpath (glfs_t *fs, const char *path, char *resolved_path);
+
+/*
+ * @cmd and @flock are as specified in man fcntl(2).
+ */
+int glfs_posix_lock (glfs_fd_t *fd, int cmd, struct flock *flock);
+
+glfs_fd_t *glfs_dup (glfs_fd_t *fd);
+
+__END_DECLS
+
+#endif /* !_GLFS_H */
diff --git a/argp-standalone/configure.ac b/argp-standalone/configure.ac
index 65ebc4518..2ecd2a801 100644
--- a/argp-standalone/configure.ac
+++ b/argp-standalone/configure.ac
@@ -8,7 +8,7 @@ AC_CONFIG_SRCDIR([argp-ba.c])
 AC_CONFIG_AUX_DIR([.])
 
 AM_INIT_AUTOMAKE
-AM_CONFIG_HEADER(config.h)
+AC_CONFIG_HEADERS(config.h)
 
 m4_ifdef([AM_SILENT_RULES], [AM_SILENT_RULES(yes)])
 
@@ -22,7 +22,7 @@ AC_GNU_SOURCE
 AC_PROG_CC
 AC_PROG_MAKE_SET
 AC_PROG_RANLIB
-AM_PROG_CC_STDC
+AC_PROG_CC
 
 if test "x$am_cv_prog_cc_stdc" = xno ; then
   AC_ERROR([the C compiler doesn't handle ANSI-C])
diff --git a/autogen.sh b/autogen.sh
index e20408bf2..f937e6be0 100755
--- a/autogen.sh
+++ b/autogen.sh
@@ -1,8 +1,105 @@
 #!/bin/sh
 
-aclocal
-autoheader
-(libtoolize --automake --copy --force || glibtoolize --automake --copy --force)
-autoconf
-automake --add-missing --copy --foreign
+echo
+echo ... GlusterFS autogen ...
+echo
+
+## Check all dependencies are present
+MISSING=""
+
+# Check for aclocal
+env aclocal --version > /dev/null 2>&1
+if [ $? -eq 0 ]; then
+  ACLOCAL=aclocal
+else
+  MISSING="$MISSING aclocal"
+fi
+
+# Check for autoconf
+env autoconf --version > /dev/null 2>&1
+if [ $? -eq 0 ]; then
+  AUTOCONF=autoconf
+else
+  MISSING="$MISSING autoconf"
+fi
+
+# Check for autoheader
+env autoheader --version > /dev/null 2>&1
+if [ $? -eq 0 ]; then
+  AUTOHEADER=autoheader
+else
+  MISSING="$MISSING autoheader"
+fi
+
+# Check for automake
+env automake --version > /dev/null 2>&1
+if [ $? -eq 0 ]; then
+  AUTOMAKE=automake
+else
+  MISSING="$MISSING automake"
+fi
+
+# Check for libtoolize or glibtoolize
+env libtoolize --version > /dev/null 2>&1
+if [ $? -eq 0 ]; then
+  # libtoolize was found, so use it
+  TOOL=libtoolize
+else
+  # libtoolize wasn't found, so check for glibtoolize
+  env glibtoolize --version > /dev/null 2>&1
+  if [ $? -eq 0 ]; then
+    TOOL=glibtoolize
+  else
+    MISSING="$MISSING libtoolize/glibtoolize"
+  fi
+fi
+
+# Check for tar
+env tar --version > /dev/null 2>&1
+if [ $? -ne 0 ]; then
+  MISSING="$MISSING tar"
+fi
+
+## If dependencies are missing, warn the user and abort
+if [ "x$MISSING" != "x" ]; then
+  echo "Aborting."
+  echo
+  echo "The following build tools are missing:"
+  echo
+  for pkg in $MISSING; do
+    echo "  * $pkg"
+  done
+  echo
+  echo "Please install them and try again."
+  echo
+  exit 1
+fi
+
+## generate gf-error-codes.h from error-codes.json
+echo "Generate gf-error-codes.h ..."
+if ./gen-headers.py; then
+    if ! mv -fv gf-error-codes.h libglusterfs/src/gf-error-codes.h; then
+	exit 1
+    fi
+else
+    exit 1
+fi
+
+## Do the autogeneration
+echo Running ${ACLOCAL}...
+$ACLOCAL -I ./contrib/aclocal
+echo Running ${AUTOHEADER}...
+$AUTOHEADER
+echo Running ${TOOL}...
+$TOOL --automake --copy --force
+echo Running ${AUTOCONF}...
+$AUTOCONF
+echo Running ${AUTOMAKE}...
+$AUTOMAKE --add-missing --copy --foreign
+
+# Run autogen in the argp-standalone sub-directory
 cd argp-standalone;./autogen.sh
+
+# Instruct user on next steps
+echo
+echo "Please proceed with configuring, compiling, and installing."
diff --git a/booster/Makefile.am b/booster/Makefile.am
deleted file mode 100644
index e1c45f305..000000000
--- a/booster/Makefile.am
+++ /dev/null
@@ -1 +0,0 @@
-SUBDIRS=src
\ No newline at end of file
diff --git a/booster/src/Makefile.am b/booster/src/Makefile.am
deleted file mode 100644
index d7d83abf5..000000000
--- a/booster/src/Makefile.am
+++ /dev/null
@@ -1,21 +0,0 @@
-ldpreload_LTLIBRARIES = libglusterfs-booster.la
-ldpreloaddir = $(libdir)/glusterfs
-noinst_HEADERS = booster_fstab.h booster-fd.h
-libglusterfs_booster_la_SOURCES = booster.c booster_stat.c booster_fstab.c booster-fd.c
-libglusterfs_booster_la_CFLAGS = -I$(top_srcdir)/libglusterfsclient/src/ -D_GNU_SOURCE -D$(GF_HOST_OS) -fPIC -Wall \
-	-pthread $(GF_BOOSTER_CFLAGS) -shared -nostartfiles 
-libglusterfs_booster_la_CPPFLAGS = -D_FILE_OFFSET_BITS=64 -D_GNU_SOURCE  \
-	-I$(top_srcdir)/libglusterfsclient/src \
-	-I$(top_srcdir)/libglusterfs/src -DDATADIR=\"$(localstatedir)\" \
-	-DCONFDIR=\"$(sysconfdir)/glusterfs\" $(ARGP_STANDALONE_CPPFLAGS)
-
-libglusterfs_booster_la_LDFLAGS = -module -avoidversion
-libglusterfs_booster_la_LIBADD =  $(top_builddir)/libglusterfs/src/libglusterfs.la $(top_builddir)/libglusterfsclient/src/libglusterfsclient.la
-
-CLEANFILES =
-
-uninstall-local:
-	rm -f $(DESTDIR)$(ldpreloaddir)/glusterfs-booster.so
-
-install-data-hook:
-	ln -sf libglusterfs-booster.so $(DESTDIR)$(ldpreloaddir)/glusterfs-booster.so
diff --git a/booster/src/booster-fd.c b/booster/src/booster-fd.c
deleted file mode 100644
index fa5b0cde2..000000000
--- a/booster/src/booster-fd.c
+++ /dev/null
@@ -1,342 +0,0 @@
-/*
-  Copyright (c) 2010 Gluster, Inc. <http://www.gluster.com>
-  This file is part of GlusterFS.
-
-  GlusterFS is free software; you can redistribute it and/or modify
-  it under the terms of the GNU General Public License as published
-  by the Free Software Foundation; either version 3 of the License,
-  or (at your option) any later version.
-
-  GlusterFS is distributed in the hope that it will be useful, but
-  WITHOUT ANY WARRANTY; without even the implied warranty of
-  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-  General Public License for more details.
-
-  You should have received a copy of the GNU General Public License
-  along with this program.  If not, see
-  <http://www.gnu.org/licenses/>.
-*/
-
-
-
-#include "booster-fd.h"
-#include <logging.h>
-#include <mem-pool.h>
-#include <stdlib.h>
-#include <errno.h>
-#include <common-utils.h>
-#include <string.h>
-
-#include <assert.h>
-
-extern fd_t *
-fd_ref (fd_t *fd);
-
-extern void
-fd_unref (fd_t *fd);
-/*
-   Allocate in memory chunks of power of 2 starting from 1024B
-   Assumes fdtable->lock is held
-   */
-static inline uint
-gf_roundup_power_of_two (uint nr)
-{
-        uint result = 1;
-
-        if (nr < 0) {
-                gf_log ("booster-fd", GF_LOG_ERROR, "Negative number passed");
-                return -1;
-        }
-        
-        while (result <= nr)
-                result *= 2;
-
-        return result;
-}
-
-#define BOOSTER_NFDBITS (sizeof (unsigned long))
-
-#define BOOSTER_FDMASK(d)             (1UL << ((d) % BOOSTER_NFDBITS))
-#define BOOSTER_FDELT(d)              (d / BOOSTER_NFDBITS)
-#define BOOSTER_FD_SET(set, d)        (set->fd_bits[BOOSTER_FDELT(d)] |= BOOSTER_FDMASK(d))
-#define BOOSTER_FD_CLR(set, d)        (set->fd_bits[BOOSTER_FDELT(d)] &= ~BOOSTER_FDMASK(d))
-#define BOOSTER_FD_ISSET(set, d)      (set->fd_bits[BOOSTER_FDELT(d)] & BOOSTER_FDMASK(d))
-
-inline int
-booster_get_close_on_exec (booster_fdtable_t *fdtable, int fd)
-{
-        return BOOSTER_FD_ISSET(fdtable->close_on_exec, fd);
-}
-
-inline void
-booster_set_close_on_exec (booster_fdtable_t *fdtable, int fd)
-{
-        BOOSTER_FD_SET(fdtable->close_on_exec, fd);
-}
-
-int
-booster_fdtable_expand (booster_fdtable_t *fdtable, uint nr)
-{
-        fd_t    **oldfds = NULL, **tmp = NULL;
-        uint    oldmax_fds = -1;
-        uint    cpy = 0;
-        int32_t ret = -1, bytes = 0;
-        booster_fd_set_t *oldclose_on_exec = NULL;
-
-        if (fdtable == NULL || nr < 0) {
-                gf_log ("booster-fd", GF_LOG_ERROR, "Invalid argument");
-                errno = EINVAL;
-                ret = -1;
-                goto out;
-        }
-
-        nr /= (1024 / sizeof (fd_t *));
-        nr = gf_roundup_power_of_two (nr + 1);
-        nr *= (1024 / sizeof (fd_t *));
-
-        oldfds = fdtable->fds;
-        oldmax_fds = fdtable->max_fds;
-        oldclose_on_exec = fdtable->close_on_exec;
-
-        fdtable->fds = CALLOC (nr, sizeof (fd_t *));
-        if (fdtable->fds == NULL) {
-                gf_log ("booster-fd", GF_LOG_ERROR, "Memory allocation failed");
-                fdtable->fds = oldfds;
-                oldfds = NULL;
-                ret = -1;
-                goto out;
-        }
-
-        fdtable->max_fds = nr;
-
-        if (oldfds) {
-                cpy = oldmax_fds * sizeof (fd_t *);
-                memcpy (fdtable->fds, oldfds, cpy);
-        }
-
-        /* nr will be either less than 8 or a multiple of 8 */
-        bytes = nr/8;
-        bytes = bytes ? bytes : 1;
-        fdtable->close_on_exec = CALLOC (bytes, 1);
-        if (fdtable->close_on_exec == NULL) {
-                gf_log ("booster-fd", GF_LOG_ERROR, "Memory allocation "
-                        "failed");
-                tmp = fdtable->fds;
-                fdtable->fds = oldfds;
-                oldfds = tmp;
-                ret = -1;
-                goto out;
-        }
-
-        if (oldclose_on_exec != NULL) {
-                bytes = oldmax_fds/8;
-                cpy = bytes ? bytes : 1;
-                memcpy (fdtable->close_on_exec, oldclose_on_exec, cpy);
-        }
-        gf_log ("booster-fd", GF_LOG_TRACE, "FD-table expanded: Old: %d,New: %d"
-                , oldmax_fds, nr);
-        ret = 0;
-
-out:
-        FREE (oldfds);
-        FREE (oldclose_on_exec);
-
-        return ret;
-}
-
-booster_fdtable_t *
-booster_fdtable_alloc (void)
-{
-        booster_fdtable_t *fdtable = NULL;
-        int32_t            ret = -1;
-
-        fdtable = CALLOC (1, sizeof (*fdtable));
-        GF_VALIDATE_OR_GOTO ("booster-fd", fdtable, out);
-
-        LOCK_INIT (&fdtable->lock);
-
-        LOCK (&fdtable->lock);
-        {
-                ret = booster_fdtable_expand (fdtable, 0);
-        }
-        UNLOCK (&fdtable->lock);
-
-        if (ret == -1) {
-                gf_log ("booster-fd", GF_LOG_ERROR, "FD-table allocation "
-                        "failed");
-                FREE (fdtable);
-                fdtable = NULL;
-        }
-
-out:
-        return fdtable;
-}
-
-fd_t **
-__booster_fdtable_get_all_fds (booster_fdtable_t *fdtable, uint *count)
-{
-        fd_t **fds = NULL;
-
-        if (count == NULL)
-                goto out;
-
-        fds = fdtable->fds;
-        fdtable->fds = calloc (fdtable->max_fds, sizeof (fd_t *));
-        *count = fdtable->max_fds;
-
-out:
-        return fds;
-}
-
-fd_t **
-booster_fdtable_get_all_fds (booster_fdtable_t *fdtable, uint *count)
-{
-        fd_t **fds = NULL;
-        if (!fdtable)
-                return NULL;
-
-        LOCK (&fdtable->lock);
-        {
-                fds = __booster_fdtable_get_all_fds (fdtable, count);
-        }
-        UNLOCK (&fdtable->lock);
-
-        return fds;
-}
-
-void
-booster_fdtable_destroy (booster_fdtable_t *fdtable)
-{
-        fd_t                    *fd = NULL;
-        fd_t                    **fds = NULL;
-        uint                    fd_count = 0;
-        int                     i = 0;
-
-        if (!fdtable)
-                return;
-
-        LOCK (&fdtable->lock);
-        {
-                fds = __booster_fdtable_get_all_fds (fdtable, &fd_count);
-                FREE (fdtable->fds);
-        }
-        UNLOCK (&fdtable->lock);
-
-        if (!fds)
-                goto free_table;
-        
-        for (i = 0; i < fd_count; i++) {
-                fd = fds[i];
-                if (fd != NULL)
-                        fd_unref (fd);
-        }
-        FREE (fds);
-free_table:
-        LOCK_DESTROY (&fdtable->lock);
-        FREE (fdtable);
-}
-
-int
-booster_fd_unused_get (booster_fdtable_t *fdtable, fd_t *fdptr, int fd)
-{
-        int ret = -1;
-        int error = 0;
-
-        if (fdtable == NULL || fdptr == NULL || fd < 0) {
-                gf_log ("booster-fd", GF_LOG_ERROR, "invalid argument");
-                errno = EINVAL;
-                return -1;
-        }
-
-        gf_log ("booster-fd", GF_LOG_TRACE, "Requested fd: %d", fd);
-        LOCK (&fdtable->lock);
-        {
-                while (fdtable->max_fds < fd) {
-                        error = 0;
-                        error = booster_fdtable_expand (fdtable,
-                                                        fdtable->max_fds + 1);
-                        if (error) {
-                                gf_log ("booster-fd", GF_LOG_ERROR,
-                                        "Cannot expand fdtable:%s",
-                                        strerror (error));
-                                goto err;
-                        }
-                }
-
-                if (!fdtable->fds[fd]) {
-                        fdtable->fds[fd] = fdptr;
-                        fd_ref (fdptr);
-                        ret = fd;
-                } else
-                        gf_log ("booster-fd", GF_LOG_ERROR, "Cannot allocate fd"
-                                " %d (slot not empty in fdtable)", fd);
-        }
-err:
-        UNLOCK (&fdtable->lock);
-
-        return ret;
-}
-
-void
-booster_fd_put (booster_fdtable_t *fdtable, int fd)
-{
-        fd_t *fdptr = NULL;
-        if (fdtable == NULL || fd < 0) {
-                gf_log ("booster-fd", GF_LOG_ERROR, "invalid argument");
-                return;
-        }
-
-        gf_log ("booster-fd", GF_LOG_TRACE, "FD put: %d", fd);
-        if (!(fd < fdtable->max_fds)) {
-                gf_log ("booster-fd", GF_LOG_ERROR, "FD not in booster fd"
-                        " table");
-                return;
-        }
-
-        LOCK (&fdtable->lock);
-        {
-                fdptr = fdtable->fds[fd];
-                fdtable->fds[fd] = NULL;
-        }
-        UNLOCK (&fdtable->lock);
-
-        if (fdptr)
-                fd_unref (fdptr);
-}
-
-fd_t *
-booster_fdptr_get (booster_fdtable_t *fdtable, int fd)
-{
-        fd_t *fdptr = NULL;
-
-        if (fdtable == NULL || fd < 0) {
-                gf_log ("booster-fd", GF_LOG_ERROR, "invalid argument");
-                errno = EINVAL;
-                return NULL;
-        }
-
-        gf_log ("booster-fd", GF_LOG_TRACE, "FD ptr request: %d", fd);
-        if (!(fd < fdtable->max_fds)) {
-                gf_log ("booster-fd", GF_LOG_ERROR, "FD not in booster fd"
-                        " table");
-                errno = EINVAL;
-                return NULL;
-        }
-
-        LOCK (&fdtable->lock);
-        {
-                fdptr = fdtable->fds[fd];
-                if (fdptr)
-                        fd_ref (fdptr);
-        }
-        UNLOCK (&fdtable->lock);
-
-        return fdptr;
-}
-
-void
-booster_fdptr_put (fd_t *booster_fd)
-{
-        if (booster_fd)
-                fd_unref (booster_fd);
-}
diff --git a/booster/src/booster-fd.h b/booster/src/booster-fd.h
deleted file mode 100644
index 595a112bd..000000000
--- a/booster/src/booster-fd.h
+++ /dev/null
@@ -1,83 +0,0 @@
-/*
-  Copyright (c) 2010 Gluster, Inc. <http://www.gluster.com>
-  This file is part of GlusterFS.
-
-  GlusterFS is free software; you can redistribute it and/or modify
-  it under the terms of the GNU General Public License as published
-  by the Free Software Foundation; either version 3 of the License,
-  or (at your option) any later version.
-
-  GlusterFS is distributed in the hope that it will be useful, but
-  WITHOUT ANY WARRANTY; without even the implied warranty of
-  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-  General Public License for more details.
-
-  You should have received a copy of the GNU General Public License
-  along with this program.  If not, see
-  <http://www.gnu.org/licenses/>.
-*/
-
-#ifndef _BOOSTER_FD_H
-#define _BOOSTER_FD_H
-
-#include <libglusterfsclient.h>
-#include <locking.h>
-#include <list.h>
-
-/* This struct must be updated if the fd_t in fd.h changes.
- * We cannot include those headers here because unistd.h, included
- * by glusterfs headers, conflicts with the syscall prototypes we
- * define for booster.
- */
-struct _fd {
-        pid_t             pid;
-	int32_t           flags;
-        int32_t           refcount;
-        struct list_head  inode_list;
-        struct _inode    *inode;
-        struct _dict     *ctx;
-        gf_lock_t         lock; /* used ONLY for manipulating
-                                   'struct _fd_ctx' array (_ctx).*/
-	struct _fd_ctx   *_ctx;
-};
-typedef struct _fd fd_t;
-
-struct _booster_fd_set {
-        unsigned long fd_bits[0];
-};
-typedef struct _booster_fd_set booster_fd_set_t;
-
-struct _booster_fdtable {
-        booster_fd_set_t *close_on_exec;
-        int               refcount;
-        unsigned int      max_fds;
-        gf_lock_t         lock;
-        fd_t            **fds;
-};
-typedef struct _booster_fdtable booster_fdtable_t;
-
-void
-booster_set_close_on_exec (booster_fdtable_t *fdtable, int fd);
-
-int
-booster_get_close_on_exec (booster_fdtable_t *fdtable, int fd);
-
-extern int
-booster_fd_unused_get (booster_fdtable_t *fdtable, fd_t *fdptr, int fd);
-
-extern void
-booster_fd_put (booster_fdtable_t *fdtable, int fd);
-
-extern fd_t *
-booster_fdptr_get (booster_fdtable_t *fdtable, int fd);
-
-extern void
-booster_fdptr_put (fd_t *fdptr);
-
-extern void
-booster_fdtable_destroy (booster_fdtable_t *fdtable);
-
-booster_fdtable_t *
-booster_fdtable_alloc (void);
-
-#endif /* #ifndef _BOOSTER_FD_H */
diff --git a/booster/src/booster.c b/booster/src/booster.c
deleted file mode 100644
index c34ec1146..000000000
--- a/booster/src/booster.c
+++ /dev/null
@@ -1,3172 +0,0 @@
-/*
-   Copyright (c) 2007-2011 Gluster, Inc. <http://www.gluster.com>
-   This file is part of GlusterFS.
-
-   GlusterFS is free software; you can redistribute it and/or modify
-   it under the terms of the GNU General Public License as published
-   by the Free Software Foundation; either version 3 of the License,
-   or (at your option) any later version.
-
-   GlusterFS is distributed in the hope that it will be useful, but
-   WITHOUT ANY WARRANTY; without even the implied warranty of
-   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-   General Public License for more details.
-
-   You should have received a copy of the GNU General Public License
-   along with this program.  If not, see
-   <http://www.gnu.org/licenses/>.
-*/
-
-#ifndef _CONFIG_H
-#define _CONFIG_H
-#include "config.h"
-#endif
-
-#include <dlfcn.h>
-#include <sys/types.h>
-#include <sys/uio.h>
-#include <stdio.h>
-#include <stdarg.h>
-#include <stdlib.h>
-#include <inttypes.h>
-#include <libglusterfsclient.h>
-#include <list.h>
-#include <pthread.h>
-#include <sys/xattr.h>
-#include <string.h>
-#include <assert.h>
-#include <errno.h>
-#include <ctype.h>
-#include <logging.h>
-#include <utime.h>
-#include <dirent.h>
-#include <sys/statfs.h>
-#include <sys/statvfs.h>
-#include <fcntl.h>
-#include "booster-fd.h"
-
-#ifndef GF_UNIT_KB
-#define GF_UNIT_KB 1024
-#endif
-
-static pthread_mutex_t cwdlock   = PTHREAD_MUTEX_INITIALIZER;
-
-/* attr constructor registers this function with libc's
- * _init function as a function that must be called before
- * the main() of the program.
- */
-static void booster_lib_init (void) __attribute__((constructor));
-
-extern fd_t *
-fd_ref (fd_t *fd);
-
-extern void
-fd_unref (fd_t *fd);
-
-extern int pipe (int filedes[2]);
-/* We define these flags so that we can remove fcntl.h from the include path.
- * fcntl.h has certain defines and other lines of code that redirect the
- * application's open and open64 calls to the syscalls defined by
- * libc, for us, thats not a Good Thing (TM).
- */
-#ifndef GF_O_CREAT
-#define GF_O_CREAT      0x40
-#endif
-
-#ifndef GF_O_TRUNC
-#define GF_O_TRUNC      0x200
-#endif
-
-#ifndef GF_O_RDWR
-#define GF_O_RDWR       0x2
-#endif
-
-#ifndef GF_O_WRONLY
-#define GF_O_WRONLY     0x1
-#endif
-
-#ifndef UNIX_PATH_MAX
-#define UNIX_PATH_MAX 108
-#endif
-
-typedef enum {
-        BOOSTER_OPEN,
-        BOOSTER_CREAT
-} booster_op_t;
-
-struct _inode;
-struct _dict;
-
-ssize_t
-write (int fd, const void *buf, size_t count);
-
-/* open, open64, creat */
-static int (*real_open) (const char *pathname, int flags, ...);
-static int (*real_open64) (const char *pathname, int flags, ...);
-static int (*real_creat) (const char *pathname, mode_t mode);
-static int (*real_creat64) (const char *pathname, mode_t mode);
-
-/* read, readv, pread, pread64 */
-static ssize_t (*real_read) (int fd, void *buf, size_t count);
-static ssize_t (*real_readv) (int fd, const struct iovec *vector, int count);
-static ssize_t (*real_pread) (int fd, void *buf, size_t count,
-                              unsigned long offset);
-static ssize_t (*real_pread64) (int fd, void *buf, size_t count,
-                                uint64_t offset);
-
-/* write, writev, pwrite, pwrite64 */
-static ssize_t (*real_write) (int fd, const void *buf, size_t count);
-static ssize_t (*real_writev) (int fd, const struct iovec *vector, int count);
-static ssize_t (*real_pwrite) (int fd, const void *buf, size_t count,
-                               unsigned long offset);
-static ssize_t (*real_pwrite64) (int fd, const void *buf, size_t count,
-                                 uint64_t offset);
-
-/* lseek, llseek, lseek64 */
-static off_t (*real_lseek) (int fildes, unsigned long offset, int whence);
-static off_t (*real_lseek64) (int fildes, uint64_t offset, int whence);
-
-/* close */
-static int (*real_close) (int fd);
-
-/* dup dup2 */
-static int (*real_dup) (int fd);
-static int (*real_dup2) (int oldfd, int newfd);
-
-static pid_t (*real_fork) (void);
-static int (*real_mkdir) (const char *pathname, mode_t mode);
-static int (*real_rmdir) (const char *pathname);
-static int (*real_chmod) (const char *pathname, mode_t mode);
-static int (*real_chown) (const char *pathname, uid_t owner, gid_t group);
-static int (*real_fchmod) (int fd, mode_t mode);
-static int (*real_fchown) (int fd, uid_t, gid_t gid);
-static int (*real_fsync) (int fd);
-static int (*real_ftruncate) (int fd, off_t length);
-static int (*real_ftruncate64) (int fd, loff_t length);
-static int (*real_link) (const char *oldpath, const char *newname);
-static int (*real_rename) (const char *oldpath, const char *newpath);
-static int (*real_utimes) (const char *path, const struct timeval times[2]);
-static int (*real_utime) (const char *path, const struct utimbuf *buf);
-static int (*real_mknod) (const char *path, mode_t mode, dev_t dev);
-static int (*real_mkfifo) (const char *path, mode_t mode);
-static int (*real_unlink) (const char *path);
-static int (*real_symlink) (const char *oldpath, const char *newpath);
-static int (*real_readlink) (const char *path, char *buf, size_t bufsize);
-static char * (*real_realpath) (const char *path, char *resolved);
-static DIR * (*real_opendir) (const char *path);
-static struct dirent * (*real_readdir) (DIR *dir);
-static struct dirent64 * (*real_readdir64) (DIR *dir);
-static int (*real_readdir_r) (DIR *dir, struct dirent *entry,
-                              struct dirent **result);
-static int (*real_readdir64_r) (DIR *dir, struct dirent64 *entry,
-                                struct dirent64 **result);
-static int (*real_closedir) (DIR *dh);
-static int (*real___xstat) (int ver, const char *path, struct stat *buf);
-static int (*real___xstat64) (int ver, const char *path, struct stat64 *buf);
-static int (*real_stat) (const char *path, struct stat *buf);
-static int (*real_stat64) (const char *path, struct stat64 *buf);
-static int (*real___fxstat) (int ver, int fd, struct stat *buf);
-static int (*real___fxstat64) (int ver, int fd, struct stat64 *buf);
-static int (*real_fstat) (int fd, struct stat *buf);
-static int (*real_fstat64) (int fd , struct stat64 *buf);
-static int (*real___lxstat) (int ver, const char *path, struct stat *buf);
-static int (*real___lxstat64) (int ver, const char *path, struct stat64 *buf);
-static int (*real_lstat) (const char *path, struct stat *buf);
-static int (*real_lstat64) (const char *path, struct stat64 *buf);
-static int (*real_statfs) (const char *path, struct statfs *buf);
-static int (*real_statfs64) (const char *path, struct statfs64 *buf);
-static int (*real_statvfs) (const char *path, struct statvfs *buf);
-static int (*real_statvfs64) (const char *path, struct statvfs64 *buf);
-static ssize_t (*real_getxattr) (const char *path, const char *name,
-                                 void *value, size_t size);
-static ssize_t (*real_lgetxattr) (const char *path, const char *name,
-                                  void *value, size_t size);
-static int (*real_remove) (const char* path);
-static int (*real_lchown) (const char *path, uid_t owner, gid_t group);
-static void (*real_rewinddir) (DIR *dirp);
-static void (*real_seekdir) (DIR *dirp, off_t offset);
-static off_t (*real_telldir) (DIR *dirp);
-
-static ssize_t (*real_sendfile) (int out_fd, int in_fd, off_t *offset,
-                                 size_t count);
-static ssize_t (*real_sendfile64) (int out_fd, int in_fd, off_t *offset,
-                                   size_t count);
-static int (*real_fcntl) (int fd, int cmd, ...);
-static int (*real_chdir) (const char *path);
-static int (*real_fchdir) (int fd);  
-static char * (*real_getcwd) (char *buf, size_t size);
-static int (*real_truncate) (const char *path, off_t length);
-static int (*real_truncate64) (const char *path, loff_t length);
-static int (*real_setxattr) (const char *path, const char *name,
-                             const void *value, size_t size, int flags);
-static int (*real_lsetxattr) (const char *path, const char *name,
-                              const void *value, size_t size, int flags);
-static int (*real_fsetxattr) (int filedes, const char *name,
-                              const void *value, size_t size, int flags);
-
-
-#define RESOLVE(sym) do {                                       \
-                if (!real_##sym)                                \
-                        real_##sym = dlsym (RTLD_NEXT, #sym);   \
-        } while (0)
-
-/*TODO: set proper value */
-#define MOUNT_HASH_SIZE 256
-
-struct booster_mount {
-        dev_t st_dev;
-        glusterfs_handle_t handle;
-        struct list_head device_list;
-};
-typedef struct booster_mount booster_mount_t;
-
-static booster_fdtable_t *booster_fdtable = NULL;
-
-extern int booster_configure (char *confpath);
-/* This is dup'ed every time VMP open/creat wants a new fd.
- * This is needed so we occupy an entry in the process' file
- * table.
- */
-int process_piped_fd = -1;
-
-static int
-booster_get_process_fd ()
-{
-        return real_dup (process_piped_fd);
-}
-
-/* The following two define which file contains
- * the FSTAB configuration for VMP-based usage.
- */
-#define DEFAULT_BOOSTER_CONF    CONFDIR"/booster.conf"
-#define BOOSTER_CONF_ENV_VAR    "GLUSTERFS_BOOSTER_FSTAB"
-
-
-/* The following define which log file is used when
- * using the old mount point bypass approach.
- */
-#define BOOSTER_DEFAULT_LOG     CONFDIR"/booster.log"
-#define BOOSTER_LOG_ENV_VAR     "GLUSTERFS_BOOSTER_LOG"
-
-void
-do_open (int fd, const char *pathname, int flags, mode_t mode, booster_op_t op)
-{
-        char                   *specfile = NULL;
-        char                   *mount_point = NULL; 
-        int32_t                 size = 0;
-        int32_t                 ret = -1;
-        FILE                   *specfp = NULL;
-        glusterfs_file_t        fh = NULL;
-        char                   *logfile = NULL;
-        glusterfs_init_params_t iparams = {
-                .loglevel = "error",
-                .lookup_timeout = 600,
-                .stat_timeout = 600,
-        };
-      
-        gf_log ("booster", GF_LOG_DEBUG, "Opening using MPB: %s", pathname);
-        size = fgetxattr (fd, "user.glusterfs-booster-volfile", NULL, 0);
-        if (size == -1) {
-                gf_log ("booster", GF_LOG_ERROR, "Xattr "
-                        "user.glusterfs-booster-volfile not found: %s",
-                        strerror (errno));
-                goto out;
-        }
-		
-        specfile = calloc (1, size);
-        if (!specfile) {
-                gf_log ("booster", GF_LOG_ERROR, "Memory allocation failed");
-                goto out;
-        }
-
-        ret = fgetxattr (fd, "user.glusterfs-booster-volfile", specfile,
-                         size);
-        if (ret == -1) {
-                gf_log ("booster", GF_LOG_ERROR, "Xattr "
-                        "user.glusterfs-booster-volfile not found: %s",
-                        strerror (errno));
-                goto out;
-        }
-    
-        specfp = tmpfile ();
-        if (!specfp) {
-                gf_log ("booster", GF_LOG_ERROR, "Temp file creation failed"
-                        ": %s", strerror (errno));
-                goto out;
-        }
-
-        ret = fwrite (specfile, size, 1, specfp);
-        if (ret != 1) {
-                gf_log ("booster", GF_LOG_ERROR, "Failed to write volfile: %s",
-                        strerror (errno));
-                goto out;
-        }
-		
-        fseek (specfp, 0L, SEEK_SET);
-
-        size = fgetxattr (fd, "user.glusterfs-booster-mount", NULL, 0);
-        if (size == -1) {
-                gf_log ("booster", GF_LOG_ERROR, "Xattr "
-                        "user.glusterfs-booster-mount not found: %s",
-                        strerror (errno));
-                goto out;
-        }
-        
-        mount_point = calloc (size, sizeof (char));
-        if (!mount_point) {
-                gf_log ("booster", GF_LOG_ERROR, "Memory allocation failed");
-                goto out;
-        }
-	
-        ret = fgetxattr (fd, "user.glusterfs-booster-mount", mount_point, size);
-        if (ret == -1) {
-                gf_log ("booster", GF_LOG_ERROR, "Xattr "
-                        "user.glusterfs-booster-mount not found: %s",
-                        strerror (errno));
-                goto out;
-        }
-
-        logfile = getenv (BOOSTER_LOG_ENV_VAR);
-        if (logfile) {
-                if (strlen (logfile) > 0)
-                        iparams.logfile = strdup (logfile);
-                else
-                        iparams.logfile = strdup (BOOSTER_DEFAULT_LOG);
-        } else {
-                iparams.logfile = strdup (BOOSTER_DEFAULT_LOG);
-        }
-
-        gf_log ("booster", GF_LOG_TRACE, "Using log-file: %s", iparams.logfile);
-        iparams.specfp = specfp;
-
-        ret = glusterfs_mount (mount_point, &iparams);
-        if (ret == -1) {
-                if (errno != EEXIST) {
-                        gf_log ("booster", GF_LOG_ERROR, "Mount failed over"
-                                " glusterfs");
-                        goto out;
-                } else
-                        gf_log ("booster", GF_LOG_ERROR, "Already mounted");
-        }
-
-        switch (op) {
-        case BOOSTER_OPEN:
-                gf_log ("booster", GF_LOG_TRACE, "Booster open call");
-                fh = glusterfs_open (pathname, flags, mode);
-                break;
-
-        case BOOSTER_CREAT:
-                gf_log ("booster", GF_LOG_TRACE, "Booster create call");
-                fh = glusterfs_creat (pathname, mode);
-                break;
-        }
-
-        if (!fh) {
-                gf_log ("booster", GF_LOG_ERROR, "Error performing operation");
-                goto out;
-        }
-
-        if (booster_fd_unused_get (booster_fdtable, fh, fd) == -1) {
-                gf_log ("booster", GF_LOG_ERROR, "Failed to get unused FD");
-                goto out;
-        }
-        fh = NULL;
-
-out:
-        if (specfile) {
-                free (specfile);
-        }
-
-        if (specfp) {
-                fclose (specfp);
-        }
-
-        if (mount_point) {
-                free (mount_point);
-        }
-
-        if (fh) {
-                glusterfs_close (fh);
-        }
-
-        return;
-}
-
-int
-vmp_open (const char *pathname, int flags, ...)
-{
-        mode_t                  mode = 0;
-        int                     fd = -1;
-        glusterfs_file_t        fh = NULL;
-        va_list                 ap;
-
-        if (flags & GF_O_CREAT) {
-                va_start (ap, flags);
-                mode = va_arg (ap, mode_t);
-                va_end (ap);
-
-                fh = glusterfs_open (pathname, flags, mode);
-        }
-        else
-                fh = glusterfs_open (pathname, flags);
-
-        if (!fh) {
-                gf_log ("booster", GF_LOG_ERROR, "VMP open failed");
-                goto out;
-        }
-
-        fd = booster_get_process_fd ();
-        if (fd == -1) {
-                gf_log ("booster", GF_LOG_ERROR, "Failed to create open fd");
-                goto fh_close_out;
-        }
-
-        if (booster_fd_unused_get (booster_fdtable, fh, fd) == -1) {
-                gf_log ("booster", GF_LOG_ERROR, "Failed to map fd into table");
-                goto realfd_close_out;
-        }
-
-        return fd;
-
-realfd_close_out:
-        real_close (fd);
-        fd = -1;
-
-fh_close_out:
-        glusterfs_close (fh);
-
-out:
-        return fd;
-}
-
-#define BOOSTER_USE_OPEN64          1
-#define BOOSTER_DONT_USE_OPEN64     0
-
-int
-booster_open (const char *pathname, int use64, int flags, ...)
-{
-        int     ret = -1;
-        mode_t  mode = 0;
-        va_list ap;
-        int     (*my_open) (const char *pathname, int flags, ...);
-
-        if (!pathname) {
-                errno = EINVAL;
-                goto out;
-        }
-
-        gf_log ("booster", GF_LOG_TRACE, "Open: %s", pathname);
-        /* First try opening through the virtual mount point.
-         * The difference lies in the fact that:
-         * 1. We depend on libglusterfsclient library to perform
-         * the translation from the path to handle.
-         * 2. We do not go to the file system for the fd, instead
-         * we use booster_get_process_fd (), which returns a dup'ed
-         * fd of a pipe created in booster_init.
-         */
-        if (flags & GF_O_CREAT) {
-                va_start (ap, flags);
-                mode = va_arg (ap, mode_t);
-                va_end (ap);
-                ret = vmp_open (pathname, flags, mode);
-        }
-        else
-                ret = vmp_open (pathname, flags);
-
-        /* We receive an ENODEV if the VMP does not exist. If we
-         * receive an error other than ENODEV, it means, there
-         * actually was an error performing vmp_open. This must
-         * be returned to the user.
-         */
-        if ((ret < 0) && (errno != ENODEV)) {
-                gf_log ("booster", GF_LOG_ERROR, "Error in opening file over "
-                        " VMP: %s", strerror (errno));
-                goto out;
-        }
-
-        if (ret > 0) {
-                gf_log ("booster", GF_LOG_TRACE, "File opened");
-                goto out;
-        }
-
-        if (use64) {
-                gf_log ("booster", GF_LOG_TRACE, "Using 64-bit open");
-		my_open = real_open64;
-        } else {
-                gf_log ("booster", GF_LOG_TRACE, "Using 32-bit open");
-		my_open = real_open;
-        }
-
-        /* It is possible the RESOLVE macro is not able
-         * to resolve the symbol of a function, in that case
-         * we dont want to seg-fault on calling a NULL functor.
-         */
-        if (my_open == NULL) {
-                gf_log ("booster", GF_LOG_ERROR, "open not resolved");
-                ret = -1;
-                errno = ENOSYS;
-                goto out;
-        }
-
-	if (flags & GF_O_CREAT) {
-		va_start (ap, flags);
-		mode = va_arg (ap, mode_t);
-		va_end (ap);
-
-                ret = my_open (pathname, flags, mode);
-	} else
-                ret = my_open (pathname, flags);
-
-        if (ret != -1) {
-                do_open (ret, pathname, flags, mode, BOOSTER_OPEN);
-        }
-
-out:
-        return ret;
-}
-
-/* This is done to over-write existing definitions of open and open64 inside
- * libc with our own copies. __REDIRECT is provided by libc.
- *
- * XXX: This will not work anywhere other than libc based systems.
- */
-int __REDIRECT (booster_false_open, (__const char *__file, int __oflag, ...),
-                open) __nonnull ((1));
-int __REDIRECT (booster_false_open64, (__const char *__file, int __oflag, ...),
-                open64) __nonnull ((1));
-int
-booster_false_open (const char *pathname, int flags, ...)
-{
-        int     ret;
-        mode_t  mode = 0;
-        va_list ap;
-
-        if (flags & GF_O_CREAT) {
-                va_start (ap, flags);
-                mode = va_arg (ap, mode_t);
-                va_end (ap);
-
-                ret = booster_open (pathname, BOOSTER_DONT_USE_OPEN64, flags,
-                                    mode);
-        }
-        else
-                ret = booster_open (pathname, BOOSTER_DONT_USE_OPEN64, flags);
-
-        return ret;
-}
-
-int
-booster_false_open64 (const char *pathname, int flags, ...)
-{
-        int     ret;
-        mode_t  mode = 0;
-        va_list ap;
-
-        if (flags & GF_O_CREAT) {
-                va_start (ap, flags);
-                mode = va_arg (ap, mode_t);
-                va_end (ap);
-
-                ret = booster_open (pathname, BOOSTER_USE_OPEN64, flags, mode);
-        }
-        else
-                ret = booster_open (pathname, BOOSTER_USE_OPEN64, flags);
-
-        return ret;
-}
-
-int
-vmp_creat (const char *pathname, mode_t mode)
-{
-        int                     fd = -1;
-        glusterfs_file_t        fh = NULL;
-
-        fh = glusterfs_creat (pathname, mode);
-        if (!fh) {
-                gf_log ("booster", GF_LOG_ERROR, "Create failed: %s: %s",
-                        pathname, strerror (errno));
-                goto out;
-        }
-
-        fd = booster_get_process_fd ();
-        if (fd == -1) {
-                gf_log ("booster", GF_LOG_ERROR, "Failed to create fd");
-                goto close_out;
-        }
-
-        if ((booster_fd_unused_get (booster_fdtable, fh, fd)) == -1) {
-                gf_log ("booster", GF_LOG_ERROR, "Failed to map unused fd");
-                goto real_close_out;
-        }
-
-        return fd;
-
-real_close_out:
-        real_close (fd);
-        fd = -1;
-
-close_out:
-        glusterfs_close (fh);
-
-out:
-        return -1;
-}
-
-int __REDIRECT (booster_false_creat, (const char *pathname, mode_t mode),
-                creat) __nonnull ((1));
-int __REDIRECT (booster_false_creat64, (const char *pathname, mode_t mode),
-                creat64) __nonnull ((1));
-
-int
-booster_false_creat (const char *pathname, mode_t mode)
-{
-        int     ret = -1;
-        if (!pathname) {
-                errno = EINVAL;
-                goto out;
-        }
-
-        gf_log ("booster", GF_LOG_TRACE, "Create: %s", pathname);
-        ret = vmp_creat (pathname, mode);
-
-        if ((ret == -1) && (errno != ENODEV)) {
-                gf_log ("booster", GF_LOG_ERROR, "VMP create failed: %s",
-                        strerror (errno));
-                goto out;
-        }
-
-        if (ret > 0) {
-                gf_log ("booster", GF_LOG_TRACE, "File created");
-                goto out;
-        }
-
-        if (real_creat == NULL) {
-                errno = ENOSYS;
-                ret = -1;
-                goto out;
-        }
-
-        ret = real_creat (pathname, mode);
-
-        if (ret != -1) {
-                do_open (ret, pathname, GF_O_WRONLY | GF_O_TRUNC, mode,
-                         BOOSTER_CREAT);
-        } else
-                gf_log ("booster", GF_LOG_ERROR, "real create failed: %s",
-                        strerror (errno));
-
-out:
-        return ret;
-}
-
-
-int
-booster_false_creat64 (const char *pathname, mode_t mode)
-{
-        int     ret = -1;
-        if (!pathname) {
-                errno = EINVAL;
-                goto out;
-        }
-
-        gf_log ("booster", GF_LOG_TRACE, "Create: %s", pathname);
-        ret = vmp_creat (pathname, mode);
-
-        if ((ret == -1) && (errno != ENODEV)) {
-                gf_log ("booster", GF_LOG_ERROR, "VMP create failed: %s",
-                        strerror (errno));
-                goto out;
-        }
-
-        if (ret > 0) {
-                gf_log ("booster", GF_LOG_TRACE, "File created");
-                goto out;
-        }
-
-        if (real_creat64 == NULL) {
-                errno = ENOSYS;
-                ret = -1;
-                goto out;
-        }
-
-        ret = real_creat64 (pathname, mode);
-
-        if (ret != -1) {
-                do_open (ret, pathname, GF_O_WRONLY | GF_O_TRUNC, mode,
-                         BOOSTER_CREAT);
-        } else
-                gf_log ("booster", GF_LOG_ERROR, "real create failed: %s",
-                        strerror (errno));
-
-out:
-        return ret;
-}
-
-
-/* pread */
-
-ssize_t
-pread (int fd, void *buf, size_t count, unsigned long offset)
-{
-        ssize_t ret;
-        glusterfs_file_t glfs_fd = 0;
-
-        gf_log ("booster", GF_LOG_TRACE, "pread: fd %d, count %lu, offset %lu"
-                ,fd, (long unsigned)count, offset);
-        glfs_fd = booster_fdptr_get (booster_fdtable, fd);
-        if (!glfs_fd) { 
-                gf_log ("booster", GF_LOG_TRACE, "Not booster fd");
-                if (real_pread == NULL) {
-                        errno = ENOSYS;
-                        ret = -1;
-                } else
-                        ret = real_pread (fd, buf, count, offset);
-        } else {
-                gf_log ("booster", GF_LOG_TRACE, "Is a booster fd");
-                ret = glusterfs_pread (glfs_fd, buf, count, offset);
-                booster_fdptr_put (glfs_fd);
-        }
-
-        return ret;
-}
-
-
-ssize_t
-pread64 (int fd, void *buf, size_t count, uint64_t offset)
-{
-        ssize_t ret;
-        glusterfs_file_t glfs_fd = 0;
-
-        gf_log ("booster", GF_LOG_TRACE, "pread64: fd %d, count %lu, offset %"
-                PRIu64, fd, (long unsigned)count, offset);
-        glfs_fd = booster_fdptr_get (booster_fdtable, fd);
-        if (!glfs_fd) { 
-                gf_log ("booster", GF_LOG_TRACE, "Not booster fd");
-                if (real_pread64 == NULL) {
-                        errno = ENOSYS;
-                        ret = -1;
-                } else
-                        ret = real_pread64 (fd, buf, count, offset);
-        } else {
-                gf_log ("booster", GF_LOG_TRACE, "Is a booster fd");
-                ret = glusterfs_pread (glfs_fd, buf, count, offset);
-                booster_fdptr_put (glfs_fd);
-        }
-
-        return ret;
-}
-
-
-ssize_t
-read (int fd, void *buf, size_t count)
-{
-        int ret;
-        glusterfs_file_t glfs_fd;
-
-        gf_log ("booster", GF_LOG_TRACE, "read: fd %d, count %lu", fd,
-                (long unsigned)count);
-        glfs_fd = booster_fdptr_get (booster_fdtable, fd);
-        if (!glfs_fd) {
-                gf_log ("booster", GF_LOG_TRACE, "Not booster fd");
-                if (real_read == NULL) {
-                        errno = ENOSYS;
-                        ret = -1;
-                } else
-                        ret = real_read (fd, buf, count);
-        } else {
-                gf_log ("booster", GF_LOG_TRACE, "Is a booster fd");
-                ret = glusterfs_read (glfs_fd, buf, count);
-                booster_fdptr_put (glfs_fd);
-        }
-
-        return ret;
-}
-
-
-ssize_t
-readv (int fd, const struct iovec *vector, int count)
-{
-        int ret;
-        glusterfs_file_t glfs_fd = 0;
-
-        gf_log ("booster", GF_LOG_TRACE, "readv: fd %d, iovecs %d", fd, count);
-        glfs_fd = booster_fdptr_get (booster_fdtable, fd);
-        if (!glfs_fd) {
-                gf_log ("booster", GF_LOG_TRACE, "Not a booster fd");
-                if (real_readv == NULL) {
-                        errno = ENOSYS;
-                        ret = -1;
-                } else
-                        ret = real_readv (fd, vector, count);
-        } else {
-                gf_log ("booster", GF_LOG_TRACE, "Is a booster fd");
-		ret = glusterfs_readv (glfs_fd, vector, count);
-                booster_fdptr_put (glfs_fd);
-        }
-
-        return ret;
-}
-
-
-ssize_t
-write (int fd, const void *buf, size_t count)
-{
-        int ret;
-        glusterfs_file_t glfs_fd = 0;
-
-        gf_log ("booster", GF_LOG_TRACE, "write: fd %d, count %"GF_PRI_SIZET,
-                fd, count);
-
-        glfs_fd = booster_fdptr_get (booster_fdtable, fd);
-
-        if (!glfs_fd) {
-                gf_log ("booster", GF_LOG_TRACE, "Not a booster fd");
-                if (real_write == NULL) {
-                        errno = ENOSYS;
-                        ret = -1;
-                } else
-                        ret = real_write (fd, buf, count);
-        } else {
-                gf_log ("booster", GF_LOG_TRACE, "Is a booster fd");
-                ret = glusterfs_write (glfs_fd, buf, count);
-                booster_fdptr_put (glfs_fd);
-        }
- 
-        return ret;
-}
-
-ssize_t
-writev (int fd, const struct iovec *vector, int count)
-{
-        int ret = 0;
-        glusterfs_file_t glfs_fd = 0; 
-
-        gf_log ("booster", GF_LOG_TRACE, "writev: fd %d, iovecs %d", fd, count);
-        glfs_fd = booster_fdptr_get (booster_fdtable, fd);
-
-        if (!glfs_fd) {
-                gf_log ("booster", GF_LOG_TRACE, "Not a booster fd");
-                if (real_writev == NULL) {
-                        errno = ENOSYS;
-                        ret = -1;
-                } else
-                        ret = real_writev (fd, vector, count);
-        } else {
-                gf_log ("booster", GF_LOG_TRACE, "Is a booster fd");
-                ret = glusterfs_writev (glfs_fd, vector, count);
-                booster_fdptr_put (glfs_fd);
-        }
-
-        return ret;
-}
-
-
-ssize_t
-pwrite (int fd, const void *buf, size_t count, unsigned long offset)
-{
-        int ret;
-        glusterfs_file_t glfs_fd = 0;
-
-        gf_log ("booster", GF_LOG_TRACE, "pwrite: fd %d, count %"GF_PRI_SIZET
-                ", offset %lu", fd, count, offset);
-        glfs_fd = booster_fdptr_get (booster_fdtable, fd);
-
-        if (!glfs_fd) {
-                gf_log ("booster", GF_LOG_TRACE, "Not a booster fd");
-                if (real_pwrite == NULL) {
-                        errno = ENOSYS;
-                        ret = -1;
-                } else
-                        ret = real_pwrite (fd, buf, count, offset);
-        } else {
-                gf_log ("booster", GF_LOG_TRACE, "Is a booster fd");
-                ret = glusterfs_pwrite (glfs_fd, buf, count, offset);
-                booster_fdptr_put (glfs_fd);
-        }
-
-        return ret;
-}
-
-
-ssize_t
-pwrite64 (int fd, const void *buf, size_t count, uint64_t offset)
-{
-        int ret;
-        glusterfs_file_t glfs_fd = 0;
-
-        gf_log ("booster", GF_LOG_TRACE, "pwrite64: fd %d, count %"GF_PRI_SIZET
-                ", offset %"PRIu64, fd, count, offset);
-        glfs_fd = booster_fdptr_get (booster_fdtable, fd);
-
-        if (!glfs_fd) {
-                gf_log ("booster", GF_LOG_TRACE, "Not a booster fd");
-                if (real_pwrite64 == NULL) {
-                        errno = ENOSYS;
-                        ret = -1;
-                } else
-                        ret = real_pwrite64 (fd, buf, count, offset);
-        } else {
-                gf_log ("booster", GF_LOG_TRACE, "Is a booster fd");
-                ret = glusterfs_pwrite (glfs_fd, buf, count, offset);
-                booster_fdptr_put (glfs_fd);
-        }
-
-        return ret;
-}
-
-
-int
-close (int fd)
-{
-        int ret = -1;
-        glusterfs_file_t glfs_fd = 0;
-
-        gf_log ("booster", GF_LOG_TRACE, "close: fd %d", fd);
-	glfs_fd = booster_fdptr_get (booster_fdtable, fd);
-    
-	if (glfs_fd) {
-                gf_log ("booster", GF_LOG_TRACE, "Is a booster fd");
-		booster_fd_put (booster_fdtable, fd);
-		ret = glusterfs_close (glfs_fd);
-		booster_fdptr_put (glfs_fd);
-	}
-
-        ret = real_close (fd);
-
-        return ret;
-}
-
-#ifndef _LSEEK_DECLARED
-#define _LSEEK_DECLARED
-off_t
-lseek (int filedes, unsigned long offset, int whence)
-{
-        int ret;
-        glusterfs_file_t glfs_fd = 0;
-
-        gf_log ("booster", GF_LOG_TRACE, "lseek: fd %d, offset %ld",
-                filedes, offset);
-
-        glfs_fd = booster_fdptr_get (booster_fdtable, filedes);
-        if (glfs_fd) {
-                gf_log ("booster", GF_LOG_TRACE, "Is a booster fd");
-                ret = glusterfs_lseek (glfs_fd, offset, whence);
-                booster_fdptr_put (glfs_fd);
-        } else {
-                gf_log ("booster", GF_LOG_TRACE, "Not a booster fd");
-                if (real_lseek == NULL) {
-                        errno = ENOSYS;
-                        ret = -1;
-                } else
-                        ret = real_lseek (filedes, offset, whence);
-        }
-
-        return ret;
-}
-#endif
-
-off_t
-lseek64 (int filedes, uint64_t offset, int whence)
-{
-        int ret;
-        glusterfs_file_t glfs_fd = 0;
-
-
-        gf_log ("booster", GF_LOG_TRACE, "lseek: fd %d, offset %"PRIu64,
-                filedes, offset);
-        glfs_fd = booster_fdptr_get (booster_fdtable, filedes);
-        if (glfs_fd) {
-                gf_log ("booster", GF_LOG_TRACE, "Is a booster fd");
-                ret = glusterfs_lseek (glfs_fd, offset, whence);
-                booster_fdptr_put (glfs_fd);
-        } else {
-                gf_log ("booster", GF_LOG_TRACE, "Not a booster fd");
-                if (real_lseek64 == NULL) {
-                        errno = ENOSYS;
-                        ret = -1;
-                } else
-                        ret = real_lseek64 (filedes, offset, whence);
-        }
-
-        return ret;
-}
-
-int 
-dup (int oldfd)
-{
-        int ret = -1, new_fd = -1;
-        glusterfs_file_t glfs_fd = 0;
-
-        gf_log ("booster", GF_LOG_TRACE, "dup: fd %d", oldfd);
-        glfs_fd = booster_fdptr_get (booster_fdtable, oldfd);
-        new_fd = real_dup (oldfd);
-
-        if (new_fd >=0 && glfs_fd) {
-                gf_log ("booster", GF_LOG_TRACE, "Is a booster fd");
-                ret = booster_fd_unused_get (booster_fdtable, glfs_fd,
-                                             new_fd);
-                fd_ref ((fd_t *)glfs_fd);
-                if (ret == -1) {
-                        gf_log ("booster", GF_LOG_ERROR,"Failed to map new fd");
-                        real_close (new_fd);
-                } 
-        }
-
-        if (glfs_fd) {
-                booster_fdptr_put (glfs_fd);
-        }
-
-        return new_fd;
-}
-
-
-int 
-dup2 (int oldfd, int newfd)
-{
-        int ret = -1;
-        glusterfs_file_t old_glfs_fd = NULL, new_glfs_fd = NULL;
-
-        if (oldfd == newfd) {
-                return newfd;
-        }
-
-        old_glfs_fd = booster_fdptr_get (booster_fdtable, oldfd);
-        new_glfs_fd = booster_fdptr_get (booster_fdtable, newfd);
- 
-        ret = real_dup2 (oldfd, newfd); 
-        if (ret >= 0) {
-                if (new_glfs_fd) {
-                        glusterfs_close (new_glfs_fd);
-                        booster_fdptr_put (new_glfs_fd);
-                        booster_fd_put (booster_fdtable, newfd);
-                        new_glfs_fd = 0;
-                }
-
-                if (old_glfs_fd) {
-                        ret = booster_fd_unused_get (booster_fdtable,
-                                                     old_glfs_fd, newfd);
-                        fd_ref ((fd_t *)old_glfs_fd);
-                        if (ret == -1) {
-                                real_close (newfd);
-                        }
-                }
-        } 
-
-        if (old_glfs_fd) {
-                booster_fdptr_put (old_glfs_fd);
-        }
-
-        if (new_glfs_fd) {
-                booster_fdptr_put (new_glfs_fd);
-        }
-
-        return ret;
-}
-
-int
-mkdir (const char *pathname, mode_t mode)
-{
-        int     ret = -1;
-
-        gf_log ("booster", GF_LOG_TRACE, "mkdir: path %s", pathname);
-        ret = glusterfs_mkdir (pathname, mode);
-
-        if ((ret == -1) && (errno != ENODEV)) {
-                gf_log ("booster", GF_LOG_ERROR, "mkdir failed: %s",
-                        strerror (errno));
-                return ret;
-        }
-
-        if (ret == 0) {
-                gf_log ("booster", GF_LOG_TRACE, "directory created");
-                return ret;
-        }
-
-        if (real_mkdir == NULL) {
-                ret = -1;
-                errno = ENOSYS;
-        } else
-                ret = real_mkdir (pathname, mode);
-
-        return ret;
-}
-
-int
-rmdir (const char *pathname)
-{
-        int     ret = -1;
-
-        gf_log ("booster", GF_LOG_TRACE, "rmdir: path %s", pathname);
-        ret = glusterfs_rmdir (pathname);
-        if ((ret == -1) && (errno != ENODEV)) {
-                gf_log ("booster", GF_LOG_ERROR, "rmdir failed: %s",
-                        strerror (errno));
-                return ret;
-        }
-
-        if (ret == 0) {
-                gf_log ("booster", GF_LOG_TRACE, "directory removed");
-                return ret;
-        }
-
-        if (real_rmdir == NULL) {
-                errno = ENOSYS;
-                ret = -1;
-        } else
-                ret = real_rmdir (pathname);
-
-        return ret;
-}
-
-int
-chmod (const char *pathname, mode_t mode)
-{
-        int     ret = -1;
-
-        gf_log ("booster", GF_LOG_TRACE, "chmod: path %s", pathname);
-        ret = glusterfs_chmod (pathname, mode);
-        if ((ret == -1) && (errno != ENODEV)) {
-                gf_log ("booster", GF_LOG_ERROR, "chmod failed: %s",
-                        strerror (errno));
-                return ret;
-        }
-
-        if (ret == 0) {
-                gf_log ("booster", GF_LOG_TRACE, "chmod succeeded");
-                return ret;
-        }
-
-        if (real_chmod == NULL) {
-                errno = ENOSYS;
-                ret = -1;
-        } else
-                ret = real_chmod (pathname, mode);
-
-        return ret;
-}
-
-int
-chown (const char *pathname, uid_t owner, gid_t group)
-{
-        int     ret = -1;
-
-        gf_log ("booster", GF_LOG_TRACE, "chown: path: %s", pathname);
-        ret = glusterfs_chown (pathname, owner, group);
-
-        if ((ret == -1) && (errno != ENODEV)) {
-                gf_log ("booster", GF_LOG_ERROR, "chown failed: %s\n",
-                        strerror (errno));
-                return ret;
-        }
-
-        if (ret == 0) {
-                gf_log ("booster", GF_LOG_TRACE, "chown succeeded");
-                return ret;
-        }
-
-        if (real_chown == NULL) {
-                errno = ENOSYS;
-                ret = -1;
-        } else
-                ret = real_chown (pathname, owner, group);
-
-        return ret;
-}
-
-int
-fchown (int fd, uid_t owner, gid_t group)
-{
-        int                     ret = -1;
-        glusterfs_file_t        fh = NULL;
-
-        gf_log ("booster", GF_LOG_TRACE, "fchown: fd %d, uid %d, gid %d", fd,
-                owner, group);
-        fh = booster_fdptr_get (booster_fdtable, fd);
-        if (!fh) {
-                gf_log ("booster", GF_LOG_TRACE, "Not a booster fd");
-                if (real_fchown == NULL) {
-                        errno = ENOSYS;
-                        ret = -1;
-                } else
-                        ret = real_fchown (fd, owner, group);
-        } else {
-                gf_log ("booster", GF_LOG_TRACE, "Is a booster fd");
-                ret = glusterfs_fchown (fh, owner, group);
-                booster_fdptr_put (fh);
-        }
-
-        return ret;
-}
-
-
-#define MOUNT_TABLE_HASH_SIZE 256
-
-
-static void booster_cleanup (void);
-static int 
-booster_init (void)
-{
-        char    *booster_conf_path = NULL;
-        int     ret = -1;
-        int     pipefd[2];
-
-        booster_fdtable = booster_fdtable_alloc ();
-        if (!booster_fdtable) {
-                fprintf (stderr, "cannot allocate fdtable: %s\n",
-                         strerror (errno));
-		goto err;
-        }
- 
-        if (pipe (pipefd) == -1) {
-                gf_log ("booster-fstab", GF_LOG_ERROR, "Pipe creation failed:%s"
-                        , strerror (errno));
-                goto err;
-        }
-
-        process_piped_fd = pipefd[0];
-        real_close (pipefd[1]);
-        /* libglusterfsclient based VMPs should be inited only
-         * after the file tables are inited so that if the socket
-         * calls use the fd based syscalls, the fd tables are
-         * correctly initialized to return a NULL handle, on which the
-         * socket calls will fall-back to the real API.
-         */
-        booster_conf_path = getenv (BOOSTER_CONF_ENV_VAR);
-        if (booster_conf_path != NULL) {
-                if (strlen (booster_conf_path) > 0)
-                        ret = booster_configure (booster_conf_path);
-                else {
-                        gf_log ("booster", GF_LOG_ERROR, "%s not defined, "
-                                "using default path: %s", BOOSTER_CONF_ENV_VAR,
-                                DEFAULT_BOOSTER_CONF);
-                        ret = booster_configure (DEFAULT_BOOSTER_CONF);
-                }
-        } else {
-                gf_log ("booster", GF_LOG_ERROR, "%s not defined, using default"
-                        " path: %s", BOOSTER_CONF_ENV_VAR,DEFAULT_BOOSTER_CONF);
-                ret = booster_configure (DEFAULT_BOOSTER_CONF);
-        }
-
-        atexit (booster_cleanup);
-        if (ret == 0)
-                gf_log ("booster", GF_LOG_DEBUG, "booster is inited");
-	return 0;
-
-err:
-        /* Sure we return an error value here
-         * but who cares about booster.
-         */
-	return -1; 
-}
-
-
-static void
-booster_cleanup (void)
-{
-        /* Ideally, we should be de-initing the fd-table
-         * here but the problem is that I've seen file accesses through booster
-         * continuing while the atexit registered function is called. That means
-         * , we cannot dealloc the fd-table since then there could be a crash
-         * while trying to determine whether a given fd is for libc or for
-         * libglusterfsclient.
-         * We should be satisfied with having cleaned up glusterfs contexts.
-         */
-        glusterfs_umount_all ();
-	glusterfs_reset ();
-}
-
-int
-fchmod (int fd, mode_t mode)
-{
-        int                     ret = -1;
-        glusterfs_file_t        fh = NULL;
-
-        gf_log ("booster", GF_LOG_TRACE, "fchmod: fd %d, mode: 0x%x", fd, mode);
-        fh = booster_fdptr_get (booster_fdtable, fd);
-        if (!fh) {
-                gf_log ("booster", GF_LOG_TRACE, "Not a booster fd");
-                if (real_fchmod == NULL) {
-                        errno = ENOSYS;
-                        ret = -1;
-                } else
-                        ret = real_fchmod (fd, mode);
-        } else {
-                gf_log ("booster", GF_LOG_TRACE, "Is a booster fd");
-                ret = glusterfs_fchmod (fh, mode);
-                booster_fdptr_put (fh);
-        }
-
-        return ret;
-}
-
-int
-fsync (int fd)
-{
-        int                     ret = -1;
-        glusterfs_file_t        fh = NULL;
-
-        gf_log ("booster", GF_LOG_TRACE, "fsync: fd %d", fd);
-        fh = booster_fdptr_get (booster_fdtable, fd);
-        if (!fh) {
-                gf_log ("booster", GF_LOG_TRACE, "Not a booster fd");
-                if (real_fsync == NULL) {
-                        errno = ENOSYS;
-                        ret = -1;
-                } else
-                        ret = real_fsync (fd);
-        } else {
-                gf_log ("booster", GF_LOG_TRACE, "Is a booster fd");
-                ret = glusterfs_fsync (fh);
-                booster_fdptr_put (fh);
-        }
-
-        return ret;
-}
-
-int __REDIRECT (booster_false_ftruncate, (int fd, off_t length),
-                ftruncate);
-int __REDIRECT (booster_false_ftruncate64, (int fd, loff_t length),
-                ftruncate64);
-
-int
-booster_false_ftruncate (int fd, off_t length)
-{
-        int                     ret = -1;
-        glusterfs_file_t        fh = NULL;
-
-        gf_log ("booster", GF_LOG_TRACE, "ftruncate: fd %d, length: %"PRIu64,fd
-                , length);
-        fh = booster_fdptr_get (booster_fdtable, fd);
-        if (!fh) {
-                gf_log ("booster", GF_LOG_TRACE, "Not a booster fd");
-                if (real_ftruncate == NULL) {
-                        errno = ENOSYS;
-                        ret = -1;
-                } else
-                        ret = real_ftruncate (fd, length);
-        } else {
-                gf_log ("booster", GF_LOG_TRACE, "Is a booster fd");
-                ret = glusterfs_ftruncate (fh, length);
-                booster_fdptr_put (fh);
-        }
-
-        return ret;
-}
-
-int
-booster_false_ftruncate64 (int fd, loff_t length)
-{
-        int                     ret = -1;
-        glusterfs_file_t        fh = NULL;
-
-        gf_log ("booster", GF_LOG_TRACE, "ftruncate: fd %d, length: %"PRIu64,fd
-                , length);
-        fh = booster_fdptr_get (booster_fdtable, fd);
-        if (!fh) {
-                gf_log ("booster", GF_LOG_TRACE, "Not a booster fd");
-                if (real_ftruncate == NULL) {
-                        errno = ENOSYS;
-                        ret = -1;
-                } else
-                        ret = real_ftruncate64 (fd, length);
-        } else {
-                gf_log ("booster", GF_LOG_TRACE, "Is a booster fd");
-                ret = glusterfs_ftruncate (fh, length);
-                booster_fdptr_put (fh);
-        }
-
-        return ret;
-}
-
-int
-link (const char *old, const char *new)
-{
-        int             ret = -1;
-
-        gf_log ("booster", GF_LOG_TRACE, "link: old: %s, new: %s", old, new);
-        ret = glusterfs_link (old, new);
-
-        if ((ret == -1) && (errno != ENODEV)) {
-                gf_log ("booster", GF_LOG_ERROR, "Link failed: %s",
-                        strerror (errno));
-                return ret;
-        }
-
-        if (ret == 0) {
-                gf_log ("booster", GF_LOG_TRACE, "link call succeeded");
-                return ret;
-        }
-
-        if (real_link == NULL) {
-                errno = ENOSYS;
-                ret = -1;
-        } else
-                ret = real_link (old, new);
-
-        return ret;
-}
-
-int
-rename (const char *old, const char *new)
-{
-        int             ret = -1;
-
-        gf_log ("booster", GF_LOG_TRACE, "link: old: %s, new: %s", old, new);
-        ret = glusterfs_rename (old, new);
-
-        if ((ret == -1) && (errno != ENODEV)) {
-                gf_log ("booster", GF_LOG_ERROR, "Rename failed: %s",
-                        strerror (errno));
-                return ret;
-        }
-
-        if (ret == 0) {
-                gf_log ("booster", GF_LOG_TRACE, "Rename succeeded");
-                return ret;
-        }
-
-        if (real_rename == NULL) {
-                errno = ENOSYS;
-                ret = -1;
-        } else
-                ret = real_rename (old, new);
-
-        return ret;
-}
-
-int
-utimes (const char *path, const struct timeval times[2])
-{
-        int             ret = -1;
-
-        gf_log ("booster", GF_LOG_TRACE, "utimes: path %s", path);
-        ret = glusterfs_utimes (path, times);
-        if ((ret == -1) && (errno != ENODEV)) {
-                gf_log ("booster", GF_LOG_ERROR, "utimes failed: %s",
-                        strerror (errno));
-                return ret;
-        }
-
-        if (ret == 0) {
-                gf_log ("booster", GF_LOG_TRACE, "utimes succeeded");
-                return ret;
-        }
-
-        if (real_utimes == NULL) {
-                errno = ENOSYS;
-                ret = -1;
-        } else
-                ret = real_utimes (path, times);
-
-        return ret;
-}
-
-int
-utime (const char *path, const struct utimbuf *buf)
-{
-        int     ret = -1;
-
-        gf_log ("booster", GF_LOG_TRACE, "utime: path %s", path);
-        ret = glusterfs_utime (path, buf);
-
-        if ((ret == -1) && (errno != ENODEV)) {
-                gf_log ("booster", GF_LOG_ERROR, "utime failed: %s",
-                        strerror (errno));
-                return ret;
-        }
-
-        if (ret == 0) {
-                gf_log ("booster", GF_LOG_TRACE, "utime succeeded");
-                return ret;
-        }
-
-        if (real_utime == NULL) {
-                errno = ENOSYS;
-                ret = -1;
-        } else
-                ret = real_utime (path, buf);
-
-        return ret;
-}
-
-int
-mknod (const char *path, mode_t mode, dev_t dev)
-{
-        int     ret = -1;
-
-        gf_log ("booster", GF_LOG_TRACE, "mknod: path %s", path);
-        ret = glusterfs_mknod (path, mode, dev);
-        if ((ret == -1) && (errno != ENODEV)) {
-                gf_log ("booster", GF_LOG_ERROR, "mknod failed: %s",
-                        strerror (errno));
-                return ret;
-        }
-
-        if (ret == 0) {
-                gf_log ("booster", GF_LOG_TRACE, "mknod succeeded");
-                return ret;
-        }
-
-        if (real_mknod) {
-                errno = ENOSYS;
-                ret = -1;
-        } else
-                ret = real_mknod (path, mode, dev);
-
-        return ret;
-}
-
-int
-mkfifo (const  char *path, mode_t mode)
-{
-        int     ret = -1;
-
-        gf_log ("booster", GF_LOG_TRACE, "mkfifo: path %s", path);
-        ret = glusterfs_mkfifo (path, mode);
-        if ((ret == -1) && (errno != ENODEV)) {
-                gf_log ("booster", GF_LOG_ERROR, "mkfifo failed: %s",
-                        strerror (errno));
-                return ret;
-        }
-
-        if (ret == 0) {
-                gf_log ("booster", GF_LOG_TRACE, "mkfifo succeeded");
-                return ret;
-        }
-
-        if (real_mkfifo == NULL) {
-                errno = ENOSYS;
-                ret = -1;
-        } else
-                ret = real_mkfifo (path, mode);
-
-        return ret;
-}
-
-int
-unlink (const char *path)
-{
-        int     ret = -1;
-
-        gf_log ("booster", GF_LOG_TRACE, "unlink: path %s", path);
-        ret = glusterfs_unlink (path);
-        if ((ret == -1) && (errno != ENODEV)) {
-                gf_log ("booster", GF_LOG_ERROR, "unlink failed: %s",
-                        strerror (errno));
-                return ret;
-        }
-
-        if (ret == 0) {
-                gf_log ("booster", GF_LOG_TRACE, "unlink succeeded");
-                return ret;
-        }
-
-        if (real_unlink == NULL) {
-                errno = ENOSYS;
-                ret = -1;
-        } else
-                ret = real_unlink (path);
-
-        return ret;
-}
-
-int
-symlink (const char *oldpath, const char *newpath)
-{
-        int     ret = -1;
-
-        gf_log ("booster", GF_LOG_TRACE, "symlink: old: %s, new: %s",
-                oldpath, newpath);
-        ret = glusterfs_symlink (oldpath, newpath);
-        if ((ret == -1) && (errno != ENODEV)) {
-                gf_log ("booster", GF_LOG_ERROR, "symlink failed: %s",
-                        strerror (errno));
-                return ret;
-        }
-
-        if (ret == 0) {
-                gf_log ("booster", GF_LOG_TRACE, "symlink succeeded");
-                return ret;
-        }
-
-        if (real_symlink == NULL) {
-                errno = ENOSYS;
-                ret = -1;
-        } else
-                ret = real_symlink (oldpath, newpath);
-
-        return ret;
-}
-
-int
-readlink (const char *path, char *buf, size_t bufsize)
-{
-        int     ret = -1;
-
-        gf_log ("booster", GF_LOG_TRACE, "readlink: path %s", path);
-        ret = glusterfs_readlink (path, buf, bufsize);
-        if ((ret == -1) && (errno != ENODEV)) {
-                gf_log ("booster", GF_LOG_ERROR, "readlink failed: %s",
-                        strerror (errno));
-                return ret;
-        }
-
-        if (ret > 0) {
-                gf_log ("booster", GF_LOG_TRACE, "readlink succeeded");
-                return ret;
-        }
-
-        if (real_readlink == NULL) {
-                errno = ENOSYS;
-                ret = -1;
-        } else
-                ret = real_readlink (path, buf, bufsize);
-
-        return ret;
-}
-
-char *
-realpath (const char *path, char *resolved_path)
-{
-        char    *res = NULL;
-
-        gf_log ("booster", GF_LOG_TRACE, "realpath: path %s", path);
-        res = glusterfs_realpath (path, resolved_path);
-        if ((res == NULL) && (errno != ENODEV)) {
-                gf_log ("booster", GF_LOG_ERROR, "realpath failed: %s",
-                        strerror (errno));
-                return res;
-        }
-
-        if (res != NULL) {
-                gf_log ("booster", GF_LOG_TRACE, "realpath succeeded");
-                return res;
-        }
-
-        if (real_realpath == NULL) {
-                errno = ENOSYS;
-                res = NULL;
-        } else
-                res = real_realpath (path, resolved_path);
-
-        return res;
-}
-
-#define BOOSTER_GL_DIR          1
-#define BOOSTER_POSIX_DIR       2
-
-struct booster_dir_handle {
-        int type;
-        void *dirh;
-};
-
-DIR *
-opendir (const char *path)
-{
-        glusterfs_dir_t                 gdir = NULL;
-        struct booster_dir_handle       *bh = NULL;
-        DIR                             *pdir = NULL;
-
-        gf_log ("booster", GF_LOG_TRACE, "opendir: path: %s", path);
-        bh = calloc (1, sizeof (struct booster_dir_handle));
-        if (!bh) {
-                gf_log ("booster", GF_LOG_ERROR, "memory allocation failed");
-                errno = ENOMEM;
-                goto out;
-        }
-
-        gdir = glusterfs_opendir (path);
-        if (gdir) {
-                gf_log ("booster", GF_LOG_TRACE, "Gluster dir opened");
-                bh->type = BOOSTER_GL_DIR;
-                bh->dirh = (void *)gdir;
-                goto out;
-        } else if ((!gdir) && (errno != ENODEV)) {
-                gf_log ("booster", GF_LOG_ERROR, "Opendir failed");
-                goto free_out;
-        }
-
-        if (real_opendir == NULL) {
-                errno = ENOSYS;
-                goto free_out;
-        }
-
-        pdir = real_opendir (path);
-
-        if (pdir) {
-                bh->type = BOOSTER_POSIX_DIR;
-                bh->dirh = (void *)pdir;
-                goto out;
-        }
-
-free_out:
-        if (bh) {
-                free (bh);
-                bh = NULL;
-        }
-out:
-        return (DIR *)bh;
-}
-
-int __REDIRECT (booster_false_readdir_r, (DIR *dir, struct dirent *entry,
-                struct dirent **result), readdir_r) __nonnull ((1));
-int __REDIRECT (booster_false_readdir64_r, (DIR *dir, struct dirent64 *entry,
-                struct dirent64 **result), readdir64_r) __nonnull ((1));
-
-int
-booster_false_readdir_r (DIR *dir, struct dirent *entry, struct dirent **result)
-{
-        struct booster_dir_handle       *bh = (struct booster_dir_handle *)dir;
-        int                              ret = 0;  
-
-        if (!bh) {
-                ret = errno = EFAULT;
-                goto out;
-        }
-
-        if (bh->type == BOOSTER_GL_DIR) {
-                gf_log ("booster", GF_LOG_TRACE, "readdir_r on gluster");
-                ret = glusterfs_readdir_r ((glusterfs_dir_t)bh->dirh, entry,
-                                           result);
-                
-        } else if (bh->type == BOOSTER_POSIX_DIR) {
-                gf_log ("booster", GF_LOG_TRACE, "readdir_r on posix");
-                if (real_readdir_r == NULL) {
-                        ret = errno = ENOSYS;
-                        goto out;
-                }
-
-                ret = real_readdir_r ((DIR *)bh->dirh, entry, result);
-        } else {
-                ret = errno = EINVAL;
-        }
-
-out:
-        return  ret;
-}
-
-int
-booster_false_readdir64_r (DIR *dir, struct dirent64 *entry,
-                           struct dirent64 **result)
-{
-        struct booster_dir_handle       *bh = (struct booster_dir_handle *)dir;
-        int                              ret = 0;  
-
-        if (!bh) {
-                ret = errno = EFAULT;
-                goto out;
-        }
-
-        if (bh->type == BOOSTER_GL_DIR) {
-                gf_log ("booster", GF_LOG_TRACE, "readdir_r on gluster");
-                ret = glusterfs_readdir_r ((glusterfs_dir_t)bh->dirh,
-                                           (struct dirent *)entry,
-                                           (struct dirent **)result);
-        } else if (bh->type == BOOSTER_POSIX_DIR) {
-                gf_log ("booster", GF_LOG_TRACE, "readdir_r on posix");
-                if (real_readdir64_r == NULL) {
-                        ret = errno = ENOSYS;
-                        goto out;
-                }
-
-                ret = real_readdir64_r ((DIR *)bh->dirh, entry, result);
-        } else {
-                ret = errno = EINVAL;
-        }
-
-out:
-        return  ret;
-}
-
-struct dirent *
-__REDIRECT (booster_false_readdir, (DIR *dir), readdir) __nonnull ((1));
-
-struct dirent64 *
-__REDIRECT (booster_false_readdir64, (DIR *dir), readdir64) __nonnull ((1));
-
-struct dirent *
-booster_false_readdir (DIR *dir)
-{
-        struct booster_dir_handle       *bh = (struct booster_dir_handle *)dir;
-        struct dirent                   *dirp = NULL;
-
-        if (!bh) {
-                errno = EFAULT;
-                goto out;
-        }
-
-        if (bh->type == BOOSTER_GL_DIR) {
-                gf_log ("booster", GF_LOG_TRACE, "readdir on gluster");
-                dirp = glusterfs_readdir ((glusterfs_dir_t)bh->dirh);
-        } else if (bh->type == BOOSTER_POSIX_DIR) {
-                gf_log ("booster", GF_LOG_TRACE, "readdir on posix");
-                if (real_readdir == NULL) {
-                        errno = ENOSYS;
-                        dirp = NULL;
-                        goto out;
-                }
-
-                dirp = real_readdir ((DIR *)bh->dirh);
-        } else {
-                dirp = NULL;
-                errno = EINVAL;
-        }
-
-out:
-        return  dirp;
-}
-
-struct dirent64 *
-booster_false_readdir64 (DIR *dir)
-{
-        struct booster_dir_handle       *bh = (struct booster_dir_handle *)dir;
-        struct dirent64                 *dirp = NULL;
-
-        if (!bh) {
-                errno = EFAULT;
-                goto out;
-        }
-
-        if (bh->type == BOOSTER_GL_DIR) {
-                gf_log ("booster", GF_LOG_TRACE, "readdir on gluster");
-                dirp = glusterfs_readdir ((glusterfs_dir_t)bh->dirh);
-        } else if (bh->type == BOOSTER_POSIX_DIR) {
-                gf_log ("booster", GF_LOG_TRACE, "readdir on posix");
-                if (real_readdir == NULL) {
-                        errno = ENOSYS;
-                        dirp = NULL;
-                        goto out;
-                }
-
-                dirp = real_readdir64 ((DIR *)bh->dirh);
-        } else {
-                dirp = NULL;
-                errno = EINVAL;
-        }
-
-out:
-        return  dirp;
-}
-
-int
-closedir (DIR *dh)
-{
-        struct booster_dir_handle       *bh = (struct booster_dir_handle *)dh;
-        int                             ret = -1;
-
-        if (!bh) {
-                errno = EFAULT;
-                goto out;
-        }
-
-        if (bh->type == BOOSTER_GL_DIR) {
-                gf_log ("booster", GF_LOG_TRACE, "closedir on gluster");
-                ret = glusterfs_closedir ((glusterfs_dir_t)bh->dirh);
-        } else if (bh->type == BOOSTER_POSIX_DIR) {
-                gf_log ("booster", GF_LOG_TRACE, "closedir on posix");
-                if (real_closedir == NULL) {
-                        errno = ENOSYS;
-                        ret = -1;
-                } else
-                        ret = real_closedir ((DIR *)bh->dirh);
-        } else {
-                errno = EBADF;
-        }
-
-        if (ret == 0) {
-                free (bh);
-                bh = NULL;
-        }
-out:
-        return ret;
-}
-
-/* The real stat functions reside in booster_stat.c to
- * prevent clash with the statX prototype and functions
- * declared from sys/stat.h
- */
-int
-booster_xstat (int ver, const char *path, void *buf)
-{
-        struct stat     *sbuf = (struct stat *)buf;
-        int             ret = -1;
-
-        gf_log ("booster", GF_LOG_TRACE, "xstat: path: %s", path);
-        ret = glusterfs_stat (path, sbuf);
-        if ((ret == -1) && (errno != ENODEV)) {
-                gf_log ("booster", GF_LOG_ERROR, "xstat failed: %s",
-                        strerror (errno));
-                goto out;
-        }
-
-        if (ret == 0) {
-                gf_log ("booster", GF_LOG_TRACE, "xstat succeeded");
-                goto out;
-        }
-
-        if (real___xstat == NULL) {
-                ret = -1;
-                errno = ENOSYS;
-                goto out;
-        }
-
-        ret = real___xstat (ver, path, sbuf);
-out:
-        return ret;
-}
-
-int
-booster_xstat64 (int ver, const char *path, void *buf)
-{
-        int             ret = -1;
-        struct stat64   *sbuf = (struct stat64 *)buf;
-
-        gf_log ("booster", GF_LOG_TRACE, "xstat64: path: %s", path);
-        ret = glusterfs_stat (path, (struct stat *)sbuf);
-        if ((ret == -1) && (errno != ENODEV)) {
-                gf_log ("booster", GF_LOG_ERROR, "xstat64 failed: %s",
-                        strerror (errno));
-                goto out;
-        }
-
-        if (ret == 0) {
-                gf_log ("booster", GF_LOG_TRACE, "xstat64 succeeded");
-                goto out;
-        }
-
-        if (real___xstat64 == NULL) {
-                errno = ENOSYS;
-                ret = -1;
-                goto out;
-        }
-
-        ret = real___xstat64 (ver, path, sbuf);
-out:
-        return ret;
-}
-
-int
-booster_stat (const char *path, void *buf)
-{
-        struct stat     *sbuf = (struct stat *)buf;
-        int             ret = -1;
-
-        gf_log ("booster", GF_LOG_TRACE, "stat: path: %s", path);
-        ret = glusterfs_stat (path, sbuf);
-        if ((ret == -1) && (errno != ENODEV)) {
-                gf_log ("booster", GF_LOG_ERROR, "stat failed: %s",
-                        strerror (errno));
-                goto out;
-        }
-
-        if (ret == 0) {
-                gf_log ("booster", GF_LOG_TRACE, "stat succeeded");
-                goto out;
-        }
-
-        if (real_stat != NULL)
-                ret = real_stat (path, sbuf);
-        else if (real___xstat != NULL)
-                ret = real___xstat (0, path, sbuf);
-        else {
-                errno = ENOSYS;
-                ret = -1;
-                goto out;
-        }
-
-
-out:
-        return ret;
-}
-
-int
-booster_stat64 (const char *path, void *buf)
-{
-        int             ret = -1;
-        struct stat64   *sbuf = (struct stat64 *)buf;
-
-        gf_log ("booster", GF_LOG_TRACE, "stat64: %s", path);
-        ret = glusterfs_stat (path, (struct stat *)sbuf);
-        if ((ret == -1) && (errno != ENODEV)) {
-                gf_log ("booster", GF_LOG_ERROR, "stat64 failed: %s",
-                        strerror (errno));
-                goto out;
-        }
-
-        if (ret == 0) {
-                gf_log ("booster", GF_LOG_TRACE, "stat64 succeeded");
-                goto out;
-        }
-
-        if (real_stat64 != NULL)
-                ret = real_stat64 (path, sbuf);
-        else if (real___xstat64 != NULL)
-                ret = real___xstat64 (0, path, sbuf);
-        else {
-                errno = ENOSYS;
-                ret = -1;
-                goto out;
-        }
-
-out:
-        return ret;
-}
-
-int
-booster_fxstat (int ver, int fd, void *buf)
-{
-        struct stat             *sbuf = (struct stat *)buf;
-        int                     ret = -1;
-        glusterfs_file_t        fh = NULL;
-
-        gf_log ("booster", GF_LOG_TRACE, "fxstat: fd %d", fd);
-        fh = booster_fdptr_get (booster_fdtable, fd);
-        if (!fh) {
-                gf_log ("booster", GF_LOG_TRACE, "Not a booster fd");
-                if (real___fxstat == NULL) {
-                        errno = ENOSYS;
-                        ret = -1;
-                        goto out;
-                }
-
-                ret = real___fxstat (ver, fd, sbuf);
-        } else {
-                gf_log ("booster", GF_LOG_TRACE, "Is a booster fd");
-                ret = glusterfs_fstat (fh, sbuf);
-                booster_fdptr_put (fh);
-        }
-
-out:
-        return ret;
-}
-
-int
-booster_fxstat64 (int ver, int fd, void *buf)
-{
-        int                     ret = -1;
-        struct stat64           *sbuf = (struct stat64 *)buf;
-        glusterfs_file_t        fh = NULL;
-
-        gf_log ("booster", GF_LOG_TRACE, "fxstat64: fd %d", fd);
-        fh = booster_fdptr_get (booster_fdtable, fd);
-        if (!fh) {
-                gf_log ("booster", GF_LOG_TRACE, "Not a booster fd");
-                if (real___fxstat64 == NULL) {
-                        ret = -1;
-                        errno = ENOSYS;
-                        goto out;
-                }
-                ret = real___fxstat64 (ver, fd, sbuf);
-        } else {
-                gf_log ("booster", GF_LOG_TRACE, "Is a booster fd");
-                ret = glusterfs_fstat (fh, (struct stat *)sbuf);
-                booster_fdptr_put (fh);
-        }
-
-out:
-        return ret;
-}
-
-int
-booster_fstat (int fd, void *buf)
-{
-        struct stat             *sbuf = (struct stat *)buf;
-        int                     ret = -1;
-        glusterfs_file_t        fh = NULL;
-
-        gf_log ("booster", GF_LOG_TRACE, "fstat: fd %d", fd);
-        fh = booster_fdptr_get (booster_fdtable, fd);
-        if (!fh) {
-                gf_log ("booster", GF_LOG_TRACE, "Not a booster fd");
-                if (real_fstat != NULL)
-                        ret = real_fstat (fd, sbuf);
-                else if (real___fxstat != NULL)
-                        ret = real___fxstat (0, fd, sbuf);
-                else {
-                        ret = -1;
-                        errno = ENOSYS;
-                        goto out;
-                }
-        } else {
-                gf_log ("booster", GF_LOG_TRACE, "Is a booster fd");
-                ret = glusterfs_fstat (fh, sbuf);
-                booster_fdptr_put (fh);
-        }
-
-out:
-        return ret;
-}
-
-int
-booster_fstat64 (int fd, void *buf)
-{
-        int                     ret = -1;
-        struct stat64           *sbuf = (struct stat64 *)buf;
-        glusterfs_file_t        fh = NULL;
-
-        gf_log ("booster", GF_LOG_TRACE, "fstat64: fd %d", fd);
-        fh = booster_fdptr_get (booster_fdtable, fd);
-        if (!fh) {
-                gf_log ("booster", GF_LOG_TRACE, "Not a booster fd");
-                if (real_fstat64 != NULL)
-                        ret = real_fstat64 (fd, sbuf);
-                else if (real___fxstat64 != NULL)
-                        /* Not sure how portable the use of 0 for
-                         * version number is but it works over glibc.
-                         * We need this because, I've
-                         * observed that all the above real* functors can be
-                         * NULL. In that case, this is our last and only option.
-                         */
-                        ret = real___fxstat64 (0, fd, sbuf);
-                else {
-                        ret = -1;
-                        errno = ENOSYS;
-                        goto out;
-                }
-        } else {
-                gf_log ("booster", GF_LOG_TRACE, "Is a booster fd");
-                ret = glusterfs_fstat (fh, (struct stat *)sbuf);
-                booster_fdptr_put (fh);
-        }
-
-out:
-        return ret;
-}
-
-int
-booster_lxstat (int ver, const char *path, void *buf)
-{
-        struct stat     *sbuf = (struct stat *)buf;
-        int             ret = -1;
-
-        gf_log ("booster", GF_LOG_TRACE, "lxstat: path %s", path);
-        ret = glusterfs_lstat (path, sbuf);
-        if ((ret == -1) && (errno != ENODEV)) {
-                gf_log ("booster", GF_LOG_ERROR, "lxstat failed: %s",
-                        strerror (errno));
-                goto out;
-        }
-
-        if (ret == 0) {
-                gf_log ("booster", GF_LOG_TRACE, "lxstat succeeded");
-                goto out;
-        }
-
-        if (real___lxstat == NULL) {
-                ret = -1;
-                errno = ENOSYS;
-                goto out;
-        }
-
-        ret = real___lxstat (ver, path, sbuf);
-out:
-        return ret;
-}
-
-int
-booster_lxstat64 (int ver, const char *path, void *buf)
-{
-        int             ret = -1;
-        struct stat64   *sbuf = (struct stat64 *)buf;
-
-        gf_log ("booster", GF_LOG_TRACE, "lxstat64: path %s", path);
-        ret = glusterfs_lstat (path, (struct stat *)sbuf);
-        if ((ret == -1) && (errno != ENODEV)) {
-                gf_log ("booster", GF_LOG_ERROR, "lxstat64 failed: %s",
-                        strerror (errno));
-                goto out;
-        }
-
-        if (ret == 0) {
-                gf_log ("booster", GF_LOG_TRACE, "lxstat64 succeeded");
-                goto out;
-        }
-
-        if (real___lxstat64 == NULL) {
-                errno = ENOSYS;
-                ret = -1;
-                goto out;
-        }
-
-        ret = real___lxstat64 (ver, path, sbuf);
-out:
-        return ret;
-}
-
-int
-booster_lstat (const char *path, void *buf)
-{
-        struct stat     *sbuf = (struct stat *)buf;
-        int             ret = -1;
-
-        gf_log ("booster", GF_LOG_TRACE, "lstat: path %s", path);
-        ret = glusterfs_lstat (path, sbuf);
-        if ((ret == -1) && (errno != ENODEV)) {
-                gf_log ("booster", GF_LOG_ERROR, "lstat failed: %s",
-                        strerror (errno));
-                goto out;
-        }
-
-        if (ret == 0) {
-                gf_log ("booster", GF_LOG_TRACE, "lstat succeeded");
-                goto out;
-        }
-
-        if (real_lstat != NULL)
-                ret = real_lstat (path, sbuf);
-        else if (real___lxstat != NULL)
-                ret = real___lxstat (0, path, sbuf);
-        else {
-                errno = ENOSYS;
-                ret = -1;
-                goto out;
-        }
-
-
-out:
-        return ret;
-}
-
-int
-booster_lstat64 (const char *path, void *buf)
-{
-        int             ret = -1;
-        struct stat64   *sbuf = (struct stat64 *)buf;
-
-        gf_log ("booster", GF_LOG_TRACE, "lstat64: path %s", path);
-        ret = glusterfs_lstat (path, (struct stat *)sbuf);
-        if ((ret == -1) && (errno != ENODEV)) {
-                gf_log ("booster", GF_LOG_ERROR, "lstat64 failed: %s",
-                        strerror (errno));
-                goto out;
-        }
-
-        if (ret == 0) {
-                gf_log ("booster", GF_LOG_TRACE, "lstat64 succeeded");
-                goto out;
-        }
-
-        if (real_lstat64 != NULL)
-                ret = real_lstat64 (path, sbuf);
-        else if (real___lxstat64 != NULL)
-                ret = real___lxstat64 (0, path, sbuf);
-        else {
-                errno = ENOSYS;
-                ret = -1;
-                goto out;
-        }
-
-out:
-        return ret;
-}
-
-int
-booster_statfs (const char *pathname, struct statfs *buf)
-{
-        int             ret = -1;
-
-        gf_log ("booster", GF_LOG_TRACE, "statfs: path %s", pathname);
-        ret = glusterfs_statfs (pathname, buf);
-        if ((ret == -1) && (errno != ENODEV)) {
-                gf_log ("booster", GF_LOG_ERROR, "statfs failed: %s",
-                        strerror (errno));
-                goto out;
-        }
-
-        if (ret == 0) {
-                gf_log ("booster", GF_LOG_TRACE, "statfs succeeded");
-                goto out;
-        }
-
-        if (real_statfs == NULL) {
-                ret = -1;
-                errno = ENOSYS;
-                goto out;
-        }
-
-        ret = real_statfs (pathname, buf);
-
-out:
-        return ret;
-}
-
-int
-booster_statfs64 (const char *pathname, struct statfs64 *buf)
-{
-        int             ret = -1;
-
-        gf_log ("booster", GF_LOG_TRACE, "stat64: path %s", pathname);
-        ret = glusterfs_statfs (pathname, (struct statfs *)buf);
-        if ((ret == -1) && (errno != ENODEV)) {
-                gf_log ("booster", GF_LOG_ERROR, "statfs64 failed: %s",
-                        strerror (errno));
-                goto out;
-        }
-
-        if (ret == 0) {
-                gf_log ("booster", GF_LOG_TRACE, "statfs64 succeeded");
-                goto out;
-        }
-
-        if (real_statfs64 == NULL) {
-                ret = -1;
-                errno = ENOSYS;
-                goto out;
-        }
-
-        ret = real_statfs64 (pathname, buf);
-
-out:
-        return ret;
-}
-
-int
-booster_statvfs (const char *pathname, struct statvfs *buf)
-{
-        int             ret = -1;
-
-        gf_log ("booster", GF_LOG_TRACE, "statvfs: path %s", pathname);
-        ret = glusterfs_statvfs (pathname, buf);
-        if ((ret == -1) && (errno != ENODEV)) {
-                gf_log ("booster", GF_LOG_ERROR, "statvfs failed: %s",
-                        strerror (errno));
-                goto out;
-        }
-
-        if (ret == 0) {
-                gf_log ("booster", GF_LOG_TRACE, "statvfs succeeded");
-                goto out;
-        }
-
-        if (real_statvfs == NULL) {
-                ret = -1;
-                errno = ENOSYS;
-                goto out;
-        }
-
-        ret = real_statvfs (pathname, buf);
-
-out:
-        return ret;
-}
-
-int
-booster_statvfs64 (const char *pathname, struct statvfs64 *buf)
-{
-        int             ret = -1;
-
-        gf_log ("booster", GF_LOG_TRACE, "statvfs64: path %s", pathname);
-        ret = glusterfs_statvfs (pathname, (struct statvfs *)buf);
-        if ((ret == -1) && (errno != ENODEV)) {
-                gf_log ("booster", GF_LOG_ERROR, "statvfs64 failed: %s",
-                        strerror (errno));
-                goto out;
-        }
-
-        if (ret == 0) {
-                gf_log ("booster", GF_LOG_TRACE, "statvfs64 succeeded");
-                goto out;
-        }
-
-        if (real_statvfs64 == NULL) {
-                ret = -1;
-                errno = ENOSYS;
-                goto out;
-        }
-
-        ret = real_statvfs64 (pathname, buf);
-
-out:
-        return ret;
-}
-
-ssize_t
-getxattr (const char *path, const char *name, void *value, size_t size)
-{
-        int     ret = -1;
-
-        gf_log ("booster", GF_LOG_TRACE, "getxattr: path %s, name %s", path,
-                name);
-        ret = glusterfs_getxattr (path, name, value, size);
-        if ((ret == -1) && (errno != ENODEV)) {
-                gf_log ("booster", GF_LOG_ERROR, "getxattr failed: %s",
-                        strerror (errno));
-                goto out;
-        }
-
-        if (ret > 0) {
-                gf_log ("booster", GF_LOG_TRACE, "getxattr succeeded");
-                return ret;
-        }
-
-        if (real_getxattr == NULL) {
-                ret = -1;
-                errno = ENOSYS;
-                goto out;
-        }
-
-        ret = real_getxattr (path, name, value, size);
-out:
-        return ret;
-}
-
-
-ssize_t
-lgetxattr (const char *path, const char *name, void *value, size_t size)
-{
-        int     ret = -1;
-
-        gf_log ("booster", GF_LOG_TRACE, "lgetxattr: path %s, name %s", path,
-                name);
-        ret = glusterfs_lgetxattr (path, name, value, size);
-        if ((ret == -1) && (errno != ENODEV)) {
-                gf_log ("booster", GF_LOG_ERROR, "lgetxattr failed: %s",
-                        strerror (errno));
-
-                goto out;
-        }
-
-        if (ret > 0) {
-                gf_log ("booster", GF_LOG_TRACE, "lgetxattr succeeded");
-                return ret;
-        }
-
-        if (real_lgetxattr == NULL) {
-                ret = -1;
-                errno = ENOSYS;
-                goto out;
-        }
-
-        ret = real_lgetxattr (path, name, value, size);
-out:
-        return ret;
-}
-
-int
-remove (const char *path)
-{
-        int     ret = -1;
-
-        gf_log ("booster", GF_LOG_TRACE, "remove: %s", path);
-        ret = glusterfs_remove (path);
-        if ((ret == -1) && (errno != ENODEV)) {
-                gf_log ("booster", GF_LOG_ERROR, "remove failed: %s",
-                        strerror (errno));
-                goto out;
-        }
-
-        if (ret == 0) {
-                gf_log ("booster", GF_LOG_TRACE, "remove succeeded");
-                goto out;
-        }
-
-        if (real_remove == NULL) {
-                errno = ENOSYS;
-                ret = -1;
-                goto out;
-        }
-
-        ret = real_remove (path);
-
-out:
-        return ret;
-}
-
-int
-lchown (const char *path, uid_t owner, gid_t group)
-{
-        int     ret = -1;
-
-        gf_log ("booster", GF_LOG_TRACE, "lchown: path %s", path);
-        ret = glusterfs_lchown (path, owner, group);
-        if ((ret == -1) && (errno != ENODEV)) {
-                gf_log ("booster", GF_LOG_ERROR, "lchown failed: %s",
-                        strerror (errno));
-                goto out;
-        }
-
-        if (ret == 0) {
-                gf_log ("booster", GF_LOG_ERROR, "lchown succeeded");
-                goto out;
-        }
-
-        if (real_lchown == NULL) {
-                errno = ENOSYS;
-                ret = -1;
-                goto out;
-        }
-
-        ret = real_lchown (path, owner, group);
-
-out:
-        return ret;
-}
-
-void
-booster_rewinddir (DIR *dir)
-{
-        struct booster_dir_handle       *bh = (struct booster_dir_handle *)dir;
-
-        if (!bh) {
-                errno = EFAULT;
-                goto out;
-        }
-
-        if (bh->type == BOOSTER_GL_DIR) {
-                gf_log ("booster", GF_LOG_TRACE, "rewinddir on glusterfs");
-                glusterfs_rewinddir ((glusterfs_dir_t)bh->dirh);
-        } else if (bh->type == BOOSTER_POSIX_DIR) {
-                if (real_rewinddir == NULL) {
-                        errno = ENOSYS;
-                        goto out;
-                }
-                gf_log ("booster", GF_LOG_TRACE, "rewinddir on posix");
-                real_rewinddir ((DIR *)bh->dirh);
-        } else
-                errno = EINVAL;
-out:
-        return;
-}
-
-void
-booster_seekdir (DIR *dir, off_t offset)
-{
-        struct booster_dir_handle       *bh = (struct booster_dir_handle *)dir;
-
-        if (!bh) {
-                errno = EFAULT;
-                goto out;
-        }
-
-        if (bh->type == BOOSTER_GL_DIR) {
-                gf_log ("booster", GF_LOG_TRACE, "seekdir on glusterfs");
-                glusterfs_seekdir ((glusterfs_dir_t)bh->dirh, offset);
-         } else if (bh->type == BOOSTER_POSIX_DIR) {
-                if (real_seekdir == NULL) {
-                        errno = ENOSYS;
-                        goto out;
-                }
-
-                gf_log ("booster", GF_LOG_TRACE, "seekdir on posix");
-                real_seekdir ((DIR *)bh->dirh, offset);
-        } else
-                errno = EINVAL;
-out:
-        return;
-}
-
-off_t
-booster_telldir (DIR *dir)
-{
-        struct booster_dir_handle       *bh = (struct booster_dir_handle *)dir;
-        off_t	offset = -1;
-
-        if (!bh) {
-                errno = EFAULT;
-                goto out;
-        }
-
-        if (bh->type == BOOSTER_GL_DIR) {
-                gf_log ("booster", GF_LOG_TRACE, "telldir on glusterfs");
-                offset = glusterfs_telldir ((glusterfs_dir_t)bh->dirh);
-        } else if (bh->type == BOOSTER_POSIX_DIR) {
-                if (real_telldir == NULL) {
-                        errno = ENOSYS;
-                        goto out;
-                }
-
-                gf_log ("booster", GF_LOG_TRACE, "telldir on posix");
-                offset = real_telldir ((DIR *)bh->dirh);
-        } else
-                errno = EINVAL;
-out:
-        return offset;
-}
-
-
-pid_t 
-fork (void)
-{
-	pid_t pid = 0;
-	char child = 0;
-
-	glusterfs_log_lock ();
-	{
-		pid = real_fork ();
-	}
-	glusterfs_log_unlock ();
-
-	child = (pid == 0);
-	if (child) {
-		booster_cleanup ();
-		booster_init ();
-	}
-
-	return pid;
-}
-
-ssize_t
-sendfile (int out_fd, int in_fd, off_t *offset, size_t count)
-{
-        glusterfs_file_t            in_fh = NULL;
-        ssize_t                     ret = -1;
-
-        gf_log ("booster", GF_LOG_TRACE, "sendfile: in fd %d, out fd %d, offset"
-                " %"PRIu64", count %"GF_PRI_SIZET, in_fd, out_fd, *offset,
-                count);
-        /*
-         * handle sendfile in booster only if in_fd corresponds to a glusterfs
-         * file handle 
-         */
-        in_fh = booster_fdptr_get (booster_fdtable, in_fd);
-        if (!in_fh) {
-                gf_log ("booster", GF_LOG_TRACE, "Not a booster fd");
-                if (real_sendfile == NULL) {
-                        errno = ENOSYS;
-                        ret = -1;
-                } else {
-                        ret = real_sendfile (out_fd, in_fd, offset, count);
-                }
-        } else {
-                gf_log ("booster", GF_LOG_TRACE, "Is a booster fd");
-                ret = glusterfs_sendfile (out_fd, in_fh, offset, count);
-                booster_fdptr_put (in_fh);
-        }
-        
-        return ret;
-}
-
-ssize_t
-sendfile64 (int out_fd, int in_fd, off_t *offset, size_t count)
-{
-        glusterfs_file_t            in_fh = NULL;
-        ssize_t                     ret = -1;
-
-        gf_log ("booster", GF_LOG_TRACE, "sendfile64: in fd %d, out fd %d,"
-                " offset %"PRIu64", count %"GF_PRI_SIZET, in_fd, out_fd,
-                *offset, count);
-        /*
-         * handle sendfile in booster only if in_fd corresponds to a glusterfs
-         * file handle 
-         */
-        in_fh = booster_fdptr_get (booster_fdtable, in_fd);
-        if (!in_fh) {
-                gf_log ("booster", GF_LOG_TRACE, "Not a booster fd");
-                if (real_sendfile64 == NULL) {
-                        errno = ENOSYS;
-                        ret = -1;
-                } else {
-                        ret = real_sendfile64 (out_fd, in_fd, offset, count);
-                }
-        } else {
-                gf_log ("booster", GF_LOG_TRACE, "Is a booster fd");
-                ret = glusterfs_sendfile (out_fd, in_fh, offset, count);
-                booster_fdptr_put (in_fh);
-        }
-        
-        return ret;
-}
-
-
-int
-fcntl (int fd, int cmd, ...)
-{
-        va_list           ap;
-        int               ret = -1;
-        long              arg = 0;
-        struct flock     *lock = NULL;
-        glusterfs_file_t  glfs_fd = 0; 
-
-        glfs_fd = booster_fdptr_get (booster_fdtable, fd);
-
-        gf_log ("booster", GF_LOG_TRACE, "fcntl: fd %d, cmd %d", fd, cmd);
-	switch (cmd) {
-	case F_DUPFD:
-                ret = dup (fd);
-                break;
-                /* 
-                 * FIXME: Consider this case when implementing F_DUPFD, F_GETFD
-                 *        etc flags in libglusterfsclient. Commenting it out for
-                 *        timebeing since it is defined only in linux kernel 
-                 *        versions >= 2.6.24.
-                 */
-                /* case F_DUPFD_CLOEXEC: */
-	case F_GETFD:
-                if (glfs_fd != NULL) {
-                        ret = booster_get_close_on_exec (booster_fdtable, fd)
-                                ? FD_CLOEXEC : 0;
-                } else {
-                        if (real_fcntl == NULL) {
-                                ret = -1;
-                                errno = ENOSYS;
-                        } else {
-                                ret = real_fcntl (fd, cmd);
-                        }
-                }
-                break;
-
-	case F_GETFL:
-	case F_GETOWN:
-	case F_GETSIG:
-	case F_GETLEASE:
-                if (glfs_fd) {
-                        gf_log ("booster", GF_LOG_TRACE, "Is a booster fd");
-                        ret = glusterfs_fcntl (glfs_fd, cmd);
-                } else {
-                        if (!real_fcntl) {
-                                errno = ENOSYS;
-                                goto out;
-                        }
-
-                        gf_log ("booster", GF_LOG_TRACE, "Not a booster fd");
-                        ret = real_fcntl (fd, cmd);
-                }
-		break;
-
-	case F_SETFD:
-                if (glfs_fd != NULL) {
-                        booster_set_close_on_exec (booster_fdtable, fd);
-                        ret = 0;
-                } else {
-                        if (real_fcntl == NULL) {
-                                ret = -1;
-                                errno = ENOSYS;
-                        } else {
-                                ret = real_fcntl (fd, cmd);
-                        }
-                }
-                break;
-
-	case F_SETFL:
-	case F_SETOWN:
-	case F_SETSIG:
-	case F_SETLEASE:
-	case F_NOTIFY:
-                va_start (ap, cmd);
-                arg = va_arg (ap, long);
-                va_end (ap);
-
-                if (glfs_fd) {
-                        gf_log ("booster", GF_LOG_TRACE, "Is a booster fd");
-                        ret = glusterfs_fcntl (glfs_fd, cmd, arg);
-                } else {
-                        if (!real_fcntl) {
-                                errno = ENOSYS;
-                                goto out;
-                        }
-
-                        gf_log ("booster", GF_LOG_TRACE, "Not a booster fd");
-                        ret = real_fcntl (fd, cmd, arg);
-                }
-		break;
-
-	case F_GETLK:
-	case F_SETLK:
-	case F_SETLKW:
-#if F_GETLK != F_GETLK64 
-        case F_GETLK64:
-#endif
-#if F_SETLK != F_SETLK64
-        case F_SETLK64:
-#endif
-#if F_SETLKW != F_SETLKW64
-        case F_SETLKW64:
-#endif
-                va_start (ap, cmd);
-                lock = va_arg (ap, struct flock *);
-                va_end (ap);
-
-                if (lock == NULL) {
-                        errno = EINVAL;
-                        goto out;
-                }
-
-                if (glfs_fd) {
-                        gf_log ("booster", GF_LOG_TRACE, "Is a booster fd");
-                        ret = glusterfs_fcntl (glfs_fd, cmd, lock);
-                } else {
-                        if (!real_fcntl) {
-                                errno = ENOSYS;
-                                goto out;
-                        }
-
-                        gf_log ("booster", GF_LOG_TRACE, "Not a booster fd");
-                        ret = real_fcntl (fd, cmd, lock);
-                }
-		break;
-
-	default:
-                errno = EINVAL;
-		break;
-	}
-
-out:
-        if (glfs_fd) {
-                booster_fdptr_put (glfs_fd);
-        }
-
-        return ret;
-}
-
-
-int
-chdir (const char *path)
-{
-        int     ret = -1;
-        char    cwd[PATH_MAX];
-        char   *res = NULL;
-
-        gf_log ("booster", GF_LOG_TRACE, "chdir: path %s", path);
-
-        pthread_mutex_lock (&cwdlock);
-        {
-                res = glusterfs_getcwd (cwd, PATH_MAX);
-                if (res == NULL) {
-                        gf_log ("booster", GF_LOG_ERROR, "getcwd failed: %s",
-                                strerror (errno));
-                        goto unlock;
-                }
-
-                ret = glusterfs_chdir (path);
-                if ((ret == -1) && (errno != ENODEV)) {
-                        gf_log ("booster", GF_LOG_ERROR, "chdir failed: %s",
-                                strerror (errno));
-                        goto unlock;
-                }
-
-                if (ret == 0) {
-                        gf_log ("booster", GF_LOG_TRACE, "chdir succeeded");
-                        goto unlock;
-                }
-
-                if (real_chdir == NULL) {
-                        errno = ENOSYS;
-                        ret = -1;
-                        goto unlock;
-                }
-
-                ret = real_chdir (path);
-                if (ret == -1) {
-                        glusterfs_chdir (cwd);
-                }
-        }
-unlock:
-        pthread_mutex_unlock (&cwdlock);
-
-        return ret;
-}
-
-
-int
-fchdir (int fd)
-{
-        int              ret     = -1;
-        glusterfs_file_t glfs_fd = 0;
-        char             cwd[PATH_MAX]; 
-        char            *res     = NULL;
-
-        glfs_fd = booster_fdptr_get (booster_fdtable, fd);
-
-        if (!glfs_fd) {
-                gf_log ("booster", GF_LOG_TRACE, "Not a booster fd");
-                if (real_write == NULL) {
-                        errno = ENOSYS;
-                        ret = -1;
-                } else {
-                        ret = real_fchdir (fd);
-                        if (ret == 0) {
-                                res = real_getcwd (cwd, PATH_MAX);
-                                if (res == NULL) {
-                                        gf_log ("booster", GF_LOG_ERROR,
-                                                "getcwd failed (%s)",
-                                                strerror (errno));
-                                        ret = -1;
-                                } else {
-                                        glusterfs_chdir (cwd);
-                                }
-                        }
-                }
-        } else {
-                gf_log ("booster", GF_LOG_TRACE, "Is a booster fd");
-                ret = glusterfs_fchdir (glfs_fd);
-                booster_fdptr_put (glfs_fd);
-        }
- 
-        return ret;
-}
-
-
-char *
-getcwd (char *buf, size_t size)
-{
-        char *res = NULL;
-
-        res = glusterfs_getcwd (buf, size);
-        if ((res == NULL) && (errno == ENODEV)) {
-                res = real_getcwd (buf, size);
-        }
-
-        return res;
-}
-
-
-int __REDIRECT (booster_false_truncate, (const char *path, off_t length),
-                truncate) __nonnull ((1));
-int __REDIRECT (booster_false_truncate64, (const char *path, loff_t length),
-                truncate64) __nonnull ((1));;
-
-int
-booster_false_truncate (const char *path, off_t length)
-{
-        int             ret = -1;
-
-        gf_log ("booster", GF_LOG_TRACE, "truncate: path (%s) length (%"PRIu64
-                ")", path, length);
-
-        ret = glusterfs_truncate (path, length);
-        if ((ret == -1) && (errno != ENODEV)) {
-                gf_log ("booster", GF_LOG_ERROR, "truncate failed: %s",
-                        strerror (errno));
-                goto out;
-        }
-
-        if (ret == 0) {
-                gf_log ("booster", GF_LOG_TRACE, "truncate succeeded");
-                goto out;
-        }
-
-        if (real_truncate != NULL)
-                ret = real_truncate (path, length);
-        else {
-                errno = ENOSYS;
-                ret = -1;
-                goto out;
-        }
-
-out:
-        return ret;
-}
-
-
-int
-booster_false_truncate64 (const char *path, loff_t length)
-{
-        int             ret = -1;
-  
-        gf_log ("booster", GF_LOG_TRACE, "truncate64: path (%s) length "
-                "(%"PRIu64")", path, length);
-
-        ret = glusterfs_truncate (path, length);
-        if ((ret == -1) && (errno != ENODEV)) {
-                gf_log ("booster", GF_LOG_ERROR, "truncate64 failed: %s",
-                        strerror (errno));
-                goto out;
-        }
-
-        if (ret == 0) {
-                gf_log ("booster", GF_LOG_TRACE, "truncate64 succeeded");
-                goto out;
-        }
-
-        if (real_truncate64 != NULL)
-                ret = real_truncate64 (path, length);
-        else {
-                errno = ENOSYS;
-                ret = -1;
-                goto out;
-        }
-
-out:
-        return ret;
-}
-
-
-int
-setxattr (const char *path, const char *name, const void *value, size_t size,
-          int flags)
-{
-        int             ret = -1;
-
-        gf_log ("booster", GF_LOG_TRACE, "setxattr: path: %s", path);
-        ret = glusterfs_setxattr (path, name, value, size, flags);
-        if ((ret == -1) && (errno != ENODEV)) {
-                gf_log ("booster", GF_LOG_ERROR, "setxattr failed: %s",
-                        strerror (errno));
-                goto out;
-        }
-
-        if (ret == 0) {
-                gf_log ("booster", GF_LOG_TRACE, "setxattr succeeded");
-                goto out;
-        }
-
-        if (real_setxattr != NULL)
-                ret = real_setxattr (path, name, value, size, flags);
-        else {
-                errno = ENOSYS;
-                ret = -1;
-                goto out;
-        }
-
-out:
-        return ret;
-}
-
-
-int
-lsetxattr (const char *path, const char *name, const void *value, size_t size,
-           int flags)
-{
-        int             ret = -1;
-
-        gf_log ("booster", GF_LOG_TRACE, "lsetxattr: path: %s", path);
-        ret = glusterfs_lsetxattr (path, name, value, size, flags);
-        if ((ret == -1) && (errno != ENODEV)) {
-                gf_log ("booster", GF_LOG_ERROR, "lsetxattr failed: %s",
-                        strerror (errno));
-                goto out;
-        }
-
-        if (ret == 0) {
-                gf_log ("booster", GF_LOG_TRACE, "lsetxattr succeeded");
-                goto out;
-        }
-
-        if (real_lsetxattr != NULL)
-                ret = real_lsetxattr (path, name, value, size, flags);
-        else {
-                errno = ENOSYS;
-                ret = -1;
-                goto out;
-        }
-
-out:
-        return ret;
-}
-
-
-int
-fsetxattr (int fd, const char *name, const void *value, size_t size, int flags)
-{
-        int                     ret = -1;
-        glusterfs_file_t        fh = NULL;
-
-        gf_log ("booster", GF_LOG_TRACE, "fsetxattr: fd %d", fd);
-        fh = booster_fdptr_get (booster_fdtable, fd);
-        if (!fh) {
-                gf_log ("booster", GF_LOG_TRACE, "Not a booster fd");
-                if (real_fsetxattr != NULL)
-                        ret = real_fsetxattr (fd, name, value, size, flags);
-                else {
-                        ret = -1;
-                        errno = ENOSYS;
-                        goto out;
-                }
-        } else {
-                gf_log ("booster", GF_LOG_TRACE, "Is a booster fd");
-                ret = glusterfs_fsetxattr (fh, name, value, size, flags);
-                booster_fdptr_put (fh);
-        }
-
-out:
-        return ret;
-}
-
-
-void
-booster_lib_init (void)
-{
-
-        RESOLVE (open);
-        RESOLVE (open64);
-        RESOLVE (creat);
-        RESOLVE (creat64);
-
-        RESOLVE (read);
-        RESOLVE (readv);
-        RESOLVE (pread);
-        RESOLVE (pread64);
-
-        RESOLVE (write);
-        RESOLVE (writev);
-        RESOLVE (pwrite);
-        RESOLVE (pwrite64);
-
-        RESOLVE (lseek);
-        RESOLVE (lseek64);
-
-        RESOLVE (close);
-
-        RESOLVE (dup);
-        RESOLVE (dup2);
-
-	RESOLVE (fork); 
-        RESOLVE (mkdir);
-        RESOLVE (rmdir);
-        RESOLVE (chmod);
-        RESOLVE (chown);
-        RESOLVE (fchmod);
-        RESOLVE (fchown);
-        RESOLVE (fsync);
-        RESOLVE (ftruncate);
-        RESOLVE (ftruncate64);
-        RESOLVE (link);
-        RESOLVE (rename);
-        RESOLVE (utimes);
-        RESOLVE (utime);
-        RESOLVE (mknod);
-        RESOLVE (mkfifo);
-        RESOLVE (unlink);
-        RESOLVE (symlink);
-        RESOLVE (readlink);
-        RESOLVE (realpath);
-        RESOLVE (opendir);
-        RESOLVE (readdir);
-        RESOLVE (readdir64);
-        RESOLVE (closedir);
-        RESOLVE (__xstat);
-        RESOLVE (__xstat64);
-        RESOLVE (stat);
-        RESOLVE (stat64);
-        RESOLVE (__fxstat);
-        RESOLVE (__fxstat64);
-        RESOLVE (fstat);
-        RESOLVE (fstat64);
-        RESOLVE (__lxstat);
-        RESOLVE (__lxstat64);
-        RESOLVE (lstat);
-        RESOLVE (lstat64);
-        RESOLVE (statfs);
-        RESOLVE (statfs64);
-        RESOLVE (statvfs);
-        RESOLVE (statvfs64);
-        RESOLVE (getxattr);
-        RESOLVE (lgetxattr);
-        RESOLVE (remove);
-        RESOLVE (lchown);
-	RESOLVE (rewinddir);
-	RESOLVE (seekdir);
-	RESOLVE (telldir);
-        RESOLVE (sendfile);
-        RESOLVE (sendfile64);
-        RESOLVE (readdir_r);
-        RESOLVE (readdir64_r);
-        RESOLVE (fcntl);
-        RESOLVE (chdir);
-        RESOLVE (fchdir);
-        RESOLVE (getcwd);
-        RESOLVE (truncate);
-        RESOLVE (truncate64);
-        RESOLVE (setxattr);
-        RESOLVE (lsetxattr);
-        RESOLVE (fsetxattr);
-
-        /* This must be called after resolving real functions
-         * above so that the socket based IO calls in libglusterfsclient
-         * can fall back to a non-NULL real_XXX function pointer.
-         * Calling booster_init before resolving the names above
-         * results in seg-faults because the function symbols above are NULL.
-         */
-	booster_init ();
-}
-
diff --git a/booster/src/booster_fstab.c b/booster/src/booster_fstab.c
deleted file mode 100644
index 202249cad..000000000
--- a/booster/src/booster_fstab.c
+++ /dev/null
@@ -1,452 +0,0 @@
-/* Utilities for reading/writing fstab, mtab, etc.
-   Copyright (C) 1995-2000, 2001, 2002, 2003, 2006
-   Free Software Foundation, Inc.
-   This file is part of the GNU C Library.
-
-   The GNU C Library is free software; you can redistribute it and/or
-   modify it under the terms of the GNU Lesser General Public
-   License as published by the Free Software Foundation; either
-   version 2.1 of the License, or (at your option) any later version.
-
-   The GNU C Library is distributed in the hope that it will be useful,
-   but WITHOUT ANY WARRANTY; without even the implied warranty of
-   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-   Lesser General Public License for more details.
-
-   You should have received a copy of the GNU Lesser General Public
-   License along with the GNU C Library; if not, write to the Free
-   Software Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA
-   02111-1307 USA.  */
-
-#include <alloca.h>
-#include <stdio.h>
-#include <string.h>
-#include <sys/types.h>
-#include "booster_fstab.h"
-#include <stdlib.h>
-#include <libglusterfsclient.h>
-#include <errno.h>
-
-/* The default timeout for inode and stat cache. */
-#define BOOSTER_DEFAULT_ATTR_TIMEO      5 /* In Secs */
-
-/* Prepare to begin reading and/or writing mount table entries from the
-   beginning of FILE.  MODE is as for `fopen'.  */
-glusterfs_fstab_t *
-glusterfs_fstab_init (const char *file, const char *mode)
-{
-        glusterfs_fstab_t *handle = NULL;
-        handle = calloc (1, sizeof (glusterfs_fstab_t));
-        if (!handle) {
-                gf_log ("booster-fstab", GF_LOG_ERROR, "Memory allocation"
-                        " failed");
-                goto out;
-        }
-
-        gf_log ("booster-fstab", GF_LOG_DEBUG, "FSTAB file: %s", file);
-        FILE *result = fopen (file,mode);
-        if (result != NULL) {
-                handle->fp = result;
-        } else {
-                gf_log ("booster-fstab", GF_LOG_ERROR, "FSTAB file open failed:"
-                        " %s", strerror (errno));
-                free (handle);
-                handle = NULL;
-        }
-
-out:
-
-        return handle;
-}
-
-int
-glusterfs_fstab_close (glusterfs_fstab_t *h)
-{
-        if (!h)
-                return -1;
-
-        if (h->fp)
-                fclose (h->fp);
-
-        return 0;
-}
-
-/* Since the values in a line are separated by spaces, a name cannot
-   contain a space.  Therefore some programs encode spaces in names
-   by the strings "\040".  We undo the encoding when reading an entry.
-   The decoding happens in place.  */
-static char *
-decode_name (char *buf)
-{
-        char *rp = buf;
-        char *wp = buf;
-
-        do
-                if (rp[0] == '\\' && rp[1] == '0' && rp[2] == '4'
-                                && rp[3] == '0')
-                {
-                        /* \040 is a SPACE.  */
-                        *wp++ = ' ';
-                        rp += 3;
-                }
-                else if (rp[0] == '\\' && rp[1] == '0' && rp[2] == '1'
-                                && rp[3] == '1')
-                {
-                        /* \011 is a TAB.  */
-                        *wp++ = '\t';
-                        rp += 3;
-                }
-                else if (rp[0] == '\\' && rp[1] == '0' && rp[2] == '1'
-                                && rp[3] == '2')
-                {
-                        /* \012 is a NEWLINE.  */
-                        *wp++ = '\n';
-                        rp += 3;
-                }
-                else if (rp[0] == '\\' && rp[1] == '\\')
-                {
-                        /* We have to escape \\ to be able to represent all
-                         * characters.  */
-                        *wp++ = '\\';
-                        rp += 1;
-                }
-                else if (rp[0] == '\\' && rp[1] == '1' && rp[2] == '3'
-                                && rp[3] == '4')
-                {
-                        /* \134 is also \\.  */
-                        *wp++ = '\\';
-                        rp += 3;
-                }
-                else
-                        *wp++ = *rp;
-        while (*rp++ != '\0');
-
-        return buf;
-}
-
-
-/* Read one mount table entry from STREAM.  Returns a pointer to storage
-   reused on the next call, or null for EOF or error (use feof/ferror to
-   check).  */
-struct glusterfs_mntent *
-__glusterfs_fstab_getent (FILE *stream, struct glusterfs_mntent *mp,
-                          char *buffer, int bufsiz)
-{
-        char *cp;
-        char *head;
-
-        do
-        {
-                char *end_ptr;
-
-                if (fgets (buffer, bufsiz, stream) == NULL)
-                {
-                        return NULL;
-                }
-
-                end_ptr = strchr (buffer, '\n');
-                if (end_ptr != NULL)	/* chop newline */
-                        *end_ptr = '\0';
-                else
-                {
-                        /* Not the whole line was read.  Do it now but forget
-                         * it.  */
-                        char tmp[1024];
-                        while (fgets (tmp, sizeof tmp, stream) != NULL)
-                                if (strchr (tmp, '\n') != NULL)
-                                        break;
-                }
-
-                head = buffer + strspn (buffer, " \t");
-                /* skip empty lines and comment lines:  */
-        }
-        while (head[0] == '\0' || head[0] == '#');
-
-        cp = strsep (&head, " \t");
-        mp->mnt_fsname = cp != NULL ? decode_name (cp) : (char *) "";
-        if (head)
-                head += strspn (head, " \t");
-        cp = strsep (&head, " \t");
-        mp->mnt_dir = cp != NULL ? decode_name (cp) : (char *) "";
-        if (head)
-                head += strspn (head, " \t");
-        cp = strsep (&head, " \t");
-        mp->mnt_type = cp != NULL ? decode_name (cp) : (char *) "";
-        if (head)
-                head += strspn (head, " \t");
-        cp = strsep (&head, " \t");
-        mp->mnt_opts = cp != NULL ? decode_name (cp) : (char *) "";
-        switch (head ? sscanf (head, " %d %d ", &mp->mnt_freq,
-                               &mp->mnt_passno) : 0)
-        {
-                case 0:
-                        mp->mnt_freq = 0;
-                case 1:
-                        mp->mnt_passno = 0;
-                case 2:
-                        break;
-        }
-
-        return mp;
-}
-
-struct glusterfs_mntent *
-glusterfs_fstab_getent (glusterfs_fstab_t *h)
-{
-        if (!h)
-                return NULL;
-
-        if (!h->fp)
-                return NULL;
-
-        return __glusterfs_fstab_getent (h->fp, &h->tmpent, h->buf,
-                                         GF_MNTENT_BUFSIZE);
-}
-
-/* We have to use an encoding for names if they contain spaces or tabs.
-   To be able to represent all characters we also have to escape the
-   backslash itself.  This "function" must be a macro since we use
-   `alloca'.  */
-#define encode_name(name)                                               \
-        do {                                                            \
-                const char *rp = name;		                        \
-                                                                        \
-                while (*rp != '\0')     	                        \
-                        if (*rp == ' ' || *rp == '\t' || *rp == '\\')   \
-                                break;                                  \
-                        else	                                        \
-                                ++rp;                                   \
-                                                                        \
-                if (*rp != '\0')                                        \
-                {                                               \
-                /* In the worst case the length of the string   \
-                 * can increase to four times the current       \
-                 * length.  */				        \
-                        char *wp;				\
-                                                                \
-                        rp = name;				\
-                        name = wp = (char *) alloca (strlen (name) * 4 + 1);                                                                 \
-                                                                \
-                        do {                            \
-                                if (*rp == ' ')		\
-                                {       		\
-                                        *wp++ = '\\';   \
-                                        *wp++ = '0';	\
-                                        *wp++ = '4';	\
-                                        *wp++ = '0';    \
-                                }			\
-                                else if (*rp == '\t')	\
-                                {			\
-                                        *wp++ = '\\';	\
-                                        *wp++ = '0';	\
-                                        *wp++ = '1';	\
-                                        *wp++ = '1';	\
-                                }			\
-                                else if (*rp == '\n')	\
-                                {	                \
-                                        *wp++ = '\\';	\
-                                        *wp++ = '0';	\
-                                        *wp++ = '1';	\
-                                        *wp++ = '2';	\
-                                }	                \
-                                else if (*rp == '\\')	\
-                                {                       \
-                                        *wp++ = '\\';	\
-                                        *wp++ = '\\';	\
-                                }                       \
-                                else	                \
-                                        *wp++ = *rp;	\
-                        } while (*rp++ != '\0');	\
-                }                                       \
-        } while (0)                                     \
-
-
-int
-glusterfs_fstab_addent (glusterfs_fstab_t *h,
-                const struct glusterfs_mntent *mnt)
-{
-        struct glusterfs_mntent mntcopy = *mnt;
-        if (!h)
-                return -1;
-
-        if (!h->fp)
-                return -1;
-
-        if (fseek (h->fp, 0, SEEK_END))
-                return -1;
-
-        /* Encode spaces and tabs in the names.  */
-        encode_name (mntcopy.mnt_fsname);
-        encode_name (mntcopy.mnt_dir);
-        encode_name (mntcopy.mnt_type);
-        encode_name (mntcopy.mnt_opts);
-
-        return (fprintf (h->fp, "%s %s %s %s %d %d\n",
-                                mntcopy.mnt_fsname,
-                                mntcopy.mnt_dir,
-                                mntcopy.mnt_type,
-                                mntcopy.mnt_opts,
-                                mntcopy.mnt_freq,
-                                mntcopy.mnt_passno)
-                        < 0 ? 1 : 0);
-}
-
-
-/* Search MNT->mnt_opts for an option matching OPT.
-   Returns the address of the substring, or null if none found.  */
-char *
-glusterfs_fstab_hasoption (const struct glusterfs_mntent *mnt, const char *opt)
-{
-        const size_t optlen = strlen (opt);
-        char *rest = mnt->mnt_opts, *p;
-
-        while ((p = strstr (rest, opt)) != NULL)
-        {
-                if ((p == rest || p[-1] == ',')
-                                && (p[optlen] == '\0' || p[optlen] == '=' || p[optlen] == ','))
-                        return p;
-
-                rest = strchr (p, ',');
-                if (rest == NULL)
-                        break;
-                ++rest;
-        }
-
-        return NULL;
-}
-
-void
-clean_init_params (glusterfs_init_params_t *ipars)
-{
-        if (!ipars)
-                return;
-
-        if (ipars->volume_name)
-                free (ipars->volume_name);
-
-        if (ipars->specfile)
-                free (ipars->specfile);
-
-        if (ipars->logfile)
-                free (ipars->logfile);
-
-        if (ipars->loglevel)
-                free (ipars->loglevel);
-
-        return;
-}
-
-char *
-get_option_value (char *opt)
-{
-        char *val = NULL;
-        char *saveptr = NULL;
-        char *copy_opt = NULL;
-        char *retval = NULL;
-
-        copy_opt = strdup (opt);
-
-        /* Get the = before the value of the option. */
-        val = index (copy_opt, '=');
-        if (val) {
-                /* Move to start of option */
-                ++val;
-
-                /* Now, to create a '\0' delimited string out of the
-                 * options string, first get the position where the
-                 * next option starts, that would be the next ','.
-                 */
-                saveptr = index (val, ',');
-                if (saveptr)
-                        *saveptr = '\0';
-                retval = strdup (val);
-        }
-
-        free (copy_opt);
-
-        return retval;
-}
-
-void
-booster_mount (struct glusterfs_mntent *ent)
-{
-        char                    *opt = NULL;
-        glusterfs_init_params_t ipars;
-        time_t                  timeout = BOOSTER_DEFAULT_ATTR_TIMEO;
-        char                    *timeostr = NULL;
-        char                    *endptr = NULL;
-
-        if (!ent)
-                return;
-
-        gf_log ("booster-fstab", GF_LOG_DEBUG, "Mount entry: volfile: %s,"
-                " VMP: %s, Type: %s, Options: %s", ent->mnt_fsname,
-                ent->mnt_dir, ent->mnt_type, ent->mnt_opts);
-        if ((strcmp (ent->mnt_type, "glusterfs") != 0)) {
-                gf_log ("booster-fstab", GF_LOG_ERROR, "Type is not glusterfs");
-                return;
-        }
-
-        memset (&ipars, 0, sizeof (glusterfs_init_params_t));
-        if (ent->mnt_fsname)
-                ipars.specfile = strdup (ent->mnt_fsname);
-
-        opt = glusterfs_fstab_hasoption (ent, "subvolume");
-        if (opt)
-                ipars.volume_name = get_option_value (opt);
-
-        opt = glusterfs_fstab_hasoption (ent, "log-file");
-        if (!opt)
-                opt = glusterfs_fstab_hasoption (ent, "logfile");
-
-        if (opt)
-                ipars.logfile = get_option_value (opt);
-
-        opt = glusterfs_fstab_hasoption (ent, "log-level");
-        if (!opt)
-                opt = glusterfs_fstab_hasoption (ent, "loglevel");
-
-        if (opt)
-                ipars.loglevel = get_option_value (opt);
-
-        /* Attribute cache timeout */
-        opt = glusterfs_fstab_hasoption (ent, "attr_timeout");
-        if (opt) {
-                 timeostr = get_option_value (opt);
-                 if (timeostr)
-                         timeout = strtol (timeostr, &endptr, 10);
-        }
-
-        ipars.lookup_timeout = timeout;
-        ipars.stat_timeout = timeout;
-
-        if ((glusterfs_mount (ent->mnt_dir, &ipars)) == -1)
-                gf_log ("booster-fstab", GF_LOG_ERROR, "VMP mounting failed");
-
-        clean_init_params (&ipars);
-}
-
-int
-booster_configure (char *confpath)
-{
-        int                     ret = -1;
-        glusterfs_fstab_t       *handle = NULL;
-        struct glusterfs_mntent *ent = NULL;
-
-        if (!confpath)
-                goto out;
-
-        handle = glusterfs_fstab_init (confpath, "r");
-        if (!handle)
-                goto out;
-
-        while ((ent = glusterfs_fstab_getent (handle)) != NULL)
-                booster_mount (ent);
-
-        glusterfs_fstab_close (handle);
-        ret = 0;
-out:
-        return ret;
-}
-
-
diff --git a/booster/src/booster_fstab.h b/booster/src/booster_fstab.h
deleted file mode 100644
index 9bab04c5a..000000000
--- a/booster/src/booster_fstab.h
+++ /dev/null
@@ -1,83 +0,0 @@
-/* Utilities for reading/writing fstab, mtab, etc.
-   Copyright (C) 1995, 1996, 1997, 1998, 1999 Free Software Foundation, Inc.
-   This file is part of the GNU C Library.
-
-   The GNU C Library is free software; you can redistribute it and/or
-   modify it under the terms of the GNU Lesser General Public
-   License as published by the Free Software Foundation; either
-   version 2.1 of the License, or (at your option) any later version.
-
-   The GNU C Library is distributed in the hope that it will be useful,
-   but WITHOUT ANY WARRANTY; without even the implied warranty of
-   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-   Lesser General Public License for more details.
-
-   You should have received a copy of the GNU Lesser General Public
-   License along with the GNU C Library; if not, write to the Free
-   Software Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA
-   02111-1307 USA.  */
-
-#ifndef	GLUSTERFS_FSTAB_MNTENT_H
-#define	GLUSTERFS_FSTAB_MNTENT_H	1
-#ifndef _CONFIG_H
-#define _CONFIG_H
-#include "config.h"
-#endif
- 
-#include "compat.h"
-
-/* General filesystem types.  */
-#define GF_MNTTYPE_IGNORE	"ignore"	/* Ignore this entry.  */
-#define GF_MNTTYPE_NFS	"nfs"		/* Network file system.  */
-#define GF_MNTTYPE_SWAP	"swap"		/* Swap device.  */
-
-
-/* Generic mount options.  */
-#define GF_MNTOPT_DEFAULTS	"defaults"	/* Use all default options.  */
-#define GF_MNTOPT_RO	        "ro"		/* Read only.  */
-#define GF_MNTOPT_RW	        "rw"		/* Read/write.  */
-#define GF_MNTOPT_SUID	        "suid"		/* Set uid allowed.  */
-#define GF_MNTOPT_NOSUID	"nosuid"	/* No set uid allowed.  */
-#define GF_MNTOPT_NOAUTO	"noauto"	/* Do not auto mount.  */
-
-
-/* Structure describing a mount table entry.  */
-struct glusterfs_mntent
-{
-        char *mnt_fsname;		/* Device or server for filesystem.  */
-        char *mnt_dir;		/* Directory mounted on.  */
-        char *mnt_type;		/* Type of filesystem: ufs, nfs, etc.  */
-        char *mnt_opts;		/* Comma-separated options for fs.  */
-        int mnt_freq;		/* Dump frequency (in days).  */
-        int mnt_passno;		/* Pass number for `fsck'.  */
-};
-
-#define GF_MNTENT_BUFSIZE       1024
-typedef struct glusterfs_fstab_handle {
-        FILE *fp;
-        char buf[GF_MNTENT_BUFSIZE];
-        struct glusterfs_mntent tmpent;
-}glusterfs_fstab_t;
-
-
-/* Prepare to begin reading and/or writing mount table entries from the
-   beginning of FILE.  MODE is as for `fopen'.  */
-extern glusterfs_fstab_t *glusterfs_fstab_init (const char *file,
-                const char *mode);
-
-extern struct glusterfs_mntent *glusterfs_fstab_getent (glusterfs_fstab_t *h);
-
-/* Write the mount table entry described by MNT to STREAM.
-   Return zero on success, nonzero on failure.  */
-extern int glusterfs_fstab_addent (glusterfs_fstab_t *h,
-                const struct glusterfs_mntent *mnt);
-
-/* Close a stream opened with `glusterfs_fstab_init'.  */
-extern int glusterfs_fstab_close (glusterfs_fstab_t *h);
-
-/* Search MNT->mnt_opts for an option matching OPT.
-   Returns the address of the substring, or null if none found.  */
-extern char *glusterfs_fstab_hasoption (const struct glusterfs_mntent *mnt,
-                const char *opt);
-
-#endif
diff --git a/booster/src/booster_stat.c b/booster/src/booster_stat.c
deleted file mode 100644
index 8f76cfe37..000000000
--- a/booster/src/booster_stat.c
+++ /dev/null
@@ -1,188 +0,0 @@
-/*
-   Copyright (c) 2007-2011 Gluster, Inc. <http://www.gluster.com>
-   This file is part of GlusterFS.
-
-   GlusterFS is free software; you can redistribute it and/or modify
-   it under the terms of the GNU General Public License as published
-   by the Free Software Foundation; either version 3 of the License,
-   or (at your option) any later version.
-
-   GlusterFS is distributed in the hope that it will be useful, but
-   WITHOUT ANY WARRANTY; without even the implied warranty of
-   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-   General Public License for more details.
-
-   You should have received a copy of the GNU General Public License
-   along with this program.  If not, see
-   <http://www.gnu.org/licenses/>.
-*/
-
-#include <sys/types.h>
-
-extern int
-booster_stat (const char *path, void *buf);
-
-extern int
-booster_stat64 (const char *path, void *buf);
-
-extern int
-booster_xstat (int ver, const char *path, void *buf);
-
-extern int
-booster_xstat64 (int ver, const char *path, void *buf);
-
-extern int
-booster_fxstat (int ver, int fd, void *buf);
-extern int
-booster_fxstat64 (int ver, int fd, void *buf);
-extern int
-booster_fstat (int fd, void *buf);
-extern int
-booster_fstat64 (int fd, void *buf);
-
-extern int
-booster_lstat (const char *path, void *buf);
-extern int
-booster_lstat64 (const char *path, void *buf);
-extern int
-booster_lxstat (int ver, const char *path, void *buf);
-extern int
-booster_lxstat64 (int ver, const char *path, void *buf);
-
-
-extern int
-booster_statfs (const char *path, void *buf);
-extern int
-booster_statfs64 (const char *path, void *buf);
-
-extern int
-booster_statvfs (const char *path, void *buf);
-
-extern int
-booster_statvfs64 (const char *path, void *buf);
-
-extern void *
-booster_readdir (void *dir);
-
-extern void
-booster_rewinddir (void *dir);
-
-extern void
-booster_seekdir (void *dir, off_t offset);
-
-extern off_t
-booster_telldir (void *dir);
-
-int
-stat (const char *path, void *buf)
-{
-        return booster_stat (path, buf);
-}
-
-int
-stat64 (const char *path, void *buf)
-{
-        return booster_stat64 (path, buf);
-}
-
-int
-__xstat (int ver, const char *path, void *buf)
-{
-        return booster_xstat (ver, path, buf);
-}
-
-int
-__xstat64 (int ver, const char *path, void *buf)
-{
-        return booster_xstat64 (ver, path, buf);
-}
-
-int
-__fxstat (int ver, int fd, void *buf)
-{
-        return booster_fxstat (ver, fd, buf);
-}
-
-int
-__fxstat64 (int ver, int fd, void *buf)
-{
-        return booster_fxstat64 (ver, fd, buf);
-}
-
-int
-fstat (int fd, void *buf)
-{
-        return booster_fstat (fd, buf);
-}
-
-int
-fstat64 (int fd, void *buf)
-{
-        return booster_fstat64 (fd, buf);
-}
-
-int
-lstat (const char *path, void *buf)
-{
-        return booster_lstat (path, buf);
-}
-
-int
-lstat64 (const char *path, void *buf)
-{
-        return booster_lstat64 (path, buf);
-}
-
-int
-__lxstat (int ver, const char *path, void *buf)
-{
-        return booster_lxstat (ver, path, buf);
-}
-
-int
-__lxstat64 (int ver, const char *path, void *buf)
-{
-        return booster_lxstat64 (ver, path, buf);
-}
-
-int
-statfs (const char *pathname, void *buf)
-{
-        return booster_statfs (pathname, buf);
-}
-
-int
-statfs64 (const char *pathname, void *buf)
-{
-        return booster_statfs64 (pathname, buf);
-}
-
-int
-statvfs (const char *pathname, void *buf)
-{
-        return booster_statvfs (pathname, buf);
-}
-
-int
-statvfs64 (const char *pathname, void *buf)
-{
-        return booster_statvfs64 (pathname, buf);
-}
-
-void
-rewinddir (void *dir)
-{
-	return booster_rewinddir (dir);
-}
-
-void
-seekdir (void *dir, off_t offset)
-{
-	return booster_seekdir (dir, offset);
-}
-
-off_t
-telldir (void *dir)
-{
-	return booster_telldir (dir);
-}
diff --git a/cli/src/Makefile.am b/cli/src/Makefile.am
index 800283618..216d1bb55 100644
--- a/cli/src/Makefile.am
+++ b/cli/src/Makefile.am
@@ -2,23 +2,26 @@ sbin_PROGRAMS = gluster
 
 gluster_SOURCES = cli.c registry.c input.c cli-cmd.c cli-rl.c \
 	 cli-cmd-volume.c cli-cmd-peer.c cli-rpc-ops.c cli-cmd-parser.c\
-	 cli-cmd-system.c cli-cmd-misc.c cli-xml-output.c
+	 cli-cmd-system.c cli-cmd-misc.c cli-xml-output.c cli-cmd-snapshot.c
 
 gluster_LDADD = $(top_builddir)/libglusterfs/src/libglusterfs.la $(GF_LDADD)\
 		$(RLLIBS) $(top_builddir)/rpc/xdr/src/libgfxdr.la \
-		$(top_builddir)/rpc/rpc-lib/src/libgfrpc.la
+		$(top_builddir)/rpc/rpc-lib/src/libgfrpc.la \
+		$(GF_GLUSTERFS_LIBS) $(XML_LIBS)
 
-gluster_LDFLAGS = $(GF_LDFLAGS) $(GF_GLUSTERFS_LDFLAGS) $(LIBXML2_LIBS)
+gluster_LDFLAGS = $(GF_LDFLAGS)
 noinst_HEADERS = cli.h cli-mem-types.h cli-cmd.h
 
-AM_CFLAGS = -fPIC -Wall -D_FILE_OFFSET_BITS=64 -D_GNU_SOURCE -D$(GF_HOST_OS)\
+AM_CPPFLAGS = $(GF_CPPFLAGS) \
 	-I$(top_srcdir)/libglusterfs/src -I$(top_srcdir)/rpc/rpc-lib/src\
 	-I$(top_srcdir)/rpc/xdr/src\
 	-DDATADIR=\"$(localstatedir)\" \
-	-DCONFDIR=\"$(sysconfdir)/glusterfs\" $(GF_GLUSTERFS_CFLAGS)\
+	-DCONFDIR=\"$(sysconfdir)/glusterfs\" \
 	-DGSYNCD_PREFIX=\"$(libexecdir)/glusterfs\"\
 	-DSYNCDAEMON_COMPILE=$(SYNCDAEMON_COMPILE) -DSBIN_DIR=\"$(sbindir)\"\
-	$(LIBXML2_CFLAGS)
+	$(XML_CPPFLAGS)
+
+AM_CFLAGS = -Wall $(GF_GLUSTERFS_CFLAGS)
 
 CLEANFILES =
 
diff --git a/cli/src/cli-cmd-misc.c b/cli/src/cli-cmd-misc.c
index 66d755fc2..566d7c978 100644
--- a/cli/src/cli-cmd-misc.c
+++ b/cli/src/cli-cmd-misc.c
@@ -1,22 +1,12 @@
 /*
-  Copyright (c) 2010-2011 Gluster, Inc. <http://www.gluster.com>
-  This file is part of GlusterFS.
-
-  GlusterFS is free software; you can redistribute it and/or modify
-  it under the terms of the GNU General Public License as published
-  by the Free Software Foundation; either version 3 of the License,
-  or (at your option) any later version.
-
-  GlusterFS is distributed in the hope that it will be useful, but
-  WITHOUT ANY WARRANTY; without even the implied warranty of
-  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-  General Public License for more details.
-
-  You should have received a copy of the GNU General Public License
-  along with this program.  If not, see
-  <http://www.gnu.org/licenses/>.
-*/
+   Copyright (c) 2010-2012 Red Hat, Inc. <http://www.redhat.com>
+   This file is part of GlusterFS.
 
+   This file is licensed to you under your choice of the GNU Lesser
+   General Public License, version 3 or any later version (LGPLv3 or
+   later), or the GNU General Public License, version 2 (GPLv2), in all
+   cases as published by the Free Software Foundation.
+*/
 #include <stdio.h>
 #include <string.h>
 #include <stdlib.h>
@@ -41,6 +31,8 @@ extern struct cli_cmd volume_cmds[];
 extern struct cli_cmd cli_probe_cmds[];
 extern struct cli_cmd cli_log_cmds[];
 extern struct cli_cmd cli_system_cmds[];
+extern struct cli_cmd cli_bd_cmds[];
+extern struct cli_cmd snapshot_cmds[];
 struct cli_cmd cli_misc_cmds[];
 
 int
@@ -55,7 +47,8 @@ cli_cmd_display_help (struct cli_state *state, struct cli_cmd_word *in_word,
                       const char **words, int wordcount)
 {
         struct cli_cmd        *cmd[] = {volume_cmds, cli_probe_cmds,
-                                       cli_misc_cmds, NULL};
+                                       cli_misc_cmds, snapshot_cmds,
+                                       NULL};
         struct cli_cmd        *cmd_ind = NULL;
         int                   i = 0;
 
diff --git a/cli/src/cli-cmd-parser.c b/cli/src/cli-cmd-parser.c
index d41493b6c..5ab208b8f 100644
--- a/cli/src/cli-cmd-parser.c
+++ b/cli/src/cli-cmd-parser.c
@@ -1,22 +1,12 @@
 /*
-  Copyright (c) 2010-2011 Gluster, Inc. <http://www.gluster.com>
-  This file is part of GlusterFS.
-
-  GlusterFS is free software; you can redistribute it and/or modify
-  it under the terms of the GNU General Public License as published
-  by the Free Software Foundation; either version 3 of the License,
-  or (at your option) any later version.
-
-  GlusterFS is distributed in the hope that it will be useful, but
-  WITHOUT ANY WARRANTY; without even the implied warranty of
-  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-  General Public License for more details.
-
-  You should have received a copy of the GNU General Public License
-  along with this program.  If not, see
-  <http://www.gnu.org/licenses/>.
-*/
+   Copyright (c) 2010-2013 Red Hat, Inc. <http://www.redhat.com>
+   This file is part of GlusterFS.
 
+   This file is licensed to you under your choice of the GNU Lesser
+   General Public License, version 3 or any later version (LGPLv3 or
+   later), or the GNU General Public License, version 2 (GPLv2), in all
+   cases as published by the Free Software Foundation.
+*/
 #include <stdio.h>
 #include <string.h>
 #include <stdlib.h>
@@ -36,6 +26,38 @@
 #include "protocol-common.h"
 #include "cli1-xdr.h"
 
+#define MAX_SNAP_DESCRIPTION_LEN 1024
+
+struct snap_config_opt_vals_ snap_confopt_vals[] = {
+        {.op_name        = "snap-max-hard-limit",
+         .question       = "Changing snapshot-max-hard-limit "
+                           "will lead to deletion of snapshots "
+                           "if they exceed the new limit.\n"
+                           "Do you want to continue?"
+        },
+        {.op_name        = "snap-max-soft-limit",
+         .question       = "Changing snapshot-max-soft-limit "
+                           "will lead to deletion of snapshots "
+                           "if they exceed the new limit.\n"
+                           "Do you want to continue?"
+        },
+        {.op_name        = "both",
+        .question        = "Changing snapshot-max-hard-limit & "
+                           "snapshot-max-soft-limit will lead to "
+                           "deletion of snapshots if they exceed "
+                           "the new limit.\nDo you want to continue?"
+        },
+        {.op_name        = NULL,
+        }
+};
+
+enum cli_snap_config_set_types {
+        GF_SNAP_CONFIG_SET_HARD = 0,
+        GF_SNAP_CONFIG_SET_SOFT = 1,
+        GF_SNAP_CONFIG_SET_BOTH = 2,
+};
+typedef enum cli_snap_config_set_types cli_snap_config_set_types;
+
 static const char *
 id_sel (void *wcon)
 {
@@ -74,21 +96,20 @@ cli_cmd_bricks_parse (const char **words, int wordcount, int brick_index,
         brick_list_len++;
         while (brick_index < wordcount) {
                 if (validate_brick_name ((char *)words[brick_index])) {
-                        cli_out ("Wrong brick type: %s, use <HOSTNAME>:"
+                        cli_err ("Wrong brick type: %s, use <HOSTNAME>:"
                                  "<export-dir-abs-path>", words[brick_index]);
                         ret = -1;
                         goto out;
                 } else {
                         delimiter = strrchr (words[brick_index], ':');
-                        ret = cli_canonicalize_path (delimiter + 1);
+                        ret = gf_canonicalize_path (delimiter + 1);
                         if (ret)
                                 goto out;
                 }
 
                 if ((brick_list_len + strlen (words[brick_index]) + 1) > sizeof (brick_list)) {
-                        gf_log ("cli", GF_LOG_ERROR,
-                                "total brick list is larger than a request "
-                                "can take (brick_count %d)", *brick_count);
+                        cli_err ("Total brick list is larger than a request. "
+                                 "Can take (brick_count %d)", *brick_count);
                         ret = -1;
                         goto out;
                 }
@@ -108,16 +129,18 @@ cli_cmd_bricks_parse (const char **words, int wordcount, int brick_index,
                 }
 
                 if (!(strcmp (host_name, "localhost") &&
-                      strcmp (host_name, "127.0.0.1"))) {
-                        cli_out ("Please provide a valid hostname/ip other "
-                                 "than localhost or 127.0.0.1");
+                      strcmp (host_name, "127.0.0.1") &&
+                      strncmp (host_name, "0.", 2))) {
+                        cli_err ("Please provide a valid hostname/ip other "
+                                 "than localhost, 127.0.0.1 or loopback "
+                                 "address (0.0.0.0 to 0.255.255.255).");
                         ret = -1;
                         GF_FREE (tmp_host);
                         goto out;
                 }
-                if (!valid_internet_address (host_name)) {
-                        cli_out ("internet address '%s' does not comform to "
-			         "standards", host_name);
+                if (!valid_internet_address (host_name, _gf_false)) {
+                        cli_err ("internet address '%s' does not conform to "
+                                 "standards", host_name);
                 }
                 GF_FREE (tmp_host);
                 tmp_list = gf_strdup (brick_list + 1);
@@ -131,7 +154,7 @@ cli_cmd_bricks_parse (const char **words, int wordcount, int brick_index,
                         strtok_r (tmp_list, " ", &tmpptr);
                         if (!(strcmp (tmp_list, words[brick_index]))) {
                                 ret = -1;
-                                cli_out ("Found duplicate"
+                                cli_err ("Found duplicate"
                                          " exports %s",words[brick_index]);
                                 goto out;
                        }
@@ -149,8 +172,7 @@ cli_cmd_bricks_parse (const char **words, int wordcount, int brick_index,
         if (!*bricks)
                 ret = -1;
 out:
-        if (free_list_ptr)
-                GF_FREE (free_list_ptr);
+        GF_FREE (free_list_ptr);
         return ret;
 }
 
@@ -170,10 +192,18 @@ cli_cmd_volume_create_parse (const char **words, int wordcount, dict_t **options
         char    *bricks = NULL;
         int32_t brick_count = 0;
         char    *opwords[] = { "replica", "stripe", "transport", NULL };
+
+        char    *invalid_volnames[] = {"volume", "type", "subvolumes", "option",
+                                       "end-volume", "all", "volume_not_in_ring",
+                                       "description", "force",
+                                       "snap-max-hard-limit",
+                                       "snap-max-soft-limit", NULL};
         char    *w = NULL;
         int      op_count = 0;
         int32_t  replica_count = 1;
         int32_t  stripe_count = 1;
+        gf_boolean_t is_force = _gf_false;
+        int wc = wordcount;
 
         GF_ASSERT (words);
         GF_ASSERT (options);
@@ -195,9 +225,12 @@ cli_cmd_volume_create_parse (const char **words, int wordcount, dict_t **options
                 if (volname[0] == '-')
                         goto out;
 
-                if (!strcmp (volname, "all")) {
-                        cli_out ("\"all\" cannot be the name of a volume.");
-                        goto out;
+                for (i = 0; invalid_volnames[i]; i++) {
+                        if (!strcmp (volname, invalid_volnames[i])) {
+                                cli_err ("\"%s\" cannot be the name of a volume.",
+                                         volname);
+                                goto out;
+                        }
                 }
 
                 if (strchr (volname, '/'))
@@ -228,7 +261,7 @@ cli_cmd_volume_create_parse (const char **words, int wordcount, dict_t **options
                         switch (type) {
                         case GF_CLUSTER_TYPE_STRIPE_REPLICATE:
                         case GF_CLUSTER_TYPE_REPLICATE:
-                                cli_out ("replica option given twice");
+                                cli_err ("replica option given twice");
                                 goto out;
                         case GF_CLUSTER_TYPE_NONE:
                                 type = GF_CLUSTER_TYPE_REPLICATE;
@@ -244,7 +277,8 @@ cli_cmd_volume_create_parse (const char **words, int wordcount, dict_t **options
                         }
                         replica_count = strtol (words[index+1], NULL, 0);
                         if (replica_count < 2) {
-                                cli_out ("replica count should be greater than 1");
+                                cli_err ("replica count should be greater"
+                                         " than 1");
                                 ret = -1;
                                 goto out;
                         }
@@ -258,7 +292,7 @@ cli_cmd_volume_create_parse (const char **words, int wordcount, dict_t **options
                         switch (type) {
                         case GF_CLUSTER_TYPE_STRIPE_REPLICATE:
                         case GF_CLUSTER_TYPE_STRIPE:
-                                cli_out ("stripe option given twice");
+                                cli_err ("stripe option given twice");
                                 goto out;
                         case GF_CLUSTER_TYPE_NONE:
                                 type = GF_CLUSTER_TYPE_STRIPE;
@@ -273,7 +307,8 @@ cli_cmd_volume_create_parse (const char **words, int wordcount, dict_t **options
                         }
                         stripe_count = strtol (words[index+1], NULL, 0);
                         if (stripe_count < 2) {
-                                cli_out ("stripe count should be greater than 1");
+                                cli_err ("stripe count should be greater"
+                                         " than 1");
                                 ret = -1;
                                 goto out;
                         }
@@ -285,7 +320,7 @@ cli_cmd_volume_create_parse (const char **words, int wordcount, dict_t **options
 
                 } else if ((strcmp (w, "transport")) == 0) {
                         if (trans_type) {
-                                cli_out ("'transport' option given more"
+                                cli_err ("'transport' option given more"
                                          " than one time");
                                 goto out;
                         }
@@ -303,7 +338,7 @@ cli_cmd_volume_create_parse (const char **words, int wordcount, dict_t **options
                                 goto out;
                         }
                         index += 2;
-                } else {
+                }              else {
                         GF_ASSERT (!"opword mismatch");
                         ret = -1;
                         goto out;
@@ -326,7 +361,12 @@ cli_cmd_volume_create_parse (const char **words, int wordcount, dict_t **options
 
         brick_index = index;
 
-        ret = cli_cmd_bricks_parse (words, wordcount, brick_index, &bricks,
+        if (strcmp (words[wordcount - 1], "force") == 0) {
+                is_force = _gf_true;
+                wc = wordcount - 1;
+        }
+
+        ret = cli_cmd_bricks_parse (words, wc, brick_index, &bricks,
                                     &brick_count);
         if (ret)
                 goto out;
@@ -334,20 +374,20 @@ cli_cmd_volume_create_parse (const char **words, int wordcount, dict_t **options
         /* If brick-count is not valid when replica or stripe is
            given, exit here */
         if (!brick_count) {
-                cli_out ("No bricks specified");
+                cli_err ("No bricks specified");
                 ret = -1;
                 goto out;
         }
 
         if (brick_count % sub_count) {
                 if (type == GF_CLUSTER_TYPE_STRIPE)
-                        cli_out ("number of bricks is not a multiple of "
+                        cli_err ("number of bricks is not a multiple of "
                                  "stripe count");
                 else if (type == GF_CLUSTER_TYPE_REPLICATE)
-                        cli_out ("number of bricks is not a multiple of "
+                        cli_err ("number of bricks is not a multiple of "
                                  "replica count");
                 else
-                        cli_out ("number of bricks given doesn't match "
+                        cli_err ("number of bricks given doesn't match "
                                  "required count");
 
                 ret = -1;
@@ -366,7 +406,7 @@ cli_cmd_volume_create_parse (const char **words, int wordcount, dict_t **options
         ret = dict_set_dynstr (dict, "transport", trans_type);
         if (ret)
                 goto out;
-
+        trans_type = NULL;
 
         ret = dict_set_dynstr (dict, "bricks", bricks);
         if (ret)
@@ -376,6 +416,10 @@ cli_cmd_volume_create_parse (const char **words, int wordcount, dict_t **options
         if (ret)
                 goto out;
 
+        ret = dict_set_int32 (dict, "force", is_force);
+        if (ret)
+                goto out;
+
         *options = dict;
 
 out:
@@ -384,6 +428,9 @@ out:
                 if (dict)
                         dict_destroy (dict);
         }
+
+        GF_FREE (trans_type);
+
         return ret;
 }
 
@@ -497,7 +544,7 @@ cli_cmd_quota_parse (const char **words, int wordcount, dict_t **options)
                         goto out;
 
                 if (!strcmp (volname, "all")) {
-                        cli_out ("\"all\" cannot be the name of a volume.");
+                        cli_err ("\"all\" cannot be the name of a volume.");
                         goto out;
                 }
 
@@ -522,16 +569,26 @@ cli_cmd_quota_parse (const char **words, int wordcount, dict_t **options)
                 goto out;
         }
 
-        if ((strcmp (w, "enable")) == 0 && wordcount == 4) {
-                type = GF_QUOTA_OPTION_TYPE_ENABLE;
-                ret = 0;
-                goto set_type;
+        if (strcmp (w, "enable") == 0) {
+                if (wordcount == 4) {
+                        type = GF_QUOTA_OPTION_TYPE_ENABLE;
+                        ret = 0;
+                        goto set_type;
+                } else {
+                        ret = -1;
+                        goto out;
+                }
         }
 
-        if (strcmp (w, "disable") == 0 && wordcount == 4) {
-                type = GF_QUOTA_OPTION_TYPE_DISABLE;
-                ret = 0;
-                goto set_type;
+        if (strcmp (w, "disable") == 0) {
+                if (wordcount == 4) {
+                        type = GF_QUOTA_OPTION_TYPE_DISABLE;
+                        ret = 0;
+                        goto set_type;
+                } else {
+                        ret = -1;
+                        goto out;
+                }
         }
 
         if (strcmp (w, "limit-usage") == 0) {
@@ -543,7 +600,7 @@ cli_cmd_quota_parse (const char **words, int wordcount, dict_t **options)
                 type = GF_QUOTA_OPTION_TYPE_LIMIT_USAGE;
 
                 if (words[4][0] != '/') {
-                        cli_out ("Please enter absolute path");
+                        cli_err ("Please enter absolute path");
 
                         return -2;
                 }
@@ -552,14 +609,14 @@ cli_cmd_quota_parse (const char **words, int wordcount, dict_t **options)
                         goto out;
 
                 if (!words[5]) {
-                        cli_out ("Please enter the limit value to be set");
+                        cli_err ("Please enter the limit value to be set");
 
                         return -2;
                 }
 
                 ret = gf_string2bytesize (words[5], &value);
                 if (ret != 0) {
-                        cli_out ("Please enter a correct value");
+                        cli_err ("Please enter a correct value");
                         return -1;
                 }
 
@@ -578,7 +635,7 @@ cli_cmd_quota_parse (const char **words, int wordcount, dict_t **options)
                 type = GF_QUOTA_OPTION_TYPE_REMOVE;
 
                 if (words[4][0] != '/') {
-                        cli_out ("Please enter absolute path");
+                        cli_err ("Please enter absolute path");
 
                         return -2;
                 }
@@ -634,54 +691,117 @@ out:
         return ret;
 }
 
-int32_t
-cli_cmd_valid_ip_list (char *iplist)
+static inline gf_boolean_t
+cli_is_key_spl (char *key)
 {
-        int     ret = 0;
-        char    *duplist = NULL;
-        char    *addr = NULL;
-        char    *saveptr = NULL;
-
-        GF_ASSERT (iplist);
-        duplist = gf_strdup (iplist);
+        return (strcmp (key, "group") == 0);
+}
 
-        if (!duplist) {
-                ret = -1;
+#define GLUSTERD_DEFAULT_WORKDIR "/var/lib/glusterd"
+static int
+cli_add_key_group (dict_t *dict, char *key, char *value, char **op_errstr)
+{
+        int             ret = -1;
+        int             opt_count = 0;
+        char            iter_key[1024] = {0,};
+        char            iter_val[1024] = {0,};
+        char            *saveptr = NULL;
+        char            *tok_key = NULL;
+        char            *tok_val = NULL;
+        char            *dkey = NULL;
+        char            *dval = NULL;
+        char            *tagpath = NULL;
+        char            *buf = NULL;
+        char            line[PATH_MAX + 256] = {0,};
+        char            errstr[2048] = "";
+        FILE            *fp = NULL;
+
+        ret = gf_asprintf (&tagpath, "%s/groups/%s",
+                           GLUSTERD_DEFAULT_WORKDIR, value);
+        if (ret == -1) {
+                tagpath = NULL;
                 goto out;
         }
 
-        addr = strtok_r (duplist, ",", &saveptr);
-        if (!addr) {
+        fp = fopen (tagpath, "r");
+        if (!fp) {
                 ret = -1;
+                snprintf(errstr, sizeof(errstr), "Unable to open file '%s'."
+                         " Error: %s", tagpath, strerror (errno));
+                if (op_errstr)
+                        *op_errstr = gf_strdup(errstr);
                 goto out;
         }
-        while (addr) {
-                if (!valid_internet_address (addr) &&
-                    !valid_wildcard_internet_address (addr)) {
-                        cli_out ("Invalid ip or wildcard : %s", addr);
-                        ret= -1;
+
+        opt_count = 0;
+        buf = line;
+        while (fscanf (fp, "%s", buf) != EOF) {
+
+                opt_count++;
+                tok_key = strtok_r (line, "=", &saveptr);
+                tok_val = strtok_r (NULL, "=", &saveptr);
+                if (!tok_key || !tok_val) {
+                        ret = -1;
+                        snprintf(errstr, sizeof(errstr), "'%s' file format "
+                                 "not valid.", tagpath);
+                        if (op_errstr)
+                                *op_errstr = gf_strdup(errstr);
                         goto out;
                 }
-                addr = strtok_r (NULL, ",", &saveptr);
+
+                snprintf (iter_key, sizeof (iter_key), "key%d", opt_count);
+                dkey = gf_strdup (tok_key);
+                ret = dict_set_dynstr (dict, iter_key, dkey);
+                if (ret)
+                        goto out;
+                dkey = NULL;
+
+                snprintf (iter_val, sizeof (iter_val), "value%d", opt_count);
+                dval = gf_strdup (tok_val);
+                ret = dict_set_dynstr (dict, iter_val, dval);
+                if (ret)
+                        goto out;
+                dval = NULL;
+
         }
+
+        if (!opt_count) {
+                ret = -1;
+                snprintf(errstr, sizeof(errstr), "'%s' file format "
+                         "not valid.", tagpath);
+                if (op_errstr)
+                        *op_errstr = gf_strdup(errstr);
+                goto out;
+        }
+        ret = dict_set_int32 (dict, "count", opt_count);
 out:
-        if (duplist)
-                GF_FREE (duplist);
-        gf_log ("cli", GF_LOG_INFO, "Returning %d", ret);
+
+        GF_FREE (tagpath);
+
+        if (ret) {
+                GF_FREE (dkey);
+                GF_FREE (dval);
+        }
+
+        if (fp)
+                fclose (fp);
+
         return ret;
 }
+#undef GLUSTERD_DEFAULT_WORKDIR
 
 int32_t
-cli_cmd_volume_set_parse (const char **words, int wordcount, dict_t **options)
+cli_cmd_volume_set_parse (const char **words, int wordcount, dict_t **options,
+                          char **op_errstr)
 {
-        dict_t  *dict = NULL;
-        char    *volname = NULL;
-        int     ret = -1;
-        int     count = 0;
-        char    *key = NULL;
-        char    *value = NULL;
-        int     i = 0;
-        char    str[50] = {0,};
+        dict_t                  *dict = NULL;
+        char                    *volname = NULL;
+        int                     ret = -1;
+        int                     count = 0;
+        char                    *key = NULL;
+        char                    *value = NULL;
+        int                     i = 0;
+        char                    str[50] = {0,};
 
         GF_ASSERT (words);
         GF_ASSERT (options);
@@ -703,42 +823,65 @@ cli_cmd_volume_set_parse (const char **words, int wordcount, dict_t **options)
         if (ret)
                 goto out;
 
-        if (wordcount == 3) {
-                if (!strcmp (volname, "help")) {
-                        ret = dict_set_str (dict, "help", volname);
-                        if (ret)
-                                goto out;
-                } else if (!strcmp (volname, "help-xml")) {
-                        ret = dict_set_str (dict, "help-xml", volname);
-                        if (ret)
-                                goto out;
-                } else {
+        if ((!strcmp (volname, "help") || !strcmp (volname, "help-xml"))
+            && wordcount == 3 ) {
+                ret = dict_set_str (dict, volname, volname);
+                if (ret)
+                        goto out;
+
+        } else if (wordcount < 5) {
+                ret = -1;
+                goto out;
+
+        } else if (wordcount == 5  && cli_is_key_spl ((char *)words[3])) {
+                key = (char *) words[3];
+                value = (char *) words[4];
+                if ( !key || !value) {
                         ret = -1;
                         goto out;
                 }
-        }
 
+                ret = gf_strip_whitespace (value, strlen (value));
+                if (ret == -1)
+                        goto out;
+
+                if (strlen (value) == 0) {
+                        ret = -1;
+                        goto out;
+                }
+
+                ret = cli_add_key_group (dict, key, value, op_errstr);
+                if (ret == 0)
+                        *options = dict;
+                goto out;
+        }
 
         for (i = 3; i < wordcount; i+=2) {
 
-		key = (char *) words[i];
-		value = (char *) words[i+1];
+                key = (char *) words[i];
+                value = (char *) words[i+1];
 
-		if ( !key || !value) {
-			ret = -1;
-			goto out;
-	        }
+                if ( !key || !value) {
+                        ret = -1;
+                        goto out;
+                }
 
                 count++;
-                if (!strncmp ("auth.allow", key, sizeof (key)) ||
-                    !strncmp ("auth.reject", key, sizeof (key))) {
-                        ret = cli_cmd_valid_ip_list (value);
-                        if (ret) {
-                                gf_log ("cli", GF_LOG_ERROR,
-                                        "invalid ips given");
-                                goto out;
-                        }
+
+                ret = gf_strip_whitespace (value, strlen (value));
+                if (ret == -1)
+                        goto out;
+
+                if (strlen (value) == 0) {
+                        ret = -1;
+                        goto out;
+                }
+
+                if (cli_is_key_spl (key)) {
+                        ret = -1;
+                        goto out;
                 }
+
                 sprintf (str, "key%d", count);
                 ret = dict_set_str (dict, str, key);
                 if (ret)
@@ -759,10 +902,8 @@ cli_cmd_volume_set_parse (const char **words, int wordcount, dict_t **options)
         *options = dict;
 
 out:
-        if (ret) {
-                if (dict)
-                        dict_destroy (dict);
-        }
+        if (ret)
+                dict_destroy (dict);
 
         return ret;
 }
@@ -781,6 +922,8 @@ cli_cmd_volume_add_brick_parse (const char **words, int wordcount,
         int     count = 1;
         char    *w = NULL;
         int     index;
+        gf_boolean_t is_force = _gf_false;
+        int wc = wordcount;
 
         GF_ASSERT (words);
         GF_ASSERT (options);
@@ -825,7 +968,7 @@ cli_cmd_volume_add_brick_parse (const char **words, int wordcount,
                 }
                 count = strtol (words[4], NULL, 0);
                 if (!count || (count < 2)) {
-                        cli_out ("replica count should be greater than 1");
+                        cli_err ("replica count should be greater than 1");
                         ret = -1;
                         goto out;
                 }
@@ -841,7 +984,7 @@ cli_cmd_volume_add_brick_parse (const char **words, int wordcount,
                 }
                 count = strtol (words[4], NULL, 0);
                 if (!count || (count < 2)) {
-                        cli_out ("stripe count should be greater than 1");
+                        cli_err ("stripe count should be greater than 1");
                         ret = -1;
                         goto out;
                 }
@@ -858,7 +1001,13 @@ cli_cmd_volume_add_brick_parse (const char **words, int wordcount,
         brick_index = index;
 
 parse_bricks:
-        ret = cli_cmd_bricks_parse (words, wordcount, brick_index, &bricks,
+
+        if (strcmp (words[wordcount - 1], "force") == 0) {
+                is_force = _gf_true;
+                wc = wordcount - 1;
+        }
+
+        ret = cli_cmd_bricks_parse (words, wc, brick_index, &bricks,
                                     &brick_count);
         if (ret)
                 goto out;
@@ -872,6 +1021,10 @@ parse_bricks:
         if (ret)
                 goto out;
 
+        ret = dict_set_int32 (dict, "force", is_force);
+        if (ret)
+                goto out;
+
         *options = dict;
 
 out:
@@ -900,7 +1053,7 @@ cli_cmd_volume_remove_brick_parse (const char **words, int wordcount,
         char    *tmp_brick = NULL;
         char    *tmp_brick1 = NULL;
         char    *type_opword[] = { "replica", NULL };
-        char    *opwords[] = { "start", "commit", "pause", "abort", "status",
+        char    *opwords[] = { "start", "commit", "stop", "status",
                                "force", NULL };
         char    *w = NULL;
         int32_t  command = GF_OP_CMD_NONE;
@@ -933,7 +1086,7 @@ cli_cmd_volume_remove_brick_parse (const char **words, int wordcount,
                 }
                 count = strtol (words[4], NULL, 0);
                 if (count < 1) {
-                        cli_out ("replica count should be greater than 0 in "
+                        cli_err ("replica count should be greater than 0 in "
                                  "case of remove-brick");
                         ret = -1;
                         goto out;
@@ -962,10 +1115,8 @@ cli_cmd_volume_remove_brick_parse (const char **words, int wordcount,
                         command = GF_OP_CMD_COMMIT;
                         if (question)
                                 *question = 1;
-                } else if (!strcmp ("pause", w)) {
-                        command = GF_OP_CMD_PAUSE;
-                } else if (!strcmp ("abort", w)) {
-                        command = GF_OP_CMD_ABORT;
+                } else if (!strcmp ("stop", w)) {
+                        command = GF_OP_CMD_STOP;
                 } else if (!strcmp ("status", w)) {
                         command = GF_OP_CMD_STATUS;
                 } else if (!strcmp ("force", w)) {
@@ -1011,13 +1162,13 @@ cli_cmd_volume_remove_brick_parse (const char **words, int wordcount,
 
         while (brick_index < wordcount) {
                 if (validate_brick_name ((char *)words[brick_index])) {
-                        cli_out ("wrong brick type: %s, use <HOSTNAME>:"
+                        cli_err ("wrong brick type: %s, use <HOSTNAME>:"
                                  "<export-dir-abs-path>", words[brick_index]);
                         ret = -1;
                         goto out;
                 } else {
                         delimiter = strrchr(words[brick_index], ':');
-                        ret = cli_canonicalize_path (delimiter + 1);
+                        ret = gf_canonicalize_path (delimiter + 1);
                         if (ret)
                                 goto out;
                 }
@@ -1029,7 +1180,7 @@ cli_cmd_volume_remove_brick_parse (const char **words, int wordcount,
                         if (!(strcmp (tmp_brick, tmp_brick1))) {
                                 gf_log("",GF_LOG_ERROR, "Duplicate bricks"
                                        " found %s", words[brick_index]);
-                                cli_out("Duplicate bricks found %s",
+                                cli_err("Duplicate bricks found %s",
                                         words[brick_index]);
                                 ret = -1;
                                 goto out;
@@ -1056,10 +1207,8 @@ out:
                         dict_destroy (dict);
         }
 
-        if (tmp_brick)
-                GF_FREE (tmp_brick);
-        if (tmp_brick1)
-                GF_FREE (tmp_brick1);
+        GF_FREE (tmp_brick);
+        GF_FREE (tmp_brick1);
 
         return ret;
 }
@@ -1078,6 +1227,7 @@ cli_cmd_volume_replace_brick_parse (const char **words, int wordcount,
         char    *opwords[] = { "start", "commit", "pause", "abort", "status",
                                 NULL };
         char    *w = NULL;
+        gf_boolean_t is_force = _gf_false;
 
         GF_ASSERT (words);
         GF_ASSERT (options);
@@ -1105,13 +1255,13 @@ cli_cmd_volume_replace_brick_parse (const char **words, int wordcount,
         }
 
         if (validate_brick_name ((char *)words[3])) {
-                cli_out ("wrong brick type: %s, use "
+                cli_err ("wrong brick type: %s, use "
                          "<HOSTNAME>:<export-dir-abs-path>", words[3]);
                 ret = -1;
                 goto out;
         } else {
                 delimiter = strrchr ((char *)words[3], ':');
-                ret = cli_canonicalize_path (delimiter + 1);
+                ret = gf_canonicalize_path (delimiter + 1);
                 if (ret)
                         goto out;
         }
@@ -1126,13 +1276,13 @@ cli_cmd_volume_replace_brick_parse (const char **words, int wordcount,
         }
 
         if (validate_brick_name ((char *)words[4])) {
-                cli_out ("wrong brick type: %s, use "
+                cli_err ("wrong brick type: %s, use "
                          "<HOSTNAME>:<export-dir-abs-path>", words[4]);
                 ret = -1;
                 goto out;
         } else {
                 delimiter = strrchr ((char *)words[4], ':');
-                ret = cli_canonicalize_path (delimiter + 1);
+                ret = gf_canonicalize_path (delimiter + 1);
                 if (ret)
                         goto out;
         }
@@ -1175,12 +1325,17 @@ cli_cmd_volume_replace_brick_parse (const char **words, int wordcount,
         }
 
         if (wordcount == (op_index + 1)) {
-                if (replace_op != GF_REPLACE_OP_COMMIT) {
+                if ((replace_op != GF_REPLACE_OP_COMMIT) &&
+                    (replace_op != GF_REPLACE_OP_START)) {
                         ret = -1;
                         goto out;
                 }
                 if (!strcmp ("force", words[op_index])) {
-                        replace_op = GF_REPLACE_OP_COMMIT_FORCE;
+                        if (replace_op == GF_REPLACE_OP_COMMIT)
+                                replace_op = GF_REPLACE_OP_COMMIT_FORCE;
+
+                        else if (replace_op == GF_REPLACE_OP_START)
+                                is_force = _gf_true;
                 }
         }
 
@@ -1194,6 +1349,9 @@ cli_cmd_volume_replace_brick_parse (const char **words, int wordcount,
         if (ret)
                 goto out;
 
+        ret = dict_set_int32 (dict, "force", is_force);
+        if (ret)
+                goto out;
 
         *options = dict;
 
@@ -1235,12 +1393,12 @@ cli_cmd_log_filename_parse (const char **words, int wordcount, dict_t **options)
                 delimiter = strchr (words[4], ':');
                 if (!delimiter || delimiter == words[4]
                     || *(delimiter+1) != '/') {
-                        cli_out ("wrong brick type: %s, use <HOSTNAME>:"
+                        cli_err ("wrong brick type: %s, use <HOSTNAME>:"
                                  "<export-dir-abs-path>", words[4]);
                         ret = -1;
                         goto out;
                 } else {
-                        ret = cli_canonicalize_path (delimiter + 1);
+                        ret = gf_canonicalize_path (delimiter + 1);
                         if (ret)
                                 goto out;
                 }
@@ -1290,8 +1448,8 @@ cli_cmd_log_level_parse (const char **words, int worcount, dict_t **options)
 
         ret = glusterd_check_log_level(words[5]);
         if (ret == -1) {
-                cli_out("Invalid log level [%s] specified", words[5]);
-                cli_out("Valid values for loglevel: (DEBUG|WARNING|ERROR"
+                cli_err("Invalid log level [%s] specified", words[5]);
+                cli_err("Valid values for loglevel: (DEBUG|WARNING|ERROR"
                         "|CRITICAL|NONE|TRACE)");
                 goto out;
         }
@@ -1351,12 +1509,12 @@ cli_cmd_log_locate_parse (const char **words, int wordcount, dict_t **options)
                 delimiter = strchr (words[4], ':');
                 if (!delimiter || delimiter == words[4]
                     || *(delimiter+1) != '/') {
-                        cli_out ("wrong brick type: %s, use <HOSTNAME>:"
+                        cli_err ("wrong brick type: %s, use <HOSTNAME>:"
                                  "<export-dir-abs-path>", words[4]);
                         ret = -1;
                         goto out;
                 } else {
-                        ret = cli_canonicalize_path (delimiter + 1);
+                        ret = gf_canonicalize_path (delimiter + 1);
                         if (ret)
                                 goto out;
                 }
@@ -1402,12 +1560,12 @@ cli_cmd_log_rotate_parse (const char **words, int wordcount, dict_t **options)
                 delimiter = strchr (words[4], ':');
                 if (!delimiter || delimiter == words[4]
                     || *(delimiter+1) != '/') {
-                        cli_out ("wrong brick type: %s, use <HOSTNAME>:"
+                        cli_err ("wrong brick type: %s, use <HOSTNAME>:"
                                  "<export-dir-abs-path>", words[4]);
                         ret = -1;
                         goto out;
                 } else {
-                        ret = cli_canonicalize_path (delimiter + 1);
+                        ret = gf_canonicalize_path (delimiter + 1);
                         if (ret)
                                 goto out;
                 }
@@ -1438,22 +1596,161 @@ gsyncd_glob_check (const char *w)
         return !!strpbrk (w, "*?[");
 }
 
+static int
+config_parse (const char **words, int wordcount, dict_t *dict,
+              unsigned cmdi, unsigned glob)
+{
+        int32_t            ret     = -1;
+        int32_t            i       = -1;
+        char               *append_str = NULL;
+        size_t             append_len = 0;
+        char               *subop = NULL;
+
+        switch ((wordcount - 1) - cmdi) {
+        case 0:
+                subop = gf_strdup ("get-all");
+                break;
+        case 1:
+                if (words[cmdi + 1][0] == '!') {
+                        (words[cmdi + 1])++;
+                        if (gf_asprintf (&subop, "del%s",
+                                         glob ? "-glob" : "") == -1)
+                                subop = NULL;
+                } else
+                        subop = gf_strdup ("get");
+
+                ret = dict_set_str (dict, "op_name", ((char *)words[cmdi + 1]));
+                if (ret < 0)
+                        goto out;
+                break;
+        default:
+                if (gf_asprintf (&subop, "set%s", glob ? "-glob" : "") == -1)
+                        subop = NULL;
+
+                ret = dict_set_str (dict, "op_name", ((char *)words[cmdi + 1]));
+                if (ret < 0)
+                        goto out;
+
+                /* join the varargs by spaces to get the op_value */
+
+                for (i = cmdi + 2; i < wordcount; i++)
+                        append_len += (strlen (words[i]) + 1);
+                /* trailing strcat will add two bytes, make space for that */
+                append_len++;
+
+                append_str = GF_CALLOC (1, append_len, cli_mt_append_str);
+                if (!append_str) {
+                        ret = -1;
+                        goto out;
+                }
+
+                for (i = cmdi + 2; i < wordcount; i++) {
+                        strcat (append_str, words[i]);
+                        strcat (append_str, " ");
+                }
+                append_str[append_len - 2] = '\0';
+                /* "checkpoint now" is special: we resolve that "now" */
+                if (strcmp (words[cmdi + 1], "checkpoint") == 0 &&
+                    strcmp (append_str, "now") == 0) {
+                        struct timeval tv = {0,};
+
+                        ret = gettimeofday (&tv, NULL);
+                        if (ret == -1)
+                                goto out; /* FIXME: free append_str? */
+
+                        GF_FREE (append_str);
+                        append_str = GF_CALLOC (1, 300, cli_mt_append_str);
+                        if (!append_str) {
+                                ret = -1;
+                                goto out;
+                        }
+                        strcpy (append_str, "as of ");
+                        gf_time_fmt (append_str + strlen ("as of "),
+                                     300 - strlen ("as of "),
+                                     tv.tv_sec, gf_timefmt_FT);
+                }
+
+                ret = dict_set_dynstr (dict, "op_value", append_str);
+        }
+
+        ret = -1;
+        if (subop) {
+                ret = dict_set_dynstr (dict, "subop", subop);
+                if (!ret)
+                      subop = NULL;
+        }
+
+out:
+        if (ret && append_str)
+                GF_FREE (append_str);
+
+        GF_FREE (subop);
+
+        gf_log ("cli", GF_LOG_DEBUG, "Returning %d", ret);
+        return ret;
+}
+
+static int32_t
+force_push_pem_parse (const char **words, int wordcount,
+                      dict_t *dict, unsigned *cmdi)
+{
+        int32_t            ret     = 0;
+
+        if (!strcmp ((char *)words[wordcount-1], "force")) {
+                if ((strcmp ((char *)words[wordcount-2], "start")) &&
+                    (strcmp ((char *)words[wordcount-2], "stop")) &&
+                    (strcmp ((char *)words[wordcount-2], "create")) &&
+                    (strcmp ((char *)words[wordcount-2], "push-pem"))) {
+                        ret = -1;
+                        goto out;
+                }
+                ret = dict_set_uint32 (dict, "force",
+                                       _gf_true);
+                if (ret)
+                        goto out;
+                (*cmdi)++;
+
+                if (!strcmp ((char *)words[wordcount-2], "push-pem")) {
+                        if (strcmp ((char *)words[wordcount-3], "create")) {
+                                ret = -1;
+                                goto out;
+                        }
+                        ret = dict_set_int32 (dict, "push_pem", 1);
+                        if (ret)
+                                goto out;
+                        (*cmdi)++;
+                }
+        } else if (!strcmp ((char *)words[wordcount-1], "push-pem")) {
+                if (strcmp ((char *)words[wordcount-2], "create")) {
+                        ret = -1;
+                        goto out;
+                }
+                ret = dict_set_int32 (dict, "push_pem", 1);
+                if (ret)
+                        goto out;
+                (*cmdi)++;
+        }
+
+out:
+        gf_log ("cli", GF_LOG_DEBUG, "Returning %d", ret);
+        return ret;
+}
+
+
 int32_t
 cli_cmd_gsync_set_parse (const char **words, int wordcount, dict_t **options)
 {
         int32_t            ret     = -1;
         dict_t             *dict   = NULL;
         gf1_cli_gsync_set  type    = GF_GSYNC_OPTION_TYPE_NONE;
-        char               *append_str = NULL;
-        size_t             append_len = 0;
-        char               *subop = NULL;
         int                i       = 0;
         unsigned           masteri = 0;
         unsigned           slavei  = 0;
         unsigned           glob    = 0;
         unsigned           cmdi    = 0;
-        char               *opwords[] = { "status", "start", "stop", "config",
-                                          "log-rotate", NULL };
+        char               *opwords[] = { "create", "status", "start", "stop",
+                                          "config", "force", "delete",
+                                          "push-pem", "detail", NULL };
         char               *w = NULL;
 
         GF_ASSERT (words);
@@ -1465,10 +1762,11 @@ cli_cmd_gsync_set_parse (const char **words, int wordcount, dict_t **options)
 
         /* new syntax:
          *
-         * volume geo-replication [$m [$s]] status
+         * volume geo-replication $m $s create [push-pem] [force]
+         * volume geo-replication [$m [$s]] status [detail]
          * volume geo-replication [$m] $s config [[!]$opt [$val]]
-         * volume geo-replication $m $s start|stop
-         * volume geo-replication $m [$s] log-rotate
+         * volume geo-replication $m $s start|stop [force]
+         * volume geo-replication $m $s delete
          */
 
         if (wordcount < 3)
@@ -1499,6 +1797,13 @@ cli_cmd_gsync_set_parse (const char **words, int wordcount, dict_t **options)
                 if (slavei == 3)
                         masteri = 2;
         } else if (i <= 3) {
+                if (!strcmp ((char *)words[wordcount-1], "detail")) {
+                        /* For status detail it is mandatory to provide
+                         * both master and slave */
+                        ret = -1;
+                        goto out;
+                }
+
                 /* no $s, can only be status cmd
                  * (with either a single $m before it or nothing)
                  * -- these conditions imply that i <= 3 after
@@ -1525,7 +1830,12 @@ cli_cmd_gsync_set_parse (const char **words, int wordcount, dict_t **options)
         if (!w)
                 goto out;
 
-        if (strcmp (w, "status") == 0) {
+        if (strcmp (w, "create") == 0) {
+                type = GF_GSYNC_OPTION_TYPE_CREATE;
+
+                if (!masteri || !slavei)
+                        goto out;
+        } else if (strcmp (w, "status") == 0) {
                 type = GF_GSYNC_OPTION_TYPE_STATUS;
 
                 if (slavei && !masteri)
@@ -1545,14 +1855,33 @@ cli_cmd_gsync_set_parse (const char **words, int wordcount, dict_t **options)
 
                 if (!masteri || !slavei)
                         goto out;
-        } else if (strcmp(w, "log-rotate") == 0) {
-                type = GF_GSYNC_OPTION_TYPE_ROTATE;
+        } else if (strcmp (w, "delete") == 0) {
+                type = GF_GSYNC_OPTION_TYPE_DELETE;
 
-                if (slavei && !masteri)
+                if (!masteri || !slavei)
                         goto out;
         } else
                 GF_ASSERT (!"opword mismatch");
 
+        ret = force_push_pem_parse (words, wordcount, dict, &cmdi);
+        if (ret)
+                goto out;
+
+        if (!strcmp ((char *)words[wordcount-1], "detail")) {
+                if (strcmp ((char *)words[wordcount-2], "status")) {
+                        ret = -1;
+                        goto out;
+                }
+                if (!slavei || !masteri) {
+                        ret = -1;
+                        goto out;
+                }
+                ret = dict_set_uint32 (dict, "status-detail", _gf_true);
+                if (ret)
+                        goto out;
+                cmdi++;
+        }
+
         if (type != GF_GSYNC_OPTION_TYPE_CONFIG &&
             (cmdi < wordcount - 1 || glob))
                 goto out;
@@ -1561,72 +1890,27 @@ cli_cmd_gsync_set_parse (const char **words, int wordcount, dict_t **options)
 
         ret = 0;
 
-        if (masteri)
+        if (masteri) {
                 ret = dict_set_str (dict, "master", (char *)words[masteri]);
+                if (!ret)
+                        ret = dict_set_str (dict, "volname",
+                                            (char *)words[masteri]);
+        }
         if (!ret && slavei)
                 ret = dict_set_str (dict, "slave", (char *)words[slavei]);
         if (!ret)
                 ret = dict_set_int32 (dict, "type", type);
-        if (!ret && type == GF_GSYNC_OPTION_TYPE_CONFIG) {
-                switch ((wordcount - 1) - cmdi) {
-                case 0:
-                        subop = gf_strdup ("get-all");
-                        break;
-                case 1:
-                        if (words[cmdi + 1][0] == '!') {
-                                (words[cmdi + 1])++;
-                                if (gf_asprintf (&subop, "del%s", glob ? "-glob" : "") == -1)
-                                        subop = NULL;
-                        } else
-                                subop = gf_strdup ("get");
-
-                        ret = dict_set_str (dict, "op_name", ((char *)words[cmdi + 1]));
-                        if (ret < 0)
-                                goto out;
-                        break;
-                default:
-                        if (gf_asprintf (&subop, "set%s", glob ? "-glob" : "") == -1)
-                                subop = NULL;
-
-                        ret = dict_set_str (dict, "op_name", ((char *)words[cmdi + 1]));
-                        if (ret < 0)
-                                goto out;
-
-                        /* join the varargs by spaces to get the op_value */
-
-                        for (i = cmdi + 2; i < wordcount; i++)
-                                append_len += (strlen (words[i]) + 1);
-                        /* trailing strcat will add two bytes, make space for that */
-                        append_len++;
-
-                        append_str = GF_CALLOC (1, append_len, cli_mt_append_str);
-                        if (!append_str) {
-                                ret = -1;
-                                goto out;
-                        }
-
-                        for (i = cmdi + 2; i < wordcount; i++) {
-                                strcat (append_str, words[i]);
-                                strcat (append_str, " ");
-                        }
-                        append_str[append_len - 2] = '\0';
-
-                        ret = dict_set_dynstr (dict, "op_value", append_str);
-                }
-
-                if (!subop || dict_set_dynstr (dict, "subop", subop) != 0)
-                        ret = -1;
-        }
+        if (!ret && type == GF_GSYNC_OPTION_TYPE_CONFIG)
+                ret = config_parse (words, wordcount, dict, cmdi, glob);
 
 out:
         if (ret) {
                 if (dict)
                         dict_destroy (dict);
-                if (append_str)
-                        GF_FREE (append_str);
         } else
                 *options = dict;
 
+
         return ret;
 }
 
@@ -1710,7 +1994,7 @@ cli_cmd_volume_top_parse (const char **words, int wordcount,
         char    *delimiter      = NULL;
         char    *opwords[]      = { "open", "read", "write", "opendir",
                                     "readdir", "read-perf", "write-perf",
-                                    NULL };
+                                    "clear", NULL };
         char    *w = NULL;
 
         GF_ASSERT (words);
@@ -1755,6 +2039,13 @@ cli_cmd_volume_top_parse (const char **words, int wordcount,
         } else if (strcmp (w, "write-perf") == 0) {
                 top_op = GF_CLI_TOP_WRITE_PERF;
                 perf = 1;
+        } else if (strcmp (w, "clear") == 0) {
+                ret = dict_set_int32 (dict, "clear-stats", 1);
+                if (ret) {
+                        gf_log ("cli", GF_LOG_ERROR,
+                                "Could not set clear-stats in dict");
+                        goto out;
+                }
         } else
                 GF_ASSERT (!"opword mismatch");
         ret = dict_set_int32 (dict, "top-op", (int32_t)top_op);
@@ -1784,12 +2075,12 @@ cli_cmd_volume_top_parse (const char **words, int wordcount,
                         delimiter = strchr (value, ':');
                         if (!delimiter || delimiter == value
                             || *(delimiter+1) != '/') {
-                                cli_out ("wrong brick type: %s, use <HOSTNAME>:"
+                                cli_err ("wrong brick type: %s, use <HOSTNAME>:"
                                          "<export-dir-abs-path>", value);
                                 ret = -1;
                                 goto out;
                         } else {
-                                ret = cli_canonicalize_path (delimiter + 1);
+                                ret = gf_canonicalize_path (delimiter + 1);
                                 if (ret)
                                         goto out;
                         }
@@ -1800,7 +2091,7 @@ cli_cmd_volume_top_parse (const char **words, int wordcount,
                         if (!ret)
                                 list_cnt = atoi (value);
                         if (ret || (list_cnt < 0) || (list_cnt > 100)) {
-                                cli_out ("list-cnt should be between 0 to 100");
+                                cli_err ("list-cnt should be between 0 to 100");
                                 ret = -1;
                                 goto out;
                         }
@@ -1810,10 +2101,11 @@ cli_cmd_volume_top_parse (const char **words, int wordcount,
                                 blk_size = atoi (value);
                         if (ret || (blk_size <= 0)) {
                                 if (blk_size < 0)
-                                        cli_out ("block size is an invalid number");
+                                        cli_err ("block size is an invalid"
+                                                 " number");
                                 else
-                                        cli_out ("block size should be an integer "
-                                         "greater than zero");
+                                        cli_err ("block size should be an "
+                                                 "integer greater than zero");
                                 ret = -1;
                                 goto out;
                         }
@@ -1824,9 +2116,9 @@ cli_cmd_volume_top_parse (const char **words, int wordcount,
                                 count = atoi(value);
                         if (ret || (count <= 0)) {
                                 if (count < 0)
-                                        cli_out ("count is an invalid number");
+                                        cli_err ("count is an invalid number");
                                 else
-                                        cli_out ("count should be an integer "
+                                        cli_err ("count should be an integer "
                                                  "greater than zero");
 
                                 ret = -1;
@@ -1852,9 +2144,17 @@ cli_cmd_volume_top_parse (const char **words, int wordcount,
         }
 
         if ((blk_size > 0) ^ (count > 0)) {
+                cli_err ("Need to give both 'bs' and 'count'");
+                ret = -1;
+                goto out;
+        } else if (((uint64_t)blk_size * count) > (10 * GF_UNIT_GB)) {
+                cli_err ("'bs * count' value %"PRIu64" is greater than "
+                         "maximum allowed value of 10GB",
+                         ((uint64_t)blk_size * count));
                 ret = -1;
                 goto out;
         }
+
         *options = dict;
 out:
         if (ret && dict)
@@ -1869,7 +2169,7 @@ cli_cmd_get_statusop (const char *arg)
         uint32_t   ret       = GF_CLI_STATUS_NONE;
         char      *w         = NULL;
         char      *opwords[] = {"detail", "mem", "clients", "fd",
-                                "inode", "callpool", NULL};
+                                "inode", "callpool", "tasks", NULL};
         struct {
                 char      *opname;
                 uint32_t   opcode;
@@ -1880,6 +2180,7 @@ cli_cmd_get_statusop (const char *arg)
                 { "fd",       GF_CLI_STATUS_FD       },
                 { "inode",    GF_CLI_STATUS_INODE    },
                 { "callpool", GF_CLI_STATUS_CALLPOOL },
+                { "tasks",    GF_CLI_STATUS_TASKS    },
                 { NULL }
         };
 
@@ -1939,7 +2240,7 @@ cli_cmd_volume_status_parse (const char **words, int wordcount,
 
                 if (!strcmp (words[2], "all")) {
                         if (cmd == GF_CLI_STATUS_NONE) {
-                                cli_out ("%s is not a valid status option",
+                                cli_err ("%s is not a valid status option",
                                          words[3]);
                                 ret = -1;
                                 goto out;
@@ -1956,6 +2257,8 @@ cli_cmd_volume_status_parse (const char **words, int wordcount,
                         if (cmd == GF_CLI_STATUS_NONE) {
                                 if (!strcmp (words[3], "nfs")) {
                                         cmd |= GF_CLI_STATUS_NFS;
+                                } else if (!strcmp (words[3], "shd")) {
+                                        cmd |= GF_CLI_STATUS_SHD;
                                 } else {
                                         cmd = GF_CLI_STATUS_BRICK;
                                         ret = dict_set_str (dict, "brick",
@@ -1972,14 +2275,14 @@ cli_cmd_volume_status_parse (const char **words, int wordcount,
 
         case 5:
                 if (!strcmp (words[2], "all")) {
-                        cli_out ("Cannot specify brick/nfs for \"all\"");
+                        cli_err ("Cannot specify brick/nfs for \"all\"");
                         ret = -1;
                         goto out;
                 }
 
                 cmd = cli_cmd_get_statusop (words[4]);
                 if (cmd == GF_CLI_STATUS_NONE) {
-                        cli_out ("%s is not a valid status option",
+                        cli_err ("%s is not a valid status option",
                                  words[4]);
                         ret = -1;
                         goto out;
@@ -1991,13 +2294,33 @@ cli_cmd_volume_status_parse (const char **words, int wordcount,
                         goto out;
 
                 if (!strcmp (words[3], "nfs")) {
-                        if (cmd == GF_CLI_STATUS_FD) {
-                                cli_out ("FD status not available for NFS");
+                        if (cmd == GF_CLI_STATUS_FD ||
+                            cmd == GF_CLI_STATUS_DETAIL ||
+                            cmd == GF_CLI_STATUS_TASKS) {
+                                cli_err ("Detail/FD/Tasks status not available"
+                                         " for NFS Servers");
                                 ret = -1;
                                 goto out;
                         }
                         cmd |= GF_CLI_STATUS_NFS;
+                } else if (!strcmp (words[3], "shd")){
+                        if (cmd == GF_CLI_STATUS_FD ||
+                            cmd == GF_CLI_STATUS_CLIENTS ||
+                            cmd == GF_CLI_STATUS_DETAIL ||
+                            cmd == GF_CLI_STATUS_TASKS) {
+                                cli_err ("Detail/FD/Clients/Tasks status not "
+                                         "available for Self-heal Daemons");
+                                ret = -1;
+                                goto out;
+                        }
+                        cmd |= GF_CLI_STATUS_SHD;
                 } else {
+                        if (cmd == GF_CLI_STATUS_TASKS) {
+                                cli_err ("Tasks status not available for "
+                                         "bricks");
+                                ret = -1;
+                                goto out;
+                        }
                         cmd |= GF_CLI_STATUS_BRICK;
                         ret = dict_set_str (dict, "brick", (char *)words[3]);
                 }
@@ -2146,12 +2469,103 @@ out:
        return ret;
 }
 
+static int
+extract_hostname_path_from_token (const char *tmp_words, char **hostname,
+                                  char **path)
+{
+        int ret = 0;
+        char *delimiter = NULL;
+        char *tmp_host = NULL;
+        char *host_name = NULL;
+        char *words = NULL;
+
+        *hostname = NULL;
+        *path = NULL;
+
+        words = GF_CALLOC (1, strlen (tmp_words) + 1, gf_common_mt_char);
+        if (!words){
+                ret = -1;
+                goto out;
+        }
+
+        strncpy (words, tmp_words, strlen (tmp_words) + 1);
+
+        if (validate_brick_name (words)) {
+                cli_err ("Wrong brick type: %s, use <HOSTNAME>:"
+                        "<export-dir-abs-path>", words);
+                ret = -1;
+                goto out;
+        } else {
+                delimiter = strrchr (words, ':');
+                ret = gf_canonicalize_path (delimiter + 1);
+                if (ret) {
+                        goto out;
+                } else {
+                        *path = GF_CALLOC (1, strlen (delimiter+1) +1,
+                                           gf_common_mt_char);
+                        if (!*path) {
+                           ret = -1;
+                                goto out;
+
+                        }
+                        strncpy (*path, delimiter +1,
+                                 strlen(delimiter + 1) + 1);
+                }
+        }
+
+        tmp_host = gf_strdup (words);
+        if (!tmp_host) {
+                gf_log ("cli", GF_LOG_ERROR, "Out of memory");
+                ret = -1;
+                goto out;
+        }
+        get_host_name (tmp_host, &host_name);
+        if (!host_name) {
+                ret = -1;
+                gf_log("cli",GF_LOG_ERROR, "Unable to allocate "
+                        "memory");
+                goto out;
+        }
+        if (!(strcmp (host_name, "localhost") &&
+            strcmp (host_name, "127.0.0.1") &&
+            strncmp (host_name, "0.", 2))) {
+                cli_err ("Please provide a valid hostname/ip other "
+                         "than localhost, 127.0.0.1 or loopback "
+                         "address (0.0.0.0 to 0.255.255.255).");
+                ret = -1;
+                goto out;
+        }
+        if (!valid_internet_address (host_name, _gf_false)) {
+                cli_err ("internet address '%s' does not conform to "
+                          "standards", host_name);
+                ret = -1;
+                goto out;
+        }
+
+        *hostname = GF_CALLOC (1, strlen (host_name) + 1,
+                                       gf_common_mt_char);
+        if (!*hostname) {
+                ret = -1;
+                goto out;
+        }
+        strncpy (*hostname, host_name, strlen (host_name) + 1);
+        ret = 0;
+
+out:
+        GF_FREE (words);
+        GF_FREE (tmp_host);
+        return ret;
+}
+
+
 int
 cli_cmd_volume_heal_options_parse (const char **words, int wordcount,
                                    dict_t **options)
 {
         int     ret = 0;
         dict_t  *dict = NULL;
+        char    *hostname = NULL;
+        char    *path = NULL;
 
         dict = dict_new ();
         if (!dict)
@@ -2173,6 +2587,11 @@ cli_cmd_volume_heal_options_parse (const char **words, int wordcount,
                         ret = dict_set_int32 (dict, "heal-op",
                                               GF_AFR_OP_HEAL_FULL);
                         goto done;
+                } else if (!strcmp (words[3], "statistics")) {
+                        ret = dict_set_int32 (dict, "heal-op",
+                                              GF_AFR_OP_STATISTICS);
+                        goto done;
+
                 } else if (!strcmp (words[3], "info")) {
                         ret = dict_set_int32 (dict, "heal-op",
                                               GF_AFR_OP_INDEX_SUMMARY);
@@ -2183,28 +2602,66 @@ cli_cmd_volume_heal_options_parse (const char **words, int wordcount,
                 }
         }
         if (wordcount == 5) {
-                if (strcmp (words[3], "info")) {
+                if (strcmp (words[3], "info") &&
+                    strcmp (words[3], "statistics")) {
                         ret = -1;
                         goto out;
                 }
-                if (!strcmp (words[4], "healed")) {
-                        ret = dict_set_int32 (dict, "heal-op",
-                                              GF_AFR_OP_HEALED_FILES);
-                        goto done;
-                }
-                if (!strcmp (words[4], "heal-failed")) {
-                        ret = dict_set_int32 (dict, "heal-op",
-                                              GF_AFR_OP_HEAL_FAILED_FILES);
-                        goto done;
+
+                if (!strcmp (words[3], "info")) {
+                        if (!strcmp (words[4], "healed")) {
+                                ret = dict_set_int32 (dict, "heal-op",
+                                                      GF_AFR_OP_HEALED_FILES);
+                                goto done;
+                        }
+                        if (!strcmp (words[4], "heal-failed")) {
+                                ret = dict_set_int32 (dict, "heal-op",
+                                                   GF_AFR_OP_HEAL_FAILED_FILES);
+                                goto done;
+                        }
+                        if (!strcmp (words[4], "split-brain")) {
+                                ret = dict_set_int32 (dict, "heal-op",
+                                                   GF_AFR_OP_SPLIT_BRAIN_FILES);
+                                goto done;
+                        }
                 }
-                if (!strcmp (words[4], "split-brain")) {
-                        ret = dict_set_int32 (dict, "heal-op",
-                                              GF_AFR_OP_SPLIT_BRAIN_FILES);
-                        goto done;
+
+                if (!strcmp (words[3], "statistics")) {
+                        if (!strcmp (words[4], "heal-count")) {
+                                ret = dict_set_int32 (dict, "heal-op",
+                                               GF_AFR_OP_STATISTICS_HEAL_COUNT);
+                                goto done;
+                        }
                 }
                 ret = -1;
                 goto out;
         }
+        if (wordcount == 7) {
+                if (!strcmp (words[3], "statistics")
+                    && !strcmp (words[4], "heal-count")
+                    && !strcmp (words[5], "replica")) {
+
+                        ret = dict_set_int32 (dict, "heal-op",
+                                   GF_AFR_OP_STATISTICS_HEAL_COUNT_PER_REPLICA);
+                        if (ret)
+                                goto out;
+                        ret = extract_hostname_path_from_token (words[6],
+                                                              &hostname, &path);
+                        if (ret)
+                                goto out;
+                        ret = dict_set_dynstr (dict, "per-replica-cmd-hostname",
+                                               hostname);
+                        if (ret)
+                                goto out;
+                        ret = dict_set_dynstr (dict, "per-replica-cmd-path",
+                                               path);
+                        if (ret)
+                                goto out;
+                        else
+                                goto done;
+
+                }
+        }
         ret = -1;
         goto out;
 done:
@@ -2217,3 +2674,1009 @@ out:
 
         return ret;
 }
+
+int
+cli_cmd_volume_defrag_parse (const char **words, int wordcount,
+                             dict_t **options)
+{
+        dict_t                 *dict = NULL;
+        int                      ret = -1;
+        char                *option  = NULL;
+        char                *volname = NULL;
+        char                *command = NULL;
+        gf_cli_defrag_type       cmd = 0;
+
+        GF_ASSERT (words);
+        GF_ASSERT (options);
+
+        dict = dict_new ();
+        if (!dict)
+                goto out;
+
+        if (!((wordcount == 4) || (wordcount == 5)))
+                goto out;
+
+        if (wordcount == 4) {
+                if (strcmp (words[3], "start") && strcmp (words[3], "stop") &&
+                    strcmp (words[3], "status"))
+                            goto out;
+        } else {
+                if (strcmp (words[3], "fix-layout") &&
+                    strcmp (words[3], "start"))
+                        goto out;
+        }
+
+        volname = (char *) words[2];
+
+        if (wordcount == 4) {
+                command = (char *) words[3];
+        }
+        if (wordcount == 5) {
+               if ((strcmp (words[3], "fix-layout") ||
+                    strcmp (words[4], "start")) &&
+                    (strcmp (words[3], "start") ||
+                    strcmp (words[4], "force"))) {
+                        ret = -1;
+                        goto out;
+                }
+                command = (char *) words[3];
+                option = (char *) words[4];
+        }
+
+        if (strcmp (command, "start") == 0) {
+                cmd = GF_DEFRAG_CMD_START;
+                if (option && strcmp (option, "force") == 0) {
+                                cmd = GF_DEFRAG_CMD_START_FORCE;
+                        }
+                goto done;
+        }
+
+        if (strcmp (command, "fix-layout") == 0) {
+                cmd = GF_DEFRAG_CMD_START_LAYOUT_FIX;
+                goto done;
+        }
+        if (strcmp (command, "stop") == 0) {
+                cmd = GF_DEFRAG_CMD_STOP;
+                goto done;
+        }
+        if (strcmp (command, "status") == 0) {
+                cmd = GF_DEFRAG_CMD_STATUS;
+        }
+
+done:
+        ret = dict_set_str (dict, "volname", volname);
+
+        if (ret) {
+                gf_log (THIS->name, GF_LOG_ERROR, "failed to set dict");
+                goto out;
+        }
+
+        ret = dict_set_int32 (dict, "rebalance-command", (int32_t) cmd);
+
+        if (ret) {
+                gf_log (THIS->name, GF_LOG_ERROR, "failed to set dict");
+                goto out;
+        }
+
+        *options = dict;
+
+out:
+        if (ret && dict)
+                dict_destroy (dict);
+
+        return ret;
+}
+
+int32_t
+cli_snap_create_desc_parse (dict_t *dict, const char **words,
+                            size_t wordcount, int32_t desc_opt_loc)
+{
+        int32_t        ret      = -1;
+        char          *desc     = NULL;
+        int32_t        desc_len = 0;
+
+        desc = GF_CALLOC (MAX_SNAP_DESCRIPTION_LEN + 1, sizeof(char),
+                          gf_common_mt_char);
+        if (!desc) {
+                ret = -1;
+                goto out;
+        }
+
+
+        if (strlen (words[desc_opt_loc]) >= MAX_SNAP_DESCRIPTION_LEN) {
+                cli_out ("snapshot create: description truncated: "
+                         "Description provided is longer than 1024 characters");
+                desc_len = MAX_SNAP_DESCRIPTION_LEN;
+        } else {
+                desc_len = strlen (words[desc_opt_loc]);
+        }
+
+        strncpy (desc, words[desc_opt_loc], desc_len);
+        desc[desc_len] = '\0';
+        /* Calculating the size of the description as given by the user */
+
+        ret = dict_set_dynstr (dict, "description", desc);
+        if (ret) {
+                gf_log ("cli", GF_LOG_ERROR, "Unable to save snap "
+                        "description");
+                goto out;
+        }
+
+        ret = 0;
+out:
+        if (ret && desc)
+                GF_FREE (desc);
+
+        return ret;
+}
+
+/* Function to check whether the Volume name is repeated */
+int
+cli_check_if_volname_repeated (const char **words, unsigned int start_index,
+                           uint64_t cur_index) {
+        uint64_t        i       =       -1;
+        int             ret     =        0;
+
+        GF_ASSERT (words);
+
+        for (i = start_index ; i < cur_index ; i++) {
+                if (strcmp (words[i], words[cur_index]) == 0) {
+                        ret = -1;
+                        goto out;
+                }
+        }
+out :
+        return ret;
+}
+
+/* snapshot create <snapname> <vol-name(s)> [description <description>]
+ *                                           [force]
+ * @arg-0, dict     : Request Dictionary to be sent to server side.
+ * @arg-1, words    : Contains individual words of CLI command.
+ * @arg-2, wordcount: Contains number of words present in the CLI command.
+ *
+ * return value : -1 on failure
+ *                 0 on success
+ */
+int
+cli_snap_create_parse (dict_t *dict, const char **words, int wordcount) {
+        uint64_t        i               =        0;
+        int             ret             =       -1;
+        uint64_t        volcount        =        0;
+        char            key[PATH_MAX]   =        "";
+        char            *snapname       =       NULL;
+        unsigned int    cmdi            =        2;
+        /* cmdi is command index, here cmdi is "2" (gluster snapshot create)*/
+
+        GF_ASSERT (words);
+        GF_ASSERT (dict);
+
+        if (wordcount <= cmdi + 1) {
+                cli_err ("Invalid Syntax.");
+                gf_log ("cli", GF_LOG_ERROR,
+                        "Too less words for snap create command");
+                goto out;
+        }
+
+        if (strlen(words[cmdi]) >= GLUSTERD_MAX_SNAP_NAME) {
+                cli_err ("snapshot create: failed: snapname cannot exceed "
+                         "255 characters.");
+                gf_log ("cli", GF_LOG_ERROR, "Snapname too long");
+
+                goto out;
+        }
+
+        snapname = (char *) words[cmdi];
+        for (i = 0 ; i < strlen (snapname); i++) {
+                /* Following volume name convention */
+                if (!isalnum (snapname[i]) && (snapname[i] != '_'
+                                           && (snapname[i] != '-'))) {
+                        /* TODO : Is this message enough?? */
+                        cli_err ("Snapname can contain only alphanumeric, "
+                                 "\"-\" and \"_\" characters");
+                        goto out;
+                }
+        }
+
+        ret = dict_set_str (dict, "snapname", (char *)words[cmdi]);
+        if (ret) {
+                gf_log ("cli", GF_LOG_ERROR, "Could not save snap "
+                        "name");
+                goto out;
+        }
+
+        /* Filling volume name in the dictionary */
+        for (i = cmdi + 1 ; i < wordcount
+                            && (strcmp (words[i], "description")) != 0
+                            && (strcmp (words[i], "force") != 0); i++) {
+                volcount++;
+                /* volume index starts from 1 */
+                ret = snprintf (key, sizeof (key),"volname%ld", volcount);
+                if (ret < 0) {
+                        goto out;
+                }
+
+                ret = dict_set_str (dict, key, (char *)words[i]);
+                if (ret) {
+                        gf_log ("cli", GF_LOG_ERROR, "Could not "
+                                "save volume name");
+                        goto out;
+                }
+
+                if (i >= cmdi + 2) {
+                        ret = -1;
+                        cli_err("Creating multiple volume snapshot is not "
+                                "supported as of now");
+                        goto out;
+                }
+                /* TODO : remove this above condition check once
+                 * multiple volume snapshot is supported */
+        }
+
+        if (volcount == 0) {
+                ret = -1;
+                cli_err ("Please provide the volume name");
+                gf_log ("cli", GF_LOG_ERROR, "Invalid Syntax");
+                goto out;
+        }
+
+        ret = dict_set_int32 (dict, "volcount", volcount);
+        if (ret) {
+                gf_log ("cli", GF_LOG_ERROR, "Could not save volcount");
+                goto out;
+        }
+
+        /* Verify how we got out of "for" loop,
+         * if it is by reaching wordcount limit then goto "out",
+         * because we need not parse for "description" and "force"
+         * after this.
+         */
+        if (i == wordcount) {
+                goto out;
+        }
+
+        if ((strcmp (words[i], "description")) == 0) {
+                ++i;
+                if (i > (wordcount - 1)) {
+                        ret = -1;
+                        cli_err ("Please provide a description");
+                        gf_log ("cli", GF_LOG_ERROR,
+                                "Description not provided");
+                        goto out;
+                }
+
+                ret = cli_snap_create_desc_parse(dict, words, wordcount, i);
+                if (ret) {
+                        gf_log ("cli", GF_LOG_ERROR, "Could not save snap "
+                                "description");
+                        goto out;
+                }
+
+                if ( i == (wordcount - 1))
+                        goto out;
+                i++;
+                /* point the index to next word.
+                 * As description might be follwed by force option.
+                 * Before that, check if wordcount limit is reached
+                 */
+        }
+
+        if ((strcmp (words[i], "force") != 0)) {
+                ret = -1;
+                cli_err ("Invalid Syntax.");
+                gf_log ("cli", GF_LOG_ERROR, "Invalid Syntax");
+                goto out;
+        }
+        ret = dict_set_int8 (dict, "snap-force", 1);
+        if (ret) {
+                gf_log ("cli", GF_LOG_ERROR, "Could not save "
+                        "snap force option");
+                goto out;
+        }
+
+        /* Check if the command has anything after "force" keyword */
+        if (++i < wordcount) {
+                ret = -1;
+                gf_log ("cli", GF_LOG_ERROR, "Invalid Syntax");
+                goto out;
+        }
+
+        ret = 0;
+
+out :
+        return ret;
+}
+
+/* snapshot list [volname]
+ * @arg-0, dict     : Request Dictionary to be sent to server side.
+ * @arg-1, words    : Contains individual words of CLI command.
+ * @arg-2, wordcount: Contains number of words present in the CLI command.
+ *
+ * return value : -1 on failure
+ *                 0 on success
+ */
+int
+cli_snap_list_parse (dict_t *dict, const char **words, int wordcount) {
+        int             ret     =       -1;
+
+        GF_ASSERT (words);
+        GF_ASSERT (dict);
+
+        if (wordcount < 2 || wordcount > 3) {
+                gf_log ("cli", GF_LOG_ERROR, "Invalid Syntax");
+                goto out;
+        }
+
+        if (wordcount == 2) {
+                ret = 0;
+                goto out;
+        }
+
+        ret = dict_set_str (dict, "volname", (char *)words[2]);
+        if (ret) {
+                gf_log ("cli", GF_LOG_ERROR,
+                        "Failed to save volname in dictionary");
+                goto out;
+        }
+out :
+        return ret;
+}
+
+/* snapshot info [(snapname |  volume <volname>)]
+ * @arg-0, dict     : Request Dictionary to be sent to server side.
+ * @arg-1, words    : Contains individual words of CLI command.
+ * @arg-2, wordcount: Contains number of words present in the CLI command.
+ *
+ * return value : -1 on failure
+ *                 0 on success
+ */
+int
+cli_snap_info_parse (dict_t *dict, const char **words, int wordcount)
+{
+
+        int             ret             =       -1;
+        int32_t         cmd             =       GF_SNAP_INFO_TYPE_ALL;
+        unsigned int    cmdi            =        2;
+        /* cmdi is command index, here cmdi is "2" (gluster snapshot info)*/
+
+        GF_ASSERT (words);
+        GF_ASSERT (dict);
+
+        if (wordcount > 4 || wordcount < cmdi) {
+                gf_log ("", GF_LOG_ERROR, "Invalid syntax");
+                goto out;
+        }
+
+        if (wordcount == cmdi) {
+                ret = 0;
+                goto out;
+        }
+
+        /* If 3rd word is not "volume", then it must
+         * be snapname.
+         */
+        if (strcmp (words[cmdi], "volume") != 0) {
+                ret = dict_set_str (dict, "snapname",
+                                   (char *)words[cmdi]);
+                if (ret) {
+                        gf_log ("cli", GF_LOG_ERROR, "Unable to save "
+                                "snapname %s", words[cmdi]);
+                        goto out;
+                }
+
+                /* Once snap name is parsed, if we encounter any other
+                 * word then fail it. Invalid Syntax.
+                 * example : snapshot info <snapname> word
+                 */
+                if ((cmdi + 1) != wordcount) {
+                        ret = -1;
+                        gf_log ("cli", GF_LOG_ERROR, "Invalid Syntax");
+                        goto out;
+                }
+
+                cmd = GF_SNAP_INFO_TYPE_SNAP;
+                ret = 0;
+                goto out;
+                /* No need to continue the parsing once we
+                 * get the snapname
+                 */
+        }
+
+        /* If 3rd word is "volume", then check if next word
+         * is present. As, "snapshot info volume" is an
+         * invalid command.
+         */
+        if ((cmdi + 1) == wordcount) {
+                ret = -1;
+                gf_log ("cli", GF_LOG_ERROR, "Invalid Syntax");
+                goto out;
+        }
+
+        ret = dict_set_str (dict, "volname", (char *)words[wordcount - 1]);
+        if (ret) {
+                gf_log ("", GF_LOG_ERROR, "Count not save "
+                        "volume name %s", words[wordcount - 1]);
+                goto out;
+        }
+        cmd = GF_SNAP_INFO_TYPE_VOL;
+out :
+        if (ret == 0) {
+                ret = dict_set_int32 (dict, "cmd", cmd);
+                if (ret) {
+                        gf_log ("cli", GF_LOG_ERROR, "Could not save "
+                                "type of snapshot info");
+                }
+        }
+        return ret;
+}
+
+
+
+/* snapshot restore <snapname>
+ * @arg-0, dict     : Request Dictionary to be sent to server side.
+ * @arg-1, words    : Contains individual words of CLI command.
+ * @arg-2, wordcount: Contains number of words present in the CLI command.
+ *
+ * return value : -1 on failure
+ *                 0 on success
+ */
+int
+cli_snap_restore_parse (dict_t *dict, const char **words, int wordcount)
+{
+
+        int             ret             =       -1;
+
+        GF_ASSERT (words);
+        GF_ASSERT (dict);
+
+        if (wordcount != 3) {
+                gf_log ("cli", GF_LOG_ERROR, "Invalid Syntax");
+                goto out;
+        }
+
+        ret = dict_set_str (dict, "snapname", (char *)words[2]);
+        if (ret) {
+                gf_log ("cli", GF_LOG_ERROR, "Unable to save snap-name %s",
+                        words[2]);
+                goto out;
+        }
+out :
+        return ret;
+}
+
+/* snapshot delete <snapname>
+ * @arg-0, dict     : Request Dictionary to be sent to server side.
+ * @arg-1, words    : Contains individual words of CLI command.
+ * @arg-2, wordcount: Contains number of words present in the CLI command.
+ *
+ * return value : -1 on failure
+ *                 0 on success
+ *                 1 if user cancel the operation
+ */
+int
+cli_snap_delete_parse (dict_t *dict, const char **words, int wordcount,
+                       struct cli_state *state) {
+
+        int             ret             =       -1;
+        const char      *question       =       NULL;
+        gf_answer_t     answer          =       GF_ANSWER_NO;
+
+        question = "Deleting snap will erase all the information about "
+                   "the snap. Do you still want to continue?";
+
+        GF_ASSERT (words);
+        GF_ASSERT (dict);
+
+        if (wordcount != 3) {
+                gf_log ("cli", GF_LOG_ERROR, "Invalid Syntax");
+                goto out;
+        }
+
+        ret = dict_set_str (dict, "snapname", (char *)words[2]);
+        if (ret) {
+                gf_log ("cli", GF_LOG_ERROR, "Unable to save snapname %s",
+                        words[2]);
+                goto out;
+        }
+
+        answer = cli_cmd_get_confirmation (state, question);
+        if (GF_ANSWER_NO == answer) {
+                ret = 1;
+                gf_log ("cli", GF_LOG_DEBUG, "User cancelled "
+                        "snapshot delete operation");
+                goto out;
+        }
+out :
+        return ret;
+}
+
+/* snapshot status [(snapname | volume <volname>)]
+ * @arg-0, dict     : Request Dictionary to be sent to server side.
+ * @arg-1, words    : Contains individual words of CLI command.
+ * @arg-2, wordcount: Contains number of words present in the CLI command.
+ *
+ * return value : -1 on failure
+ *                 0 on success
+ */
+int
+cli_snap_status_parse (dict_t *dict, const char **words, int wordcount)
+{
+
+        int             ret  =        -1;
+        int32_t         cmd  =       GF_SNAP_STATUS_TYPE_ALL;
+        unsigned int    cmdi =        2;
+        /* cmdi is command index, here cmdi is "2" (gluster snapshot status)*/
+
+        GF_ASSERT (words);
+        GF_ASSERT (dict);
+
+        if (wordcount > 4 || wordcount < cmdi) {
+                gf_log ("cli", GF_LOG_ERROR, "Invalid Syntax");
+                goto out;
+        }
+
+        if (wordcount == cmdi) {
+                ret = 0;
+                goto out;
+        }
+
+        /* if 3rd word is not "volume", then it must be "snapname"
+        */
+        if (strcmp (words[cmdi], "volume") != 0) {
+                ret = dict_set_str (dict, "snapname",
+                                   (char *)words[cmdi]);
+                if (ret) {
+                        gf_log ("cli", GF_LOG_ERROR, "Count not save "
+                                "snap name %s", words[cmdi]);
+                        goto out;
+                }
+
+                if ((cmdi + 1) != wordcount) {
+                        ret = -1;
+                        gf_log ("cli", GF_LOG_ERROR, "Invalid Syntax");
+                        goto out;
+                }
+
+                ret = 0;
+                cmd = GF_SNAP_STATUS_TYPE_SNAP;
+                goto out;
+        }
+
+        /* If 3rd word is "volume", then check if next word is present.
+         * As, "snapshot info volume" is an invalid command
+         */
+        if ((cmdi + 1) == wordcount) {
+                ret = -1;
+                gf_log ("cli", GF_LOG_ERROR, "Invalid Syntax");
+                goto out;
+        }
+
+        ret = dict_set_str (dict, "volname", (char *)words [wordcount - 1]);
+        if (ret) {
+                gf_log ("cli", GF_LOG_ERROR, "Count not save "
+                        "volume name %s", words[wordcount - 1]);
+                goto out;
+        }
+        cmd = GF_SNAP_STATUS_TYPE_VOL;
+
+out :
+        if (ret == 0) {
+                ret = dict_set_int32 (dict, "cmd", cmd);
+                if (ret) {
+                        gf_log ("cli", GF_LOG_ERROR, "Could not save cmd "
+                                "of snapshot status");
+                }
+        }
+        return ret;
+}
+
+
+int32_t
+cli_snap_config_limit_parse (const char **words, dict_t *dict,
+                             unsigned int wordcount, unsigned int index,
+                             char *key)
+{
+        int             ret             =       -1;
+        int             limit           =       0;
+
+        GF_ASSERT (words);
+        GF_ASSERT (dict);
+        GF_ASSERT (key);
+
+        if (index >= wordcount) {
+                ret = -1;
+                cli_err ("Please provide a value for %s.",key);
+                gf_log ("cli", GF_LOG_ERROR, "Value not provided for %s", key);
+                goto out;
+        }
+
+        limit = strtol (words[index], NULL, 0);
+        if (limit <= 0) {
+                ret = -1;
+                cli_err ("%s should be greater than 0.", key);
+                goto out;
+        }
+
+        ret = dict_set_int32 (dict, key, limit);
+        if (ret) {
+                gf_log ("cli", GF_LOG_ERROR, "Could not set "
+                        "%s in dictionary", key);
+                goto out;
+        }
+
+out :
+        return ret;
+}
+
+/* function cli_snap_config_parse
+ * Config Syntax : gluster snapshot config [volname]
+ *                                         [snap-max-hard-limit <count>]
+ *                                         [snap-max-soft-limit <count>]
+ *
+   return value: <0  on failure
+                  1  if user cancels the operation
+                  0  on success
+
+  NOTE : snap-max-soft-limit can only be set for system.
+*/
+int32_t
+cli_snap_config_parse (const char **words, int wordcount, dict_t *dict,
+                       struct cli_state *state)
+{
+        int                            ret                 = -1;
+        gf_answer_t                    answer              = GF_ANSWER_NO;
+        gf_boolean_t                   vol_presence        = _gf_false;
+        struct snap_config_opt_vals_  *conf_vals           = NULL;
+        int8_t                         hard_limit          = 0;
+        int8_t                         soft_limit          = 0;
+        int8_t                         config_type         = -1;
+        const char                    *question            = NULL;
+        unsigned int                    cmdi               = 2;
+        /* cmdi is command index, here cmdi is "2" (gluster snapshot config)*/
+
+        GF_ASSERT (words);
+        GF_ASSERT (dict);
+        GF_ASSERT (state);
+
+        if ((wordcount < 2) || (wordcount > 7)) {
+                gf_log ("cli", GF_LOG_ERROR,
+                        "Invalid wordcount(%d)", wordcount);
+                goto out;
+        }
+
+        if (wordcount == 2) {
+                config_type = GF_SNAP_CONFIG_DISPLAY;
+                ret = 0;
+                goto set;
+        }
+
+        /* Check whether the 3rd word is volname */
+        if (strcmp (words[cmdi], "snap-max-hard-limit") != 0
+             && strcmp (words[cmdi], "snap-max-soft-limit") != 0) {
+                ret = dict_set_str (dict, "volname", (char *)words[cmdi]);
+                if (ret) {
+                        gf_log ("cli", GF_LOG_ERROR, "Failed to set volname");
+                        goto out;
+                }
+                cmdi++;
+                vol_presence = _gf_true;
+
+                if (cmdi == wordcount) {
+                        config_type = GF_SNAP_CONFIG_DISPLAY;
+                        ret = 0;
+                        goto set;
+                }
+        }
+
+        config_type = GF_SNAP_CONFIG_TYPE_SET;
+
+        if (strcmp (words[cmdi], "snap-max-hard-limit") == 0) {
+                ret = cli_snap_config_limit_parse (words, dict, wordcount,
+                                                ++cmdi, "snap-max-hard-limit");
+                if (ret) {
+                        gf_log ("cli", GF_LOG_ERROR, "Failed to parse snap "
+                                "config hard limit");
+                        goto out;
+                }
+                hard_limit = 1;
+
+                if (++cmdi == wordcount) {
+                        ret = 0;
+                        goto set;
+                }
+        }
+
+        if (strcmp (words[cmdi], "snap-max-soft-limit") == 0) {
+                if (vol_presence == 1) {
+                        ret = -1;
+                        cli_err ("Soft limit cannot be set to individual "
+                                  "volumes.");
+                        gf_log ("cli", GF_LOG_ERROR, "Soft limit cannot be "
+                                "set to volumes");
+                        goto out;
+                }
+
+                ret = cli_snap_config_limit_parse (words, dict, wordcount,
+                                                ++cmdi, "snap-max-soft-limit");
+                if (ret) {
+                        gf_log ("cli", GF_LOG_ERROR, "Failed to parse snap "
+                                "config soft limit");
+                        goto out;
+                }
+
+                if (++cmdi != wordcount) {
+                        ret = -1;
+                        gf_log ("cli", GF_LOG_ERROR, "Invalid Syntax");
+                        goto out;
+                }
+                soft_limit = 1;
+        } else {
+                ret = -1;
+                gf_log ("cli", GF_LOG_ERROR, "Invalid Syntax");
+                goto out;
+        }
+        ret = 0; /* Success */
+
+set:
+        ret = dict_set_int32 (dict, "config-command", config_type);
+        if (ret) {
+                gf_log ("cli", GF_LOG_ERROR, "Unable to set "
+                        "config-command");
+                goto out;
+        }
+
+        if (config_type == GF_SNAP_CONFIG_TYPE_SET) {
+                conf_vals = snap_confopt_vals;
+                if (hard_limit && soft_limit) {
+                        question = conf_vals[GF_SNAP_CONFIG_SET_BOTH].question;
+                } else if (soft_limit) {
+                        question = conf_vals[GF_SNAP_CONFIG_SET_SOFT].question;
+                } else if (hard_limit) {
+                        question = conf_vals[GF_SNAP_CONFIG_SET_HARD].question;
+                }
+
+                answer = cli_cmd_get_confirmation (state, question);
+                if (GF_ANSWER_NO == answer) {
+                        ret = 1;
+                        gf_log ("cli", GF_LOG_DEBUG, "User cancelled "
+                        "snapshot config operation");
+                }
+        }
+
+out:
+        return ret;
+}
+
+int
+validate_snapname (const char *snapname, char **opwords) {
+        int     ret     =       -1;
+        int     i       =       0;
+
+        GF_ASSERT (snapname);
+        GF_ASSERT (opwords);
+
+        for (i = 0 ; opwords[i] != NULL; i++) {
+                if (strcmp (opwords[i], snapname) == 0) {
+                        cli_out ("\"%s\" cannot be a snapname", snapname);
+                        goto out;
+                }
+        }
+        ret = 0;
+out :
+        return ret;
+}
+
+int32_t
+cli_cmd_snapshot_parse (const char **words, int wordcount, dict_t **options,
+                        struct cli_state *state)
+{
+        int32_t            ret        = -1;
+        dict_t             *dict      = NULL;
+        gf1_cli_snapshot   type       = GF_SNAP_OPTION_TYPE_NONE;
+        char               *w         = NULL;
+        char               *opwords[] = {"create", "delete", "restore", "start",
+                                         "stop", "list", "status", "config",
+                                         "info", NULL};
+        char               *invalid_snapnames[] = {"description", "force",
+                                                  "volume", NULL};
+
+        GF_ASSERT (words);
+        GF_ASSERT (options);
+        GF_ASSERT (state);
+
+        dict = dict_new ();
+        if (!dict)
+                goto out;
+
+        /* Lowest wordcount possible */
+        if (wordcount < 2) {
+                gf_log ("", GF_LOG_ERROR,
+                        "Invalid command: Not enough arguments");
+                goto out;
+        }
+
+        w = str_getunamb (words[1], opwords);
+        if (!w) {
+                /* Checks if the operation is a valid operation */
+                gf_log ("", GF_LOG_ERROR, "Opword Mismatch");
+                goto out;
+        }
+
+        if (!strcmp (w, "create")) {
+                type = GF_SNAP_OPTION_TYPE_CREATE;
+        } else if (!strcmp (w, "list")) {
+                type = GF_SNAP_OPTION_TYPE_LIST;
+        } else if (!strcmp (w, "info")) {
+                type = GF_SNAP_OPTION_TYPE_INFO;
+        } else if (!strcmp (w, "delete")) {
+                type = GF_SNAP_OPTION_TYPE_DELETE;
+        } else if (!strcmp (w, "config")) {
+                type = GF_SNAP_OPTION_TYPE_CONFIG;
+        } else if (!strcmp (w, "restore")) {
+                type = GF_SNAP_OPTION_TYPE_RESTORE;
+        } else if (!strcmp (w, "status")) {
+                type = GF_SNAP_OPTION_TYPE_STATUS;
+        }
+
+        if (type != GF_SNAP_OPTION_TYPE_CONFIG) {
+                ret = dict_set_int32 (dict, "hold_snap_locks", _gf_true);
+                if (ret) {
+                        gf_log ("cli", GF_LOG_ERROR,
+                                "Unable to set hold-snap-locks value "
+                                "as _gf_true");
+                        goto out;
+                }
+        }
+
+        /* Check which op is intended */
+        switch (type) {
+               case GF_SNAP_OPTION_TYPE_CREATE:
+               {
+                        /* Syntax :
+                         * gluster snapshot create <snapname> <vol-name(s)>
+                         *                         [description <description>]
+                         *                         [force]
+                         */
+
+                        /* In cases where the snapname is not given then
+                         * parsing fails & snapname cannot be "description",
+                         * "force" and "volume", that check is made here
+                         */
+                       if (wordcount == 2){
+                                ret = -1;
+                                gf_log ("cli", GF_LOG_ERROR,
+                                        "Invalid Syntax");
+                                goto out;
+                       }
+
+                       ret = validate_snapname (words[2], invalid_snapnames);
+                       if (ret) {
+                               goto out;
+                       }
+
+                       ret = cli_snap_create_parse (dict, words, wordcount);
+                       if (ret) {
+                               gf_log ("cli", GF_LOG_ERROR,
+                                       "create command parsing failed.");
+                               goto out;
+                       }
+                       break;
+               }
+                case GF_SNAP_OPTION_TYPE_INFO:
+                {
+                        /* Syntax :
+                         * gluster snapshot info [(snapname] | [vol <volname>)]
+                         */
+                        ret = cli_snap_info_parse (dict, words, wordcount);
+                        if (ret) {
+                                gf_log ("cli", GF_LOG_ERROR, "Failed to parse "
+                                        "snapshot info command");
+                                goto out;
+                        }
+                        break;
+                }
+
+                case GF_SNAP_OPTION_TYPE_LIST:
+                {
+                        /* Syntax :
+                         * gluster snaphsot list [volname]
+                         */
+
+                        ret = cli_snap_list_parse (dict, words, wordcount);
+                        if (ret) {
+                                gf_log ("cli", GF_LOG_ERROR, "Failed to parse "
+                                        "snapshot list command");
+                                goto out;
+                        }
+                        break;
+                }
+
+                case GF_SNAP_OPTION_TYPE_DELETE:
+                {
+                        /* Syntax :
+                         * gluster snapshot delete <snapname>
+                         */
+                        ret = cli_snap_delete_parse (dict, words, wordcount,
+                                                     state);
+                        if (ret) {
+                                gf_log ("cli", GF_LOG_ERROR, "Failed to parse "
+                                        "snapshot delete command");
+                                goto out;
+                        }
+                        break;
+                }
+
+               case GF_SNAP_OPTION_TYPE_CONFIG:
+               {
+                /* snapshot config [volname]  [snap-max-hard-limit <count>]
+                 *                            [snap-max-soft-limit <percent>] */
+                       ret = cli_snap_config_parse (words, wordcount, dict,
+                                                    state);
+                       if (ret) {
+                                if (ret < 0)
+                                        gf_log ("cli", GF_LOG_ERROR,
+                                              "config command parsing failed.");
+                               goto out;
+                       }
+
+                       ret = dict_set_int32 (dict, "type",
+                                             GF_SNAP_OPTION_TYPE_CONFIG);
+                       if (ret) {
+                               gf_log ("cli", GF_LOG_ERROR, "Unable to set "
+                                       "config type");
+                               ret = -1;
+                               goto out;
+                       }
+                       break;
+               }
+                case GF_SNAP_OPTION_TYPE_STATUS:
+                {
+                        /* Syntax :
+                         * gluster snapshot status [(snapname |
+                         *                         volume <volname>)]
+                         */
+                        ret = cli_snap_status_parse (dict, words, wordcount);
+                        if (ret) {
+                                gf_log ("cli", GF_LOG_ERROR, "Failed to parse "
+                                        "snapshot status command");
+                                goto out;
+                        }
+                        break;
+                }
+
+               case GF_SNAP_OPTION_TYPE_RESTORE:
+               {
+                       /* Syntax:
+                        * snapshot restore <snapname>
+                        */
+                       ret = cli_snap_restore_parse (dict, words, wordcount);
+                       if (ret) {
+                               gf_log ("cli", GF_LOG_ERROR, "Failed to parse "
+                                       "restore command");
+                               goto out;
+                       }
+                       break;
+               }
+               default:
+                       gf_log ("", GF_LOG_ERROR, "Opword Mismatch");
+                       goto out;
+                       break;
+        }
+
+        ret = dict_set_int32 (dict, "type", type);
+        if (ret) {
+                gf_log ("", GF_LOG_ERROR,
+                        "Failed to set type.");
+                goto out;
+        }
+        /* If you got so far, input is valid */
+        ret = 0;
+out:
+        if (ret) {
+                if (dict)
+                        dict_destroy (dict);
+        } else
+                *options = dict;
+
+        return ret;
+}
diff --git a/cli/src/cli-cmd-peer.c b/cli/src/cli-cmd-peer.c
index cad520dfc..551312411 100644
--- a/cli/src/cli-cmd-peer.c
+++ b/cli/src/cli-cmd-peer.c
@@ -1,22 +1,12 @@
 /*
-  Copyright (c) 2010-2011 Gluster, Inc. <http://www.gluster.com>
-  This file is part of GlusterFS.
-
-  GlusterFS is free software; you can redistribute it and/or modify
-  it under the terms of the GNU General Public License as published
-  by the Free Software Foundation; either version 3 of the License,
-  or (at your option) any later version.
-
-  GlusterFS is distributed in the hope that it will be useful, but
-  WITHOUT ANY WARRANTY; without even the implied warranty of
-  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-  General Public License for more details.
-
-  You should have received a copy of the GNU General Public License
-  along with this program.  If not, see
-  <http://www.gnu.org/licenses/>.
-*/
+   Copyright (c) 2010-2012 Red Hat, Inc. <http://www.redhat.com>
+   This file is part of GlusterFS.
 
+   This file is licensed to you under your choice of the GNU Lesser
+   General Public License, version 3 or any later version (LGPLv3 or
+   later), or the GNU General Public License, version 2 (GPLv2), in all
+   cases as published by the Free Software Foundation.
+*/
 #include <stdio.h>
 #include <string.h>
 #include <stdlib.h>
@@ -51,6 +41,7 @@ cli_cmd_peer_probe_cbk (struct cli_state *state, struct cli_cmd_word *word,
         dict_t                  *dict = NULL;
         int                     sent = 0;
         int                     parse_error = 0;
+        cli_local_t             *local = NULL;
 
         if (!(wordcount == 3)) {
                 cli_usage_out (word->pattern);
@@ -72,12 +63,14 @@ cli_cmd_peer_probe_cbk (struct cli_state *state, struct cli_cmd_word *word,
         if (ret)
                 goto out;
 
-        ret = valid_internet_address ((char *) words[2]);
+        ret = valid_internet_address ((char *) words[2], _gf_false);
         if (ret == 1) {
                 ret = 0;
         } else {
+                cli_out ("%s is an invalid address", words[2]);
                 cli_usage_out (word->pattern);
                 parse_error = 1;
+                ret = -1;
                 goto out;
         }
 /*        if (words[3]) {
@@ -86,6 +79,9 @@ cli_cmd_peer_probe_cbk (struct cli_state *state, struct cli_cmd_word *word,
                         goto out;
         }
 */
+
+        CLI_LOCAL_INIT (local, words, frame, dict);
+
         if (proc->fn) {
                 ret = proc->fn (frame, THIS, dict);
         }
@@ -96,6 +92,9 @@ out:
                 if ((sent == 0) && (parse_error == 0))
                         cli_out ("Peer probe failed");
         }
+
+        CLI_STACK_DESTROY (frame);
+
         return ret;
 }
 
@@ -111,6 +110,7 @@ cli_cmd_peer_deprobe_cbk (struct cli_state *state, struct cli_cmd_word *word,
         int                  flags = 0;
         int                  sent = 0;
         int                  parse_error = 0;
+        cli_local_t          *local = NULL;
 
         if ((wordcount < 3) || (wordcount > 4)) {
                 cli_usage_out (word->pattern);
@@ -150,6 +150,8 @@ cli_cmd_peer_deprobe_cbk (struct cli_state *state, struct cli_cmd_word *word,
         if (ret)
                 goto out;
 
+        CLI_LOCAL_INIT (local, words, frame, dict);
+
         if (proc->fn) {
                 ret = proc->fn (frame, THIS, dict);
         }
@@ -161,6 +163,8 @@ out:
                         cli_out ("Peer detach failed");
         }
 
+        CLI_STACK_DESTROY (frame);
+
         return ret;
 }
 
@@ -187,7 +191,7 @@ cli_cmd_peer_status_cbk (struct cli_state *state, struct cli_cmd_word *word,
                 goto out;
 
         if (proc->fn) {
-                ret = proc->fn (frame, THIS, (char *)words[1] );
+                ret = proc->fn (frame, THIS, (void *)GF_CLI_LIST_PEERS);
         }
 
 out:
@@ -196,6 +200,48 @@ out:
                 if ((sent == 0) && (parse_error == 0))
                         cli_out ("Peer status failed");
         }
+
+        CLI_STACK_DESTROY (frame);
+
+        return ret;
+}
+
+int
+cli_cmd_pool_list_cbk (struct cli_state *state, struct cli_cmd_word *word,
+                       const char **words, int wordcount)
+{
+        int                     ret = -1;
+        rpc_clnt_procedure_t    *proc = NULL;
+        call_frame_t            *frame = NULL;
+        int                     sent = 0;
+        int                     parse_error = 0;
+
+        if (wordcount != 2) {
+                cli_usage_out (word->pattern);
+                parse_error = 1;
+                goto out;
+        }
+
+        proc = &cli_rpc_prog->proctable[GLUSTER_CLI_LIST_FRIENDS];
+
+        frame = create_frame (THIS, THIS->ctx->pool);
+        if (!frame)
+                goto out;
+
+        if (proc->fn) {
+                ret = proc->fn (frame, THIS,
+                                (void *)GF_CLI_LIST_POOL_NODES);
+        }
+
+out:
+        if (ret) {
+                cli_cmd_sent_status_get (&sent);
+                if ((sent == 0) && (parse_error == 0))
+                        cli_err ("pool list: command execution failed");
+        }
+
+        CLI_STACK_DESTROY (frame);
+
         return ret;
 }
 
@@ -216,6 +262,10 @@ struct cli_cmd cli_probe_cmds[] = {
            cli_cmd_peer_help_cbk,
            "Help command for peer "},
 
+        { "pool list",
+          cli_cmd_pool_list_cbk,
+          "list all the nodes in the pool (including localhost)"},
+
         { NULL, NULL, NULL }
 };
 
diff --git a/cli/src/cli-cmd-snapshot.c b/cli/src/cli-cmd-snapshot.c
new file mode 100644
index 000000000..de492d683
--- /dev/null
+++ b/cli/src/cli-cmd-snapshot.c
@@ -0,0 +1,146 @@
+/*
+   Copyright (c) 2013-2014 Red Hat, Inc. <http://www.redhat.com>
+   This file is part of GlusterFS.
+
+   This file is licensed to you under your choice of the GNU Lesser
+   General Public License, version 3 or any later version (LGPLv3 or
+   later), or the GNU General Public License, version 2 (GPLv2), in all
+   cases as published by the Free Software Foundation.
+*/
+#include <stdio.h>
+#include <string.h>
+#include <stdlib.h>
+#include <stdint.h>
+#include <pthread.h>
+
+#ifndef _CONFIG_H
+#define _CONFIG_H
+#include "config.h"
+#endif
+
+#include "cli.h"
+#include "cli-cmd.h"
+
+extern rpc_clnt_prog_t *cli_rpc_prog;
+
+int
+cli_cmd_snapshot_help_cbk (struct cli_state *state, struct cli_cmd_word *in_word,
+                           const char **words, int wordcount);
+
+int
+cli_cmd_snapshot_cbk (struct cli_state *state, struct cli_cmd_word *word,
+                      const char **words, int wordcount)
+{
+        int                      ret       = 0;
+        int                      parse_err = 0;
+        dict_t                  *options   = NULL;
+        rpc_clnt_procedure_t    *proc      = NULL;
+        call_frame_t            *frame     = NULL;
+        cli_local_t             *local     = NULL;
+
+        proc = &cli_rpc_prog->proctable [GLUSTER_CLI_SNAP];
+        if (proc == NULL) {
+               ret = -1;
+                goto out;
+        }
+
+        frame = create_frame (THIS, THIS->ctx->pool);
+        if (frame == NULL) {
+                ret = -1;
+                goto out;
+        }
+
+        /* Parses the command entered by the user */
+        ret = cli_cmd_snapshot_parse (words, wordcount, &options, state);
+        if (ret) {
+                if (ret < 0) {
+                        cli_usage_out (word->pattern);
+                        parse_err = 1;
+                }
+                else {
+                        /* User might have cancelled the snapshot operation */
+                        ret = 0;
+                }
+                goto out;
+        }
+
+        CLI_LOCAL_INIT (local, words, frame, options);
+
+        if (proc->fn)
+                ret = proc->fn (frame, THIS, options);
+
+out:
+        if (ret && parse_err == 0)
+                cli_out ("Snapshot command failed");
+
+        CLI_STACK_DESTROY (frame);
+
+        return ret;
+}
+
+struct cli_cmd snapshot_cmds[] = {
+        { "snapshot help",
+          cli_cmd_snapshot_help_cbk,
+          "display help for snapshot commands"
+        },
+        { "snapshot create <snapname> <volname(s)> [description <description>] [force]",
+          cli_cmd_snapshot_cbk,
+          "Snapshot Create."
+        },
+        { "snapshot restore <snapname>",
+          cli_cmd_snapshot_cbk,
+          "Snapshot Restore."
+        },
+        { "snapshot status [(snapname | volume <volname>)]",
+          cli_cmd_snapshot_cbk,
+          "Snapshot Status."
+        },
+        { "snapshot info [(snapname | volume <volname>)]",
+          cli_cmd_snapshot_cbk,
+          "Snapshot Info."
+        },
+        { "snapshot list [volname]",
+          cli_cmd_snapshot_cbk,
+          "Snapshot List."
+        },
+        {"snapshot config [volname] [snap-max-hard-limit <count>] [snap-max-soft-limit <percent>]",
+          cli_cmd_snapshot_cbk,
+          "Snapshot Config."
+        },
+        {"snapshot delete <snapname>",
+          cli_cmd_snapshot_cbk,
+          "Snapshot Delete."
+        },
+        { NULL, NULL, NULL }
+};
+
+int
+cli_cmd_snapshot_help_cbk (struct cli_state *state,
+                           struct cli_cmd_word *in_word,
+                           const char **words,
+                           int wordcount)
+{
+        struct cli_cmd        *cmd = NULL;
+
+        for (cmd = snapshot_cmds; cmd->pattern; cmd++)
+                if (_gf_false == cmd->disable)
+                        cli_out ("%s - %s", cmd->pattern, cmd->desc);
+
+        return 0;
+}
+
+int
+cli_cmd_snapshot_register (struct cli_state *state)
+{
+        int  ret = 0;
+        struct cli_cmd *cmd = NULL;
+
+        for (cmd = snapshot_cmds; cmd->pattern; cmd++) {
+
+                ret = cli_cmd_register (&state->tree, cmd);
+                if (ret)
+                        goto out;
+        }
+out:
+        return ret;
+}
diff --git a/cli/src/cli-cmd-system.c b/cli/src/cli-cmd-system.c
index 25938b897..8cfa5e70c 100644
--- a/cli/src/cli-cmd-system.c
+++ b/cli/src/cli-cmd-system.c
@@ -1,22 +1,12 @@
 /*
-  Copyright (c) 2010-2011 Gluster, Inc. <http://www.gluster.com>
-  This file is part of GlusterFS.
-
-  GlusterFS is free software; you can redistribute it and/or modify
-  it under the terms of the GNU General Public License as published
-  by the Free Software Foundation; either version 3 of the License,
-  or (at your option) any later version.
-
-  GlusterFS is distributed in the hope that it will be useful, but
-  WITHOUT ANY WARRANTY; without even the implied warranty of
-  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-  General Public License for more details.
-
-  You should have received a copy of the GNU General Public License
-  along with this program.  If not, see
-  <http://www.gnu.org/licenses/>.
-*/
+   Copyright (c) 2010-2012 Red Hat, Inc. <http://www.redhat.com>
+   This file is part of GlusterFS.
 
+   This file is licensed to you under your choice of the GNU Lesser
+   General Public License, version 3 or any later version (LGPLv3 or
+   later), or the GNU General Public License, version 2 (GPLv2), in all
+   cases as published by the Free Software Foundation.
+*/
 #include <stdio.h>
 #include <string.h>
 #include <stdlib.h>
@@ -41,6 +31,12 @@ extern rpc_clnt_prog_t *cli_rpc_prog;
 int cli_cmd_system_help_cbk (struct cli_state *state, struct cli_cmd_word *in_word,
                              const char **words, int wordcount);
 
+int cli_cmd_copy_file_cbk (struct cli_state *state, struct cli_cmd_word *word,
+                           const char **words, int wordcount);
+
+int cli_cmd_sys_exec_cbk (struct cli_state *state, struct cli_cmd_word *word,
+                          const char **words, int wordcount);
+
 int
 cli_cmd_getspec_cbk (struct cli_state *state, struct cli_cmd_word *word,
                      const char **words, int wordcount)
@@ -288,6 +284,114 @@ cli_cmd_umount_cbk (struct cli_state *state, struct cli_cmd_word *word,
         return ret;
 }
 
+int
+cli_cmd_uuid_get_cbk (struct cli_state *state, struct cli_cmd_word *word,
+                      const char **words, int wordcount)
+{
+        int                     ret = -1;
+        int                     sent = 0;
+        int                     parse_error = 0;
+        dict_t                  *dict  = NULL;
+        rpc_clnt_procedure_t    *proc = NULL;
+        call_frame_t            *frame = NULL;
+        cli_local_t             *local = NULL;
+        xlator_t                *this  = NULL;
+
+        this = THIS;
+        if (wordcount != 3) {
+                cli_usage_out (word->pattern);
+                parse_error = 1;
+                goto out;
+        }
+
+        proc = &cli_rpc_prog->proctable[GLUSTER_CLI_UUID_GET];
+        frame = create_frame (this, this->ctx->pool);
+        if (!frame)
+                goto out;
+
+        dict = dict_new ();
+        if (!dict)
+                goto out;
+
+        CLI_LOCAL_INIT (local, words, frame, dict);
+        if (proc->fn)
+                ret = proc->fn (frame, this, dict);
+
+out:
+        if (ret) {
+                cli_cmd_sent_status_get (&sent);
+                if ((sent == 0) && (parse_error == 0))
+                        cli_out ("uuid get failed");
+        }
+
+        if (dict)
+                dict_unref (dict);
+
+        CLI_STACK_DESTROY (frame);
+        return ret;
+}
+
+int
+cli_cmd_uuid_reset_cbk (struct cli_state *state, struct cli_cmd_word *word,
+                        const char **words, int wordcount)
+{
+        int                     ret = -1;
+        rpc_clnt_procedure_t    *proc = NULL;
+        call_frame_t            *frame = NULL;
+        int                     sent = 0;
+        int                     parse_error = 0;
+        gf_answer_t             answer = GF_ANSWER_NO;
+        char                    *question = NULL;
+        cli_local_t             *local = NULL;
+        dict_t                  *dict  = NULL;
+        xlator_t                *this  = NULL;
+
+        question = "Resetting uuid changes the uuid of local glusterd. "
+                   "Do you want to continue?";
+
+        if (wordcount != 3) {
+                cli_usage_out (word->pattern);
+                parse_error = 1;
+                goto out;
+        }
+
+        proc = &cli_rpc_prog->proctable[GLUSTER_CLI_UUID_RESET];
+
+        this = THIS;
+        frame = create_frame (this, this->ctx->pool);
+        if (!frame)
+                goto out;
+
+        dict = dict_new ();
+        if (!dict) {
+                ret = -1;
+                goto out;
+        }
+        CLI_LOCAL_INIT (local, words, frame, dict);
+        answer = cli_cmd_get_confirmation (state, question);
+
+        if (GF_ANSWER_NO == answer) {
+                ret = 0;
+                goto out;
+        }
+
+        //send NULL as argument since no dictionary is sent to glusterd
+        if (proc->fn) {
+                ret = proc->fn (frame, this, dict);
+        }
+
+out:
+        if (ret) {
+                cli_cmd_sent_status_get (&sent);
+                if ((sent == 0) && (parse_error == 0))
+                        cli_out ("uuid reset failed");
+        }
+
+        CLI_STACK_DESTROY (frame);
+
+        return ret;
+}
+
 struct cli_cmd cli_system_cmds[] = {
         { "system:: getspec <VOLID>",
           cli_cmd_getspec_cbk,
@@ -313,14 +417,163 @@ struct cli_cmd cli_system_cmds[] = {
           cli_cmd_umount_cbk,
           "request an umount"},
 
+        { "system:: uuid get",
+          cli_cmd_uuid_get_cbk,
+          "get uuid of glusterd"},
+
+        { "system:: uuid reset",
+          cli_cmd_uuid_reset_cbk,
+          "reset the uuid of glusterd"},
+
         { "system:: help",
            cli_cmd_system_help_cbk,
            "display help for system commands"},
 
+        { "system:: copy file [<filename>]",
+           cli_cmd_copy_file_cbk,
+           "Copy file from current node's $working_dir to "
+           "$working_dir of all cluster nodes"},
+
+        { "system:: execute <command> <args>",
+           cli_cmd_sys_exec_cbk,
+           "Execute the command on all the nodes "
+           "in the cluster and display their output."},
+
         { NULL, NULL, NULL }
 };
 
 int
+cli_cmd_sys_exec_cbk (struct cli_state *state, struct cli_cmd_word *word,
+                      const char **words, int wordcount)
+{
+        char                   cmd_arg_name[PATH_MAX] = "";
+        char                  *command                = NULL;
+        char                  *saveptr                = NULL;
+        char                  *tmp                    = NULL;
+        int                    ret                    = -1;
+        int                    i                      = -1;
+        int                    cmd_args_count         = 0;
+        int                    in_cmd_args_count      = 0;
+        rpc_clnt_procedure_t  *proc                   = NULL;
+        call_frame_t          *frame                  = NULL;
+        dict_t                *dict                   = NULL;
+        cli_local_t           *local                  = NULL;
+
+        if (wordcount < 3) {
+                cli_usage_out (word->pattern);
+                goto out;
+        }
+
+        dict = dict_new ();
+        if (!dict)
+                goto out;
+
+        command = strtok_r ((char *)words[2], " ", &saveptr);
+        do {
+                tmp = strtok_r (NULL, " ", &saveptr);
+                if (tmp) {
+                        in_cmd_args_count++;
+                        memset (cmd_arg_name, '\0', sizeof(cmd_arg_name));
+                        snprintf (cmd_arg_name, sizeof(cmd_arg_name),
+                                  "cmd_arg_%d", in_cmd_args_count);
+                        ret = dict_set_str (dict, cmd_arg_name, tmp);
+                        if (ret) {
+                                gf_log ("", GF_LOG_ERROR, "Unable to set "
+                                        "%s in dict", cmd_arg_name);
+                                goto out;
+                        }
+                }
+        } while (tmp);
+
+        cmd_args_count = wordcount - 3;
+
+        ret = dict_set_str (dict, "command", command);
+        if (ret) {
+                gf_log ("", GF_LOG_ERROR, "Unable to set command in dict");
+                goto out;
+        }
+
+        for (i=1; i <= cmd_args_count; i++) {
+                in_cmd_args_count++;
+                memset (cmd_arg_name, '\0', sizeof(cmd_arg_name));
+                snprintf (cmd_arg_name, sizeof(cmd_arg_name),
+                          "cmd_arg_%d", in_cmd_args_count);
+                ret = dict_set_str (dict, cmd_arg_name,
+                                    (char *)words[2+i]);
+                if (ret) {
+                        gf_log ("", GF_LOG_ERROR, "Unable to set %s in dict",
+                               cmd_arg_name);
+                        goto out;
+                }
+        }
+
+        ret = dict_set_int32 (dict, "cmd_args_count", in_cmd_args_count);
+        if (ret) {
+                gf_log ("", GF_LOG_ERROR,
+                        "Unable to set cmd_args_count in dict");
+                goto out;
+        }
+
+        ret = dict_set_str (dict, "volname", "N/A");
+        if (ret) {
+                gf_log ("", GF_LOG_ERROR, "Unable to set volname in dict");
+                goto out;
+        }
+
+        proc = &cli_rpc_prog->proctable[GLUSTER_CLI_SYS_EXEC];
+        if (proc && proc->fn) {
+                frame = create_frame (THIS, THIS->ctx->pool);
+                if (!frame)
+                        goto out;
+                CLI_LOCAL_INIT (local, words, frame, dict);
+                ret = proc->fn (frame, THIS, (void*)dict);
+        }
+out:
+        return ret;
+}
+
+int
+cli_cmd_copy_file_cbk (struct cli_state *state, struct cli_cmd_word *word,
+                       const char **words, int wordcount)
+{
+        int                    ret      = -1;
+        rpc_clnt_procedure_t  *proc     = NULL;
+        call_frame_t          *frame    = NULL;
+        char                  *filename = "";
+        dict_t                *dict     = NULL;
+        cli_local_t           *local    = NULL;
+
+        if (wordcount != 4) {
+                cli_usage_out (word->pattern);
+                goto out;
+        }
+
+        dict = dict_new ();
+        if (!dict)
+                goto out;
+
+        filename = (char*)words[3];
+        ret = dict_set_str (dict, "source", filename);
+        if (ret)
+                 gf_log ("", GF_LOG_ERROR, "Unable to set filename in dict");
+
+        ret = dict_set_str (dict, "volname", "N/A");
+        if (ret)
+                 gf_log ("", GF_LOG_ERROR, "Unable to set volname in dict");
+
+        proc = &cli_rpc_prog->proctable[GLUSTER_CLI_COPY_FILE];
+        if (proc && proc->fn) {
+                frame = create_frame (THIS, THIS->ctx->pool);
+                if (!frame)
+                        goto out;
+                CLI_LOCAL_INIT (local, words, frame, dict);
+                ret = proc->fn (frame, THIS, (void*)dict);
+        }
+out:
+        return ret;
+}
+
+int
 cli_cmd_system_help_cbk (struct cli_state *state, struct cli_cmd_word *in_word,
                          const char **words, int wordcount)
 {
diff --git a/cli/src/cli-cmd-volume.c b/cli/src/cli-cmd-volume.c
index 9546831ab..100be0b73 100644
--- a/cli/src/cli-cmd-volume.c
+++ b/cli/src/cli-cmd-volume.c
@@ -1,22 +1,12 @@
 /*
-  Copyright (c) 2010-2011 Gluster, Inc. <http://www.gluster.com>
-  This file is part of GlusterFS.
-
-  GlusterFS is free software; you can redistribute it and/or modify
-  it under the terms of the GNU General Public License as published
-  by the Free Software Foundation; either version 3 of the License,
-  or (at your option) any later version.
-
-  GlusterFS is distributed in the hope that it will be useful, but
-  WITHOUT ANY WARRANTY; without even the implied warranty of
-  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-  General Public License for more details.
-
-  You should have received a copy of the GNU General Public License
-  along with this program.  If not, see
-  <http://www.gnu.org/licenses/>.
-*/
+   Copyright (c) 2010-2012 Red Hat, Inc. <http://www.redhat.com>
+   This file is part of GlusterFS.
 
+   This file is licensed to you under your choice of the GNU Lesser
+   General Public License, version 3 or any later version (LGPLv3 or
+   later), or the GNU General Public License, version 2 (GPLv2), in all
+   cases as published by the Free Software Foundation.
+*/
 #include <stdio.h>
 #include <string.h>
 #include <stdlib.h>
@@ -104,9 +94,8 @@ out:
                 if ((sent == 0) && (parse_error == 0))
                         cli_out ("Getting Volume information failed!");
         }
-        frame->local = NULL;
-        if (frame)
-                FRAME_DESTROY (frame);
+
+        CLI_STACK_DESTROY (frame);
 
         return ret;
 
@@ -122,6 +111,12 @@ cli_cmd_sync_volume_cbk (struct cli_state *state, struct cli_cmd_word *word,
         int                     sent = 0;
         int                     parse_error = 0;
         dict_t                  *dict = NULL;
+        cli_local_t             *local = NULL;
+        gf_answer_t             answer = GF_ANSWER_NO;
+        const char              *question = "Sync volume may make data "
+                                            "inaccessible while the sync "
+                                            "is in progress. Do you want "
+                                            "to continue?";
 
         if ((wordcount < 3) || (wordcount > 4)) {
                 cli_usage_out (word->pattern);
@@ -156,12 +151,22 @@ cli_cmd_sync_volume_cbk (struct cli_state *state, struct cli_cmd_word *word,
                 goto out;
         }
 
+        if (!(state->mode & GLUSTER_MODE_SCRIPT)) {
+                answer = cli_cmd_get_confirmation (state, question);
+                if (GF_ANSWER_NO == answer) {
+                        ret = 0;
+                        goto out;
+                }
+        }
+
         proc = &cli_rpc_prog->proctable[GLUSTER_CLI_SYNC_VOLUME];
 
         frame = create_frame (THIS, THIS->ctx->pool);
         if (!frame)
                 goto out;
 
+        CLI_LOCAL_INIT (local, words, frame, dict);
+
         if (proc->fn) {
                 ret = proc->fn (frame, THIS, dict);
         }
@@ -173,8 +178,8 @@ out:
                         cli_out ("Volume sync failed");
         }
 
-        if (dict)
-                dict_unref (dict);
+        CLI_STACK_DESTROY (frame);
+
         return ret;
 }
 
@@ -315,13 +320,11 @@ found_bad_brick_order:
 out:
         ai_list_tmp2 = NULL;
         i = 0;
-        if (brick_list_dup)
-                GF_FREE (brick_list_dup);
+        GF_FREE (brick_list_dup);
         list_for_each_entry (ai_list_tmp1, &ai_list->list, list) {
                 if (ai_list_tmp1->info)
                         freeaddrinfo (ai_list_tmp1->info);
-                if (ai_list_tmp2)
-                        free (ai_list_tmp2);
+                free (ai_list_tmp2);
                 ai_list_tmp2 = ai_list_tmp1;
         }
         free (ai_list_tmp2);
@@ -342,7 +345,7 @@ cli_cmd_volume_create_cbk (struct cli_state *state, struct cli_cmd_word *word,
         int32_t                 brick_count = 0;
         int32_t                 sub_count = 0;
         int32_t                 type = GF_CLUSTER_TYPE_NONE;
-
+        cli_local_t             *local = NULL;
 
         proc = &cli_rpc_prog->proctable[GLUSTER_CLI_CREATE_VOLUME];
 
@@ -388,19 +391,31 @@ cli_cmd_volume_create_cbk (struct cli_state *state, struct cli_cmd_word *word,
                         goto out;
                 }
         }
+
+        if (state->mode & GLUSTER_MODE_SCRIPT) {
+                ret = dict_set_int32 (options, "force", _gf_true);
+                if (ret) {
+                        gf_log ("cli", GF_LOG_ERROR, "Failed to set force "
+                                "option");
+                        goto out;
+                }
+        }
+
+        CLI_LOCAL_INIT (local, words, frame, options);
+
         if (proc->fn) {
                 ret = proc->fn (frame, THIS, options);
         }
 
 out:
-        if (options)
-                dict_unref (options);
         if (ret) {
                 cli_cmd_sent_status_get (&sent);
                 if ((sent == 0) && (parse_error == 0))
                         cli_out ("Volume create failed");
         }
 
+        CLI_STACK_DESTROY (frame);
+
         return ret;
 }
 
@@ -417,6 +432,8 @@ cli_cmd_volume_delete_cbk (struct cli_state *state, struct cli_cmd_word *word,
         const char              *question = NULL;
         int                     sent = 0;
         int                     parse_error = 0;
+        cli_local_t             *local = NULL;
+        dict_t                  *dict = NULL;
 
         question = "Deleting volume will erase all information about the volume. "
                    "Do you want to continue?";
@@ -426,6 +443,10 @@ cli_cmd_volume_delete_cbk (struct cli_state *state, struct cli_cmd_word *word,
         if (!frame)
                 goto out;
 
+        dict = dict_new ();
+        if (!dict)
+                goto out;
+
         if (wordcount != 3) {
                 cli_usage_out (word->pattern);
                 parse_error = 1;
@@ -441,8 +462,17 @@ cli_cmd_volume_delete_cbk (struct cli_state *state, struct cli_cmd_word *word,
 
         volname = (char *)words[2];
 
+        ret = dict_set_str (dict, "volname", volname);
+
+        if (ret) {
+                gf_log (THIS->name, GF_LOG_WARNING, "dict set failed");
+                goto out;
+        }
+
+        CLI_LOCAL_INIT (local, words, frame, dict);
+
         if (proc->fn) {
-                ret = proc->fn (frame, THIS, volname);
+                ret = proc->fn (frame, THIS, dict);
         }
 
 out:
@@ -452,6 +482,8 @@ out:
                         cli_out ("Volume delete failed");
         }
 
+        CLI_STACK_DESTROY (frame);
+
         return ret;
 }
 
@@ -466,6 +498,7 @@ cli_cmd_volume_start_cbk (struct cli_state *state, struct cli_cmd_word *word,
         int                     parse_error = 0;
         dict_t                  *dict = NULL;
         int                     flags = 0;
+        cli_local_t             *local = NULL;
 
         frame = create_frame (THIS, THIS->ctx->pool);
         if (!frame)
@@ -516,19 +549,21 @@ cli_cmd_volume_start_cbk (struct cli_state *state, struct cli_cmd_word *word,
 
         proc = &cli_rpc_prog->proctable[GLUSTER_CLI_START_VOLUME];
 
+        CLI_LOCAL_INIT (local, words, frame, dict);
+
         if (proc->fn) {
                 ret = proc->fn (frame, THIS, dict);
         }
 
 out:
-        if (dict)
-                dict_unref (dict);
         if (ret) {
                 cli_cmd_sent_status_get (&sent);
                 if ((sent == 0) && (parse_error == 0))
                         cli_out ("Volume start failed");
         }
 
+        CLI_STACK_DESTROY (frame);
+
         return ret;
 }
 
@@ -537,7 +572,7 @@ cli_cmd_get_confirmation (struct cli_state *state, const char *question)
 {
         char                    answer[5] = {'\0', };
         char                    flush = '\0';
-	int			len = 0;
+        size_t			len;
 
         if (state->mode & GLUSTER_MODE_SCRIPT)
                 return GF_ANSWER_YES;
@@ -551,7 +586,7 @@ cli_cmd_get_confirmation (struct cli_state *state, const char *question)
 
 	len = strlen (answer);
 
-	if (answer [len - 1] == '\n'){
+	if (len && answer [len - 1] == '\n'){
 		answer [--len] = '\0';
 	} else {
 		do{
@@ -587,6 +622,7 @@ cli_cmd_volume_stop_cbk (struct cli_state *state, struct cli_cmd_word *word,
         int                     parse_error = 0;
         dict_t                  *dict = NULL;
         char                    *volname = NULL;
+        cli_local_t             *local = NULL;
 
         const char *question = "Stopping volume will make its data inaccessible. "
                                "Do you want to continue?";
@@ -636,6 +672,8 @@ cli_cmd_volume_stop_cbk (struct cli_state *state, struct cli_cmd_word *word,
 
         proc = &cli_rpc_prog->proctable[GLUSTER_CLI_STOP_VOLUME];
 
+        CLI_LOCAL_INIT (local, words, frame, dict);
+
         if (proc->fn) {
                 ret = proc->fn (frame, THIS, dict);
         }
@@ -646,8 +684,9 @@ out:
                 if ((sent == 0) && (parse_error == 0))
                         cli_out ("Volume stop on '%s' failed", volname);
         }
-        if (dict)
-                dict_unref (dict);
+
+        CLI_STACK_DESTROY (frame);
+
         return ret;
 }
 
@@ -704,6 +743,8 @@ out:
                         cli_out ("Volume rename on '%s' failed", (char *)words[2]);
         }
 
+        CLI_STACK_DESTROY (frame);
+
         return ret;
 }
 
@@ -717,7 +758,7 @@ cli_cmd_volume_defrag_cbk (struct cli_state *state, struct cli_cmd_word *word,
         dict_t               *dict = NULL;
         int                     sent = 0;
         int                     parse_error = 0;
-        int                     index = 0;
+        cli_local_t          *local = NULL;
 #ifdef GF_SOLARIS_HOST_OS
         cli_out ("Command not supported on Solaris");
         goto out;
@@ -727,78 +768,30 @@ cli_cmd_volume_defrag_cbk (struct cli_state *state, struct cli_cmd_word *word,
         if (!frame)
                 goto out;
 
-        dict = dict_new ();
-        if (!dict)
-                goto out;
+        ret = cli_cmd_volume_defrag_parse (words, wordcount, &dict);
 
-        if (!((wordcount == 4) || (wordcount == 5))) {
+        if (ret) {
                 cli_usage_out (word->pattern);
                 parse_error = 1;
-                goto out;
-        }
-
-        if (wordcount == 4) {
-                index = 3;
-        } else {
-                if (strcmp (words[3], "fix-layout") &&
-                    strcmp (words[3], "start")) {
-                        cli_usage_out (word->pattern);
-                        parse_error = 1;
-                        goto out;
-                }
-                index = 4;
-        }
-
-	if (strcmp (words[index], "start") && strcmp (words[index], "stop") &&
-            strcmp (words[index], "status") && strcmp (words[index], "force")) {
-	        cli_usage_out (word->pattern);
-		parse_error = 1;
-		goto out;
-	}
-
-        ret = dict_set_str (dict, "volname", (char *)words[2]);
-        if (ret)
-                goto out;
-
-        if (wordcount == 4) {
-                ret = dict_set_str (dict, "command", (char *)words[3]);
-                if (ret)
-                        goto out;
-        }
-        if (wordcount == 5) {
-               if ((strcmp (words[3], "fix-layout") ||
-                    strcmp (words[4], "start")) &&
-                    (strcmp (words[3], "start") ||
-                    strcmp (words[4], "force"))) {
-                        cli_usage_out (word->pattern);
-                        parse_error = 1;
-                        goto out;
-                }
-
-                ret = dict_set_str (dict, "option", (char *)words[4]);
-                if (ret)
-                        goto out;
-                ret = dict_set_str (dict, "command", (char *)words[3]);
-                if (ret)
-                        goto out;
         }
 
         proc = &cli_rpc_prog->proctable[GLUSTER_CLI_DEFRAG_VOLUME];
 
+        CLI_LOCAL_INIT (local, words, frame, dict);
+
         if (proc->fn) {
                 ret = proc->fn (frame, THIS, dict);
         }
 
 out:
-        if (dict)
-                dict_destroy (dict);
-
         if (ret) {
                 cli_cmd_sent_status_get (&sent);
                 if ((sent == 0) && (parse_error == 0))
                         cli_out ("Volume rebalance failed");
         }
 
+        CLI_STACK_DESTROY (frame);
+
         return ret;
 }
 
@@ -808,11 +801,11 @@ cli_cmd_volume_reset_cbk (struct cli_state *state, struct cli_cmd_word *word,
 {
         int                     sent = 0;
         int                     parse_error = 0;
-
         int                     ret = -1;
         rpc_clnt_procedure_t    *proc = NULL;
         call_frame_t            *frame = NULL;
         dict_t                  *options = NULL;
+        cli_local_t             *local = NULL;
 
         proc = &cli_rpc_prog->proctable[GLUSTER_CLI_RESET_VOLUME];
 
@@ -821,27 +814,27 @@ cli_cmd_volume_reset_cbk (struct cli_state *state, struct cli_cmd_word *word,
                 goto out;
 
         ret = cli_cmd_volume_reset_parse (words, wordcount, &options);
-
         if (ret) {
                 cli_usage_out (word->pattern);
                 parse_error = 1;
                 goto out;
         }
 
+        CLI_LOCAL_INIT (local, words, frame, options);
+
         if (proc->fn) {
                 ret = proc->fn (frame, THIS, options);
         }
 
 out:
-        if (options)
-                dict_unref (options);
-
         if (ret) {
                 cli_cmd_sent_status_get (&sent);
                 if ((sent == 0) && (parse_error == 0))
                         cli_out ("Volume reset failed");
         }
 
+        CLI_STACK_DESTROY (frame);
+
         return ret;
 
 }
@@ -857,6 +850,7 @@ cli_cmd_volume_profile_cbk (struct cli_state *state, struct cli_cmd_word *word,
         rpc_clnt_procedure_t    *proc    = NULL;
         call_frame_t            *frame   = NULL;
         dict_t                  *options = NULL;
+        cli_local_t             *local = NULL;
 
         ret = cli_cmd_volume_profile_parse (words, wordcount, &options);
 
@@ -872,20 +866,21 @@ cli_cmd_volume_profile_cbk (struct cli_state *state, struct cli_cmd_word *word,
         if (!frame)
                 goto out;
 
+        CLI_LOCAL_INIT (local, words, frame, options);
+
         if (proc->fn) {
                 ret = proc->fn (frame, THIS, options);
         }
 
 out:
-        if (options)
-                dict_unref (options);
-
         if (ret) {
                 cli_cmd_sent_status_get (&sent);
                 if ((sent == 0) && (parse_error == 0))
                         cli_out ("Volume profile failed");
         }
 
+        CLI_STACK_DESTROY (frame);
+
         return ret;
 
 }
@@ -901,6 +896,8 @@ cli_cmd_volume_set_cbk (struct cli_state *state, struct cli_cmd_word *word,
         rpc_clnt_procedure_t    *proc = NULL;
         call_frame_t            *frame = NULL;
         dict_t                  *options = NULL;
+        cli_local_t             *local = NULL;
+        char                    *op_errstr = NULL;
 
         proc = &cli_rpc_prog->proctable[GLUSTER_CLI_SET_VOLUME];
 
@@ -908,28 +905,33 @@ cli_cmd_volume_set_cbk (struct cli_state *state, struct cli_cmd_word *word,
         if (!frame)
                 goto out;
 
-        ret = cli_cmd_volume_set_parse (words, wordcount, &options);
-
+        ret = cli_cmd_volume_set_parse (words, wordcount, &options, &op_errstr);
         if (ret) {
-                cli_usage_out (word->pattern);
+                if (op_errstr) {
+                    cli_err ("%s", op_errstr);
+                    GF_FREE (op_errstr);
+                } else
+                    cli_usage_out (word->pattern);
+
                 parse_error = 1;
                 goto out;
         }
 
+        CLI_LOCAL_INIT (local, words, frame, options);
+
         if (proc->fn) {
                 ret = proc->fn (frame, THIS, options);
         }
 
 out:
-        if (options)
-                dict_unref (options);
-
         if (ret) {
                 cli_cmd_sent_status_get (&sent);
                 if ((sent == 0) && (parse_error == 0))
                         cli_out ("Volume set failed");
         }
 
+        CLI_STACK_DESTROY (frame);
+
         return ret;
 
 }
@@ -945,35 +947,63 @@ cli_cmd_volume_add_brick_cbk (struct cli_state *state,
         dict_t                  *options = NULL;
         int                     sent = 0;
         int                     parse_error = 0;
+        gf_answer_t             answer = GF_ANSWER_NO;
+        cli_local_t             *local = NULL;
+
+        const char *question = "Changing the 'stripe count' of the volume is "
+                "not a supported feature. In some cases it may result in data "
+                "loss on the volume. Also there may be issues with regular "
+                "filesystem operations on the volume after the change. Do you "
+                "really want to continue with 'stripe' count option ? ";
 
         frame = create_frame (THIS, THIS->ctx->pool);
         if (!frame)
                 goto out;
 
         ret = cli_cmd_volume_add_brick_parse (words, wordcount, &options);
-
         if (ret) {
                 cli_usage_out (word->pattern);
                 parse_error = 1;
                 goto out;
         }
 
+        /* TODO: there are challenges in supporting changing of
+           stripe-count, untill it is properly supported give warning to user */
+        if (dict_get (options, "stripe-count")) {
+                answer = cli_cmd_get_confirmation (state, question);
+
+                if (GF_ANSWER_NO == answer) {
+                        ret = 0;
+                        goto out;
+                }
+        }
+
+        if (state->mode & GLUSTER_MODE_SCRIPT) {
+                ret = dict_set_int32 (options, "force", _gf_true);
+                if (ret) {
+                        gf_log ("cli", GF_LOG_ERROR, "Failed to set force "
+                                "option");
+                        goto out;
+                }
+        }
+
         proc = &cli_rpc_prog->proctable[GLUSTER_CLI_ADD_BRICK];
 
+        CLI_LOCAL_INIT (local, words, frame, options);
+
         if (proc->fn) {
                 ret = proc->fn (frame, THIS, options);
         }
 
 out:
-        if (options)
-                dict_unref (options);
-
         if (ret) {
                 cli_cmd_sent_status_get (&sent);
                 if ((sent == 0) && (parse_error == 0))
                         cli_out ("Volume add-brick failed");
         }
 
+        CLI_STACK_DESTROY (frame);
+
         return ret;
 }
 
@@ -989,6 +1019,7 @@ cli_cmd_quota_cbk (struct cli_state *state, struct cli_cmd_word *word,
         call_frame_t            *frame     = NULL;
         dict_t                  *options   = NULL;
         gf_answer_t              answer    = GF_ANSWER_NO;
+        cli_local_t             *local     = NULL;
         const char *question = "Disabling quota will delete all the quota "
                                "configuration. Do you want to continue?";
 
@@ -1005,6 +1036,7 @@ cli_cmd_quota_cbk (struct cli_state *state, struct cli_cmd_word *word,
         }
 
         ret = cli_cmd_quota_parse (words, wordcount, &options);
+
         if (ret < 0) {
                 cli_usage_out (word->pattern);
                 parse_err = 1;
@@ -1016,16 +1048,17 @@ cli_cmd_quota_cbk (struct cli_state *state, struct cli_cmd_word *word,
                         goto out;
         }
 
+        CLI_LOCAL_INIT (local, words, frame, options);
+
         if (proc->fn)
                 ret = proc->fn (frame, THIS, options);
 
 out:
-        if (options)
-                dict_unref (options);
-
         if (ret && parse_err == 0)
                 cli_out ("Quota command failed");
 
+        CLI_STACK_DESTROY (frame);
+
         return ret;
 
 }
@@ -1043,6 +1076,7 @@ cli_cmd_volume_remove_brick_cbk (struct cli_state *state,
         int                     sent = 0;
         int                     parse_error = 0;
         int                     need_question = 0;
+        cli_local_t             *local = NULL;
 
         const char *question = "Removing brick(s) can result in data loss. "
                                "Do you want to Continue?";
@@ -1053,7 +1087,6 @@ cli_cmd_volume_remove_brick_cbk (struct cli_state *state,
 
         ret = cli_cmd_volume_remove_brick_parse (words, wordcount, &options,
                                                  &need_question);
-
         if (ret) {
                 cli_usage_out (word->pattern);
                 parse_error = 1;
@@ -1071,6 +1104,8 @@ cli_cmd_volume_remove_brick_cbk (struct cli_state *state,
 
         proc = &cli_rpc_prog->proctable[GLUSTER_CLI_REMOVE_BRICK];
 
+        CLI_LOCAL_INIT (local, words, frame, options);
+
         if (proc->fn) {
                 ret = proc->fn (frame, THIS, options);
         }
@@ -1082,8 +1117,8 @@ out:
                         cli_out ("Volume remove-brick failed");
         }
 
-        if (options)
-                dict_unref (options);
+        CLI_STACK_DESTROY (frame);
+
         return ret;
 
 }
@@ -1100,6 +1135,7 @@ cli_cmd_volume_replace_brick_cbk (struct cli_state *state,
         dict_t                  *options = NULL;
         int                     sent = 0;
         int                     parse_error = 0;
+        cli_local_t             *local = NULL;
 
 #ifdef GF_SOLARIS_HOST_OS
         cli_out ("Command not supported on Solaris");
@@ -1119,20 +1155,30 @@ cli_cmd_volume_replace_brick_cbk (struct cli_state *state,
                 goto out;
         }
 
+        if (state->mode & GLUSTER_MODE_SCRIPT) {
+                ret = dict_set_int32 (options, "force", _gf_true);
+                if (ret) {
+                        gf_log ("cli", GF_LOG_ERROR, "Failed to set force"
+                                "option");
+                        goto out;
+                }
+        }
+
+        CLI_LOCAL_INIT (local, words, frame, options);
+
         if (proc->fn) {
                 ret = proc->fn (frame, THIS, options);
         }
 
 out:
-        if (options)
-                dict_unref (options);
-
         if (ret) {
                 cli_cmd_sent_status_get (&sent);
                 if ((sent == 0) && (parse_error == 0))
                         cli_out ("Volume replace-brick failed");
         }
 
+        CLI_STACK_DESTROY (frame);
+
         return ret;
 }
 
@@ -1157,6 +1203,7 @@ cli_cmd_volume_top_cbk (struct cli_state *state, struct cli_cmd_word *word,
         dict_t                  *options = NULL;
         int                     sent     = 0;
         int                     parse_error = 0;
+        cli_local_t             *local = NULL;
 
         ret = cli_cmd_volume_top_parse (words, wordcount, &options);
 
@@ -1172,20 +1219,21 @@ cli_cmd_volume_top_cbk (struct cli_state *state, struct cli_cmd_word *word,
         if (!frame)
                 goto out;
 
+        CLI_LOCAL_INIT (local, words, frame, options);
+
         if (proc->fn) {
                 ret = proc->fn (frame, THIS, options);
         }
 
 out:
-        if (options)
-                dict_unref (options);
-
         if (ret) {
                 cli_cmd_sent_status_get (&sent);
                 if ((sent == 0) && (parse_error == 0))
                         cli_out ("Volume top failed");
         }
 
+        CLI_STACK_DESTROY (frame);
+
         return ret;
 
 }
@@ -1201,6 +1249,7 @@ cli_cmd_log_rotate_cbk (struct cli_state *state, struct cli_cmd_word *word,
         dict_t                  *options = NULL;
         int                     sent = 0;
         int                     parse_error = 0;
+        cli_local_t             *local = NULL;
 
         if (!((wordcount == 4) || (wordcount == 5))) {
                 cli_usage_out (word->pattern);
@@ -1218,19 +1267,19 @@ cli_cmd_log_rotate_cbk (struct cli_state *state, struct cli_cmd_word *word,
         if (ret)
                 goto out;
 
+        CLI_LOCAL_INIT (local, words, frame, options);
+
         if (proc->fn) {
                 ret = proc->fn (frame, THIS, options);
         }
 
 out:
-        if (options)
-                dict_destroy (options);
-
         if (ret) {
                 cli_cmd_sent_status_get (&sent);
                 if ((sent == 0) && (parse_error == 0))
                         cli_out ("Volume log rotate failed");
         }
+        CLI_STACK_DESTROY (frame);
 
         return ret;
 }
@@ -1304,6 +1353,7 @@ cli_cmd_volume_gsync_set_cbk (struct cli_state *state, struct cli_cmd_word *word
         dict_t                  *options = NULL;
         rpc_clnt_procedure_t    *proc    = NULL;
         call_frame_t            *frame   = NULL;
+        cli_local_t             *local   = NULL;
 
         proc = &cli_rpc_prog->proctable [GLUSTER_CLI_GSYNC_SET];
         if (proc == NULL) {
@@ -1324,16 +1374,17 @@ cli_cmd_volume_gsync_set_cbk (struct cli_state *state, struct cli_cmd_word *word
                 goto out;
         }
 
+        CLI_LOCAL_INIT (local, words, frame, options);
+
         if (proc->fn)
                 ret = proc->fn (frame, THIS, options);
 
 out:
-        if (options)
-                dict_unref (options);
-
         if (ret && parse_err == 0)
                 cli_out (GEOREP" command failed");
 
+        CLI_STACK_DESTROY (frame);
+
         return ret;
 }
 
@@ -1347,6 +1398,7 @@ cli_cmd_volume_status_cbk (struct cli_state *state,
         call_frame_t         *frame       = NULL;
         dict_t               *dict        = NULL;
         uint32_t              cmd         = 0;
+        cli_local_t          *local       = NULL;
 
         ret = cli_cmd_volume_status_parse (words, wordcount, &dict);
 
@@ -1374,11 +1426,13 @@ cli_cmd_volume_status_cbk (struct cli_state *state,
         if (!frame)
                 goto out;
 
+        CLI_LOCAL_INIT (local, words, frame, dict);
+
         ret = proc->fn (frame, THIS, dict);
 
- out:
-        if (dict)
-                dict_unref (dict);
+out:
+        CLI_STACK_DESTROY (frame);
+
         return ret;
 }
 
@@ -1386,16 +1440,14 @@ cli_cmd_volume_status_cbk (struct cli_state *state,
 int
 cli_get_detail_status (dict_t *dict, int i, cli_volume_status_t *status)
 {
-        uint64_t                   free            = -1;
-        uint64_t                   total           = -1;
+        uint64_t                   free            = 0;
+        uint64_t                   total           = 0;
         char                       key[1024]       = {0};
         int                        ret             = 0;
 
         memset (key, 0, sizeof (key));
         snprintf (key, sizeof (key), "brick%d.free", i);
         ret = dict_get_uint64 (dict, key, &free);
-        if (ret)
-                goto out;
 
         status->free = gf_uint64_2human_readable (free);
         if (!status->free)
@@ -1404,70 +1456,65 @@ cli_get_detail_status (dict_t *dict, int i, cli_volume_status_t *status)
         memset (key, 0, sizeof (key));
         snprintf (key, sizeof (key), "brick%d.total", i);
         ret = dict_get_uint64 (dict, key, &total);
-        if (ret)
-                goto out;
 
         status->total = gf_uint64_2human_readable (total);
         if (!status->total)
                 goto out;
 
+#ifdef GF_LINUX_HOST_OS
         memset (key, 0, sizeof (key));
         snprintf (key, sizeof (key), "brick%d.device", i);
         ret = dict_get_str (dict, key, &(status->device));
         if (ret)
-                goto out;
+                status->device = NULL;
+#endif
 
         memset (key, 0, sizeof (key));
         snprintf (key, sizeof (key), "brick%d.block_size", i);
         ret = dict_get_uint64 (dict, key, &(status->block_size));
-        if (ret)
-                goto out;
+        if (ret) {
+                ret = 0;
+                status->block_size = 0;
+        }
 
 #ifdef GF_LINUX_HOST_OS
         memset (key, 0, sizeof (key));
         snprintf (key, sizeof (key), "brick%d.mnt_options", i);
         ret = dict_get_str (dict, key, &(status->mount_options));
         if (ret)
-                goto out;
+                status->mount_options = NULL;
 
         memset (key, 0, sizeof (key));
         snprintf (key, sizeof (key), "brick%d.fs_name", i);
         ret = dict_get_str (dict, key, &(status->fs_name));
-        if (ret)
-                goto out;
-#endif
-
-        if (IS_EXT_FS(status->fs_name) ||
-            !strcmp (status->fs_name, "xfs")) {
-
-#ifdef GF_LINUX_HOST_OS
-                memset (key, 0, sizeof (key));
-                snprintf (key, sizeof (key), "brick%d.inode_size", i);
-                ret = dict_get_str (dict, key, &(status->inode_size));
-                if (ret)
-                        status->inode_size = NULL;
-#endif
-
-                memset (key, 0, sizeof (key));
-                snprintf (key, sizeof (key), "brick%d.total_inodes", i);
-                ret = dict_get_uint64 (dict, key,
-                                       &(status->total_inodes));
-                if (ret)
-                        goto out;
+        if (ret) {
+                ret = 0;
+                status->fs_name = NULL;
+        }
 
-                memset (key, 0, sizeof (key));
-                snprintf (key, sizeof (key), "brick%d.free_inodes", i);
-                ret = dict_get_uint64 (dict, key, &(status->free_inodes));
-                if (ret)
-                        goto out;
-        } else {
-#ifdef GF_LINUX_HOST_OS
+        memset (key, 0, sizeof (key));
+        snprintf (key, sizeof (key), "brick%d.inode_size", i);
+        ret = dict_get_str (dict, key, &(status->inode_size));
+        if (ret)
                 status->inode_size = NULL;
-#endif
+#endif /* GF_LINUX_HOST_OS */
+
+        memset (key, 0, sizeof (key));
+        snprintf (key, sizeof (key), "brick%d.total_inodes", i);
+        ret = dict_get_uint64 (dict, key,
+                        &(status->total_inodes));
+        if (ret)
                 status->total_inodes = 0;
+
+        memset (key, 0, sizeof (key));
+        snprintf (key, sizeof (key), "brick%d.free_inodes", i);
+        ret = dict_get_uint64 (dict, key, &(status->free_inodes));
+        if (ret) {
+                ret = 0;
                 status->free_inodes = 0;
         }
 
+
  out:
         return ret;
 }
@@ -1476,13 +1523,23 @@ void
 cli_print_detailed_status (cli_volume_status_t *status)
 {
         cli_out ("%-20s : %-20s", "Brick", status->brick);
-        cli_out ("%-20s : %-20d", "Port", status->port);
+        if (status->online)
+                cli_out ("%-20s : %-20d", "Port", status->port);
+        else
+                cli_out ("%-20s : %-20s", "Port", "N/A");
         cli_out ("%-20s : %-20c", "Online", (status->online) ? 'Y' : 'N');
         cli_out ("%-20s : %-20s", "Pid", status->pid_str);
 
 #ifdef GF_LINUX_HOST_OS
-        cli_out ("%-20s : %-20s", "File System", status->fs_name);
-        cli_out ("%-20s : %-20s", "Device", status->device);
+        if (status->fs_name)
+                cli_out ("%-20s : %-20s", "File System", status->fs_name);
+        else
+                cli_out ("%-20s : %-20s", "File System", "N/A");
+
+        if (status->device)
+                cli_out ("%-20s : %-20s", "Device", status->device);
+        else
+                cli_out ("%-20s : %-20s", "Device", "N/A");
 
         if (status->mount_options) {
                 cli_out ("%-20s : %-20s", "Mount Options",
@@ -1498,8 +1555,16 @@ cli_print_detailed_status (cli_volume_status_t *status)
                 cli_out ("%-20s : %-20s", "Inode Size", "N/A");
         }
 #endif
-        cli_out ("%-20s : %-20s", "Disk Space Free", status->free);
-        cli_out ("%-20s : %-20s", "Total Disk Space", status->total);
+        if (status->free)
+                cli_out ("%-20s : %-20s", "Disk Space Free", status->free);
+        else
+                cli_out ("%-20s : %-20s", "Disk Space Free", "N/A");
+
+        if (status->total)
+                cli_out ("%-20s : %-20s", "Total Disk Space", status->total);
+        else
+                cli_out ("%-20s : %-20s", "Total Disk Space", "N/A");
+
 
         if (status->total_inodes) {
                 cli_out ("%-20s : %-20ld", "Inode Count",
@@ -1520,30 +1585,38 @@ int
 cli_print_brick_status (cli_volume_status_t *status)
 {
         int  fieldlen = CLI_VOL_STATUS_BRICK_LEN;
-        char buf[80] = {0,};
         int  bricklen = 0;
-        int  i = 0;
         char *p = NULL;
         int  num_tabs = 0;
 
-        bricklen = strlen (status->brick);
         p = status->brick;
+        bricklen = strlen (p);
         while (bricklen > 0) {
                 if (bricklen > fieldlen) {
-                        i++;
-                        strncpy (buf, p, fieldlen);
-                        buf[strlen(buf) + 1] = '\0';
-                        cli_out ("%s", buf);
-                        p = status->brick + i * fieldlen;
+                        cli_out ("%.*s", fieldlen, p);
+                        p += fieldlen;
                         bricklen -= fieldlen;
                 } else {
                         num_tabs = (fieldlen - bricklen) / CLI_TAB_LENGTH + 1;
                         printf ("%s", p);
                         while (num_tabs-- != 0)
                                 printf ("\t");
-                        cli_out ("%d\t%c\t%s",
-                                 status->port, status->online?'Y':'N',
-                                 status->pid_str);
+                        if (status->port) {
+                                if (status->online)
+                                        cli_out ("%d\t%c\t%s",
+                                                 status->port,
+                                                 status->online?'Y':'N',
+                                                 status->pid_str);
+                                else
+                                        cli_out ("%s\t%c\t%s",
+                                                 "N/A",
+                                                 status->online?'Y':'N',
+                                                 status->pid_str);
+                        }
+                        else
+                                cli_out ("%s\t%c\t%s",
+                                         "N/A", status->online?'Y':'N',
+                                         status->pid_str);
                         bricklen = 0;
                 }
         }
@@ -1561,8 +1634,11 @@ cli_cmd_volume_heal_cbk (struct cli_state *state, struct cli_cmd_word *word,
         int                     sent = 0;
         int                     parse_error = 0;
         dict_t                  *options = NULL;
+        xlator_t                *this = NULL;
+        cli_local_t             *local = NULL;
 
-        frame = create_frame (THIS, THIS->ctx->pool);
+        this = THIS;
+        frame = create_frame (this, this->ctx->pool);
         if (!frame)
                 goto out;
 
@@ -1581,6 +1657,8 @@ cli_cmd_volume_heal_cbk (struct cli_state *state, struct cli_cmd_word *word,
 
         proc = &cli_rpc_prog->proctable[GLUSTER_CLI_HEAL_VOLUME];
 
+        CLI_LOCAL_INIT (local, words, frame, options);
+
         if (proc->fn) {
                 ret = proc->fn (frame, THIS, options);
         }
@@ -1592,8 +1670,7 @@ out:
                         cli_out ("Volume heal failed");
         }
 
-        if (options)
-                dict_unref (options);
+        CLI_STACK_DESTROY (frame);
 
         return ret;
 }
@@ -1608,6 +1685,7 @@ cli_cmd_volume_statedump_cbk (struct cli_state *state, struct cli_cmd_word *word
         dict_t                          *options = NULL;
         int                             sent = 0;
         int                             parse_error = 0;
+        cli_local_t                     *local = NULL;
 
         frame = create_frame (THIS, THIS->ctx->pool);
         if (!frame)
@@ -1636,6 +1714,9 @@ cli_cmd_volume_statedump_cbk (struct cli_state *state, struct cli_cmd_word *word
                 goto out;
 
         proc = &cli_rpc_prog->proctable[GLUSTER_CLI_STATEDUMP_VOLUME];
+
+        CLI_LOCAL_INIT (local, words, frame, options);
+
         if (proc->fn) {
                 ret = proc->fn (frame, THIS, options);
         }
@@ -1647,6 +1728,8 @@ out:
                         cli_out ("Volume statedump failed");
         }
 
+        CLI_STACK_DESTROY (frame);
+
         return ret;
 }
 
@@ -1675,6 +1758,8 @@ out:
                         cli_out ("Volume list failed");
         }
 
+        CLI_STACK_DESTROY (frame);
+
         return ret;
 }
 
@@ -1689,6 +1774,7 @@ cli_cmd_volume_clearlocks_cbk (struct cli_state *state,
         dict_t                          *options = NULL;
         int                             sent = 0;
         int                             parse_error = 0;
+        cli_local_t                     *local = NULL;
 
         frame = create_frame (THIS, THIS->ctx->pool);
         if (!frame)
@@ -1718,6 +1804,9 @@ cli_cmd_volume_clearlocks_cbk (struct cli_state *state,
                 goto out;
 
         proc = &cli_rpc_prog->proctable[GLUSTER_CLI_CLRLOCKS_VOLUME];
+
+        CLI_LOCAL_INIT (local, words, frame, options);
+
         if (proc->fn) {
                 ret = proc->fn (frame, THIS, options);
         }
@@ -1729,6 +1818,8 @@ out:
                         cli_out ("Volume clear-locks failed");
         }
 
+        CLI_STACK_DESTROY (frame);
+
         return ret;
 }
 
@@ -1737,7 +1828,13 @@ struct cli_cmd volume_cmds[] = {
           cli_cmd_volume_info_cbk,
           "list information of all volumes"},
 
-        { "volume create <NEW-VOLNAME> [stripe <COUNT>] [replica <COUNT>] [transport <tcp|rdma|tcp,rdma>] <NEW-BRICK> ...",
+        { "volume create <NEW-VOLNAME> [stripe <COUNT>] [replica <COUNT>] "
+          "[transport <tcp|rdma|tcp,rdma>] <NEW-BRICK>"
+#ifdef HAVE_BD_XLATOR
+          "?<vg_name>"
+#endif
+          "... [force]",
+
           cli_cmd_volume_create_cbk,
           "create a new volume of specified type with mentioned bricks"},
 
@@ -1757,11 +1854,11 @@ struct cli_cmd volume_cmds[] = {
           cli_cmd_volume_rename_cbk,
           "rename volume <VOLNAME> to <NEW-VOLNAME>"},*/
 
-        { "volume add-brick <VOLNAME> [<stripe|replica> <COUNT>] <NEW-BRICK> ...",
+        { "volume add-brick <VOLNAME> [<stripe|replica> <COUNT>] <NEW-BRICK> ... [force]",
           cli_cmd_volume_add_brick_cbk,
           "add brick to volume <VOLNAME>"},
 
-        { "volume remove-brick <VOLNAME> [replica <COUNT>] <BRICK> ... {start|pause|abort|status|commit|force}",
+        { "volume remove-brick <VOLNAME> [replica <COUNT>] <BRICK> ... [start|stop|status|commit|force]",
           cli_cmd_volume_remove_brick_cbk,
           "remove brick from volume <VOLNAME>"},
 
@@ -1769,7 +1866,7 @@ struct cli_cmd volume_cmds[] = {
           cli_cmd_volume_defrag_cbk,
           "rebalance operations"},
 
-        { "volume replace-brick <VOLNAME> <BRICK> <NEW-BRICK> {start|pause|abort|status|commit [force]}",
+        { "volume replace-brick <VOLNAME> <BRICK> <NEW-BRICK> {start [force]|pause|abort|status|commit [force]}",
           cli_cmd_volume_replace_brick_cbk,
           "replace-brick operations"},
 
@@ -1798,13 +1895,14 @@ struct cli_cmd volume_cmds[] = {
          "reset all the reconfigured options"},
 
 #if (SYNCDAEMON_COMPILE)
-        {"volume "GEOREP" [<VOLNAME>] [<SLAVE-URL>] {start|stop|config|status|log-rotate} [options...]",
+        {"volume "GEOREP" [<VOLNAME>] [<SLAVE-URL>] {create [push-pem] [force]"
+         "|start [force]|stop [force]|config|status [detail]|delete} [options...]",
          cli_cmd_volume_gsync_set_cbk,
          "Geo-sync operations",
          cli_cmd_check_gsync_exists_cbk},
 #endif
 
-         { "volume profile <VOLNAME> {start|info|stop} [nfs]",
+         { "volume profile <VOLNAME> {start|stop|info [nfs]}",
            cli_cmd_volume_profile_cbk,
            "volume profile operations"},
 
@@ -1812,18 +1910,17 @@ struct cli_cmd volume_cmds[] = {
           cli_cmd_quota_cbk,
           "quota translator specific operations"},
 
-         { "volume top <VOLNAME> {[open|read|write|opendir|readdir [nfs]] "
-           "|[read-perf|write-perf [nfs|{bs <size> count <count>}]]} "
-           " [brick <brick>] [list-cnt <count>]",
+         { "volume top <VOLNAME> {open|read|write|opendir|readdir|clear} [nfs|brick <brick>] [list-cnt <value>] |\n"
+           "volume top <VOLNAME> {read-perf|write-perf} [bs <size> count <count>] [brick <brick>] [list-cnt <value>]",
            cli_cmd_volume_top_cbk,
            "volume top operations"},
 
-        { "volume status [all | <VOLNAME> [nfs|<BRICK>]]"
-          " [detail|clients|mem|inode|fd|callpool]",
+        { "volume status [all | <VOLNAME> [nfs|shd|<BRICK>]]"
+          " [detail|clients|mem|inode|fd|callpool|tasks]",
           cli_cmd_volume_status_cbk,
           "display status of all or specified volume(s)/brick"},
 
-        { "volume heal <VOLNAME> [{full | info {healed | heal-failed | split-brain}}]",
+        { "volume heal <VOLNAME> [{full | statistics {heal-count {replica <hostname:brickname>}} |info {healed | heal-failed | split-brain}}]",
           cli_cmd_volume_heal_cbk,
           "self-heal commands on volume specified by <VOLNAME>"},
 
diff --git a/cli/src/cli-cmd.c b/cli/src/cli-cmd.c
index f2b434ac7..b81f75b5b 100644
--- a/cli/src/cli-cmd.c
+++ b/cli/src/cli-cmd.c
@@ -1,22 +1,12 @@
 /*
-  Copyright (c) 2010-2011 Gluster, Inc. <http://www.gluster.com>
-  This file is part of GlusterFS.
-
-  GlusterFS is free software; you can redistribute it and/or modify
-  it under the terms of the GNU General Public License as published
-  by the Free Software Foundation; either version 3 of the License,
-  or (at your option) any later version.
-
-  GlusterFS is distributed in the hope that it will be useful, but
-  WITHOUT ANY WARRANTY; without even the implied warranty of
-  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-  General Public License for more details.
-
-  You should have received a copy of the GNU General Public License
-  along with this program.  If not, see
-  <http://www.gnu.org/licenses/>.
-*/
+   Copyright (c) 2010-2012 Red Hat, Inc. <http://www.redhat.com>
+   This file is part of GlusterFS.
 
+   This file is licensed to you under your choice of the GNU Lesser
+   General Public License, version 3 or any later version (LGPLv3 or
+   later), or the GNU General Public License, version 2 (GPLv2), in all
+   cases as published by the Free Software Foundation.
+*/
 #include <stdio.h>
 #include <string.h>
 #include <stdlib.h>
@@ -211,8 +201,7 @@ cli_cmd_process_line (struct cli_state *state, const char *text)
 
         ret = cli_cmd_process (state, count, tokens);
 out:
-        if (copy)
-                free (copy);
+        free (copy);
 
         if (tokens)
                 cli_cmd_tokens_destroy (tokens);
@@ -242,6 +231,9 @@ cli_cmds_register (struct cli_state *state)
         if (ret)
                 goto out;
 
+        ret = cli_cmd_snapshot_register (state);
+        if (ret)
+                goto out;
 out:
         return ret;
 }
@@ -371,8 +363,11 @@ cli_cmd_submit (void *req, call_frame_t *frame,
         int             ret = -1;
         unsigned        timeout = 0;
 
-        timeout = (GLUSTER_CLI_PROFILE_VOLUME == procnum) ?
-                   CLI_TOP_CMD_TIMEOUT : CLI_DEFAULT_CMD_TIMEOUT;
+        if ((GLUSTER_CLI_PROFILE_VOLUME == procnum) ||
+            (GLUSTER_CLI_HEAL_VOLUME == procnum))
+                timeout = CLI_TEN_MINUTES_TIMEOUT;
+        else
+                timeout = CLI_DEFAULT_CMD_TIMEOUT;
 
         cli_cmd_lock ();
         cmd_sent = 0;
diff --git a/cli/src/cli-cmd.h b/cli/src/cli-cmd.h
index 82e0ff113..041729276 100644
--- a/cli/src/cli-cmd.h
+++ b/cli/src/cli-cmd.h
@@ -1,22 +1,12 @@
 /*
-   Copyright (c) 2006-2011 Gluster, Inc. <http://www.gluster.com>
+   Copyright (c) 2006-2012 Red Hat, Inc. <http://www.redhat.com>
    This file is part of GlusterFS.
 
-   GlusterFS is free software; you can redistribute it and/or modify
-   it under the terms of the GNU General Public License as published
-   by the Free Software Foundation; either version 3 of the License,
-   or (at your option) any later version.
-
-   GlusterFS is distributed in the hope that it will be useful, but
-   WITHOUT ANY WARRANTY; without even the implied warranty of
-   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-   General Public License for more details.
-
-   You should have received a copy of the GNU General Public License
-   along with this program.  If not, see
-   <http://www.gnu.org/licenses/>.
+   This file is licensed to you under your choice of the GNU Lesser
+   General Public License, version 3 or any later version (LGPLv3 or
+   later), or the GNU General Public License, version 2 (GPLv2), in all
+   cases as published by the Free Software Foundation.
 */
-
 #ifndef __CLI_CMD_H__
 #define __CLI_CMD_H__
 
@@ -30,6 +20,32 @@
 #include "cli.h"
 #include "list.h"
 
+#define CLI_LOCAL_INIT(local, words, frame, dictionary) \
+        do {                                                 \
+                local = cli_local_get ();                    \
+                                                             \
+                if (local) {                                 \
+                        local->words = words;                \
+                        if (dictionary)                      \
+                                local->dict = dictionary;    \
+                        if (frame)                           \
+                                frame->local = local;        \
+                }                                            \
+        } while (0)
+
+#define CLI_STACK_DESTROY(_frame)                                       \
+        do {                                                            \
+                if (_frame) {                                           \
+                        if (_frame->local) {                            \
+                                gf_log ("cli", GF_LOG_DEBUG, "frame->local " \
+                                        "is not NULL (%p)", _frame->local); \
+                                cli_local_wipe (_frame->local);         \
+                                _frame->local = NULL;                   \
+                        }                                               \
+                        STACK_DESTROY (_frame->root);                   \
+                }                                                       \
+        } while (0);
+
 typedef enum {
         GF_ANSWER_YES = 1,
         GF_ANSWER_NO  = 2
@@ -77,6 +93,8 @@ int cli_cmd_probe_register (struct cli_state *state);
 
 int cli_cmd_system_register (struct cli_state *state);
 
+int cli_cmd_snapshot_register (struct cli_state *state);
+
 int cli_cmd_misc_register (struct cli_state *state);
 
 struct cli_cmd_word *cli_cmd_nextword (struct cli_cmd_word *word,
@@ -102,4 +120,5 @@ cli_cmd_submit (void *req, call_frame_t *frame,
 gf_answer_t
 cli_cmd_get_confirmation (struct cli_state *state, const char *question);
 int cli_cmd_sent_status_get (int *status);
+
 #endif /* __CLI_CMD_H__ */
diff --git a/cli/src/cli-mem-types.h b/cli/src/cli-mem-types.h
index 3c49d2183..09fcb639b 100644
--- a/cli/src/cli-mem-types.h
+++ b/cli/src/cli-mem-types.h
@@ -1,22 +1,12 @@
 /*
-   Copyright (c) 2010-2011 Gluster, Inc. <http://www.gluster.com>
+   Copyright (c) 2010-2012 Red Hat, Inc. <http://www.redhat.com>
    This file is part of GlusterFS.
 
-   GlusterFS is free software; you can redistribute it and/or modify
-   it under the terms of the GNU General Public License as published
-   by the Free Software Foundation; either version 3 of the License,
-   or (at your option) any later version.
-
-   GlusterFS is distributed in the hope that it will be useful, but
-   WITHOUT ANY WARRANTY; without even the implied warranty of
-   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-   General Public License for more details.
-
-   You should have received a copy of the GNU General Public License
-   along with this program.  If not, see
-   <http://www.gnu.org/licenses/>.
+   This file is licensed to you under your choice of the GNU Lesser
+   General Public License, version 3 or any later version (LGPLv3 or
+   later), or the GNU General Public License, version 2 (GPLv2), in all
+   cases as published by the Free Software Foundation.
 */
-
 #ifndef __CLI_MEM_TYPES_H__
 #define __CLI_MEM_TYPES_H__
 
diff --git a/cli/src/cli-rl.c b/cli/src/cli-rl.c
index 9a37a77de..ade1c8ebb 100644
--- a/cli/src/cli-rl.c
+++ b/cli/src/cli-rl.c
@@ -1,22 +1,12 @@
 /*
-  Copyright (c) 2010-2011 Gluster, Inc. <http://www.gluster.com>
-  This file is part of GlusterFS.
-
-  GlusterFS is free software; you can redistribute it and/or modify
-  it under the terms of the GNU General Public License as published
-  by the Free Software Foundation; either version 3 of the License,
-  or (at your option) any later version.
-
-  GlusterFS is distributed in the hope that it will be useful, but
-  WITHOUT ANY WARRANTY; without even the implied warranty of
-  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-  General Public License for more details.
-
-  You should have received a copy of the GNU General Public License
-  along with this program.  If not, see
-  <http://www.gnu.org/licenses/>.
-*/
+   Copyright (c) 2010-2012 Red Hat, Inc. <http://www.redhat.com>
+   This file is part of GlusterFS.
 
+   This file is licensed to you under your choice of the GNU Lesser
+   General Public License, version 3 or any later version (LGPLv3 or
+   later), or the GNU General Public License, version 2 (GPLv2), in all
+   cases as published by the Free Software Foundation.
+*/
 #include <stdio.h>
 #include <string.h>
 #include <stdlib.h>
@@ -48,7 +38,6 @@ cli_rl_out (struct cli_state *state, const char *fmt, va_list ap)
 {
         int tmp_rl_point = rl_point;
         int            n = rl_end;
-        int            i = 0;
         int            ret = 0;
 
         if (rl_end >= 0 ) {
@@ -56,12 +45,7 @@ cli_rl_out (struct cli_state *state, const char *fmt, va_list ap)
                 rl_redisplay ();
         }
 
-        printf ("\r");
-
-        for (i = 0; i <= strlen (state->prompt); i++)
-                printf (" ");
-
-        printf ("\r");
+        printf ("\r%*s\r", (int)strlen (state->prompt), "");
 
         ret = vprintf (fmt, ap);
 
@@ -77,6 +61,34 @@ cli_rl_out (struct cli_state *state, const char *fmt, va_list ap)
         return ret;
 }
 
+int
+cli_rl_err (struct cli_state *state, const char *fmt, va_list ap)
+{
+        int tmp_rl_point = rl_point;
+        int            n = rl_end;
+        int            ret = 0;
+
+        if (rl_end >= 0 ) {
+                rl_kill_text (0, rl_end);
+                rl_redisplay ();
+        }
+
+        fprintf (stderr, "\r%*s\r", (int)strlen (state->prompt), "");
+
+        ret = vfprintf (stderr, fmt, ap);
+
+        fprintf (stderr, "\n");
+        fflush(stderr);
+
+        if (n) {
+                rl_do_undo ();
+                rl_point = tmp_rl_point;
+                rl_reset_line_state ();
+        }
+
+        return ret;
+}
+
 
 void
 cli_rl_process_line (char *line)
@@ -204,8 +216,7 @@ cli_rl_tokenize (const char *text)
         }
 
 out:
-        if (copy)
-                free (copy);
+        free (copy);
 
         if (i < count) {
                 cli_cmd_tokens_destroy (tokens);
@@ -352,9 +363,10 @@ cli_rl_input (void *_data)
         for (;;) {
                 line = readline (state->prompt);
                 if (!line)
-                        break;
+                        exit(0);  //break;
 
-                cli_rl_process_line (line);
+                if (*line)
+                        cli_rl_process_line (line);
 
                 free (line);
         }
diff --git a/cli/src/cli-rpc-ops.c b/cli/src/cli-rpc-ops.c
index 4be25c9e4..bfeb854ad 100644
--- a/cli/src/cli-rpc-ops.c
+++ b/cli/src/cli-rpc-ops.c
@@ -1,32 +1,18 @@
 /*
-  Copyright (c) 2010-2011 Gluster, Inc. <http://www.gluster.com>
-  This file is part of GlusterFS.
-
-  GlusterFS is free software; you can redistribute it and/or modify
-  it under the terms of the GNU General Public License as published
-  by the Free Software Foundation; either version 3 of the License,
-  or (at your option) any later version.
-
-  GlusterFS is distributed in the hope that it will be useful, but
-  WITHOUT ANY WARRANTY; without even the implied warranty of
-  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-  General Public License for more details.
-
-  You should have received a copy of the GNU General Public License
-  along with this program.  If not, see
-  <http://www.gnu.org/licenses/>.
-*/
+   Copyright (c) 2010-2012 Red Hat, Inc. <http://www.redhat.com>
+   This file is part of GlusterFS.
 
+   This file is licensed to you under your choice of the GNU Lesser
+   General Public License, version 3 or any later version (LGPLv3 or
+   later), or the GNU General Public License, version 2 (GPLv2), in all
+   cases as published by the Free Software Foundation.
+*/
 
 #ifndef _CONFIG_H
 #define _CONFIG_H
 #include "config.h"
 #endif
 
-#ifndef GSYNC_CONF
-#define GSYNC_CONF GEOREP"/gsyncd.conf"
-#endif
-
 /* Widths of various columns in top read/write-perf output
  * Total width of top read/write-perf should be 80 chars
  * including one space between column
@@ -36,6 +22,8 @@
 #define VOL_TOP_PERF_SPEED_WIDTH        4
 #define VOL_TOP_PERF_TIME_WIDTH         26
 
+#define INDENT_MAIN_HEAD "%-25s %s "
+
 #include "cli.h"
 #include "compat-errno.h"
 #include "cli-cmd.h"
@@ -58,25 +46,39 @@ extern rpc_clnt_prog_t *cli_rpc_prog;
 extern int              cli_op_ret;
 extern int              connected;
 
-char *cli_volume_type[] = {"Distribute",
-                           "Stripe",
-                           "Replicate",
-                           "Striped-Replicate",
-                           "Distributed-Stripe",
-                           "Distributed-Replicate",
-                           "Distributed-Striped-Replicate",
-};
-
-
-char *cli_volume_status[] = {"Created",
-                             "Started",
-                             "Stopped"
+char *cli_vol_type_str[] = {"Distribute",
+                            "Stripe",
+                            "Replicate",
+                            "Striped-Replicate",
+                            "Distributed-Stripe",
+                            "Distributed-Replicate",
+                            "Distributed-Striped-Replicate",
+                           };
+
+char *cli_vol_status_str[] = {"Created",
+                              "Started",
+                              "Stopped",
+                             };
+
+char *cli_vol_task_status_str[] = {"not started",
+                                   "in progress",
+                                   "stopped",
+                                   "completed",
+                                   "failed",
+                                   "fix-layout in progress",
+                                   "fix-layout stopped",
+                                   "fix-layout completed",
+                                   "fix-layout failed",
 };
 
 int32_t
-gf_cli3_1_get_volume (call_frame_t *frame, xlator_t *this,
+gf_cli_get_volume (call_frame_t *frame, xlator_t *this,
                       void *data);
 
+int
+cli_to_glusterd (gf_cli_req *req, call_frame_t *frame, fop_cbk_fn_t cbkfn,
+                 xdrproc_t xdrproc, dict_t *dict, int procnum, xlator_t *this,
+                 rpc_clnt_prog_t *prog, struct iobref *iobref);
 
 rpc_clnt_prog_t cli_handshake_prog = {
         .progname  = "cli handshake",
@@ -91,10 +93,10 @@ rpc_clnt_prog_t cli_pmap_prog = {
 };
 
 int
-gf_cli3_1_probe_cbk (struct rpc_req *req, struct iovec *iov,
+gf_cli_probe_cbk (struct rpc_req *req, struct iovec *iov,
                         int count, void *myframe)
 {
-        gf1_cli_probe_rsp     rsp   = {0,};
+        gf_cli_rsp            rsp   = {0,};
         int                   ret   = -1;
         char                  msg[1024] = {0,};
 
@@ -102,88 +104,39 @@ gf_cli3_1_probe_cbk (struct rpc_req *req, struct iovec *iov,
                 goto out;
         }
 
-        ret = xdr_to_generic (*iov, &rsp, (xdrproc_t)xdr_gf1_cli_probe_rsp);
+        ret = xdr_to_generic (*iov, &rsp, (xdrproc_t)xdr_gf_cli_rsp);
         if (ret < 0) {
-                gf_log ("", GF_LOG_ERROR, "error");
+                gf_log (((call_frame_t *) myframe)->this->name, GF_LOG_ERROR,
+                        "Failed to decode xdr response");
                 //rsp.op_ret   = -1;
                 //rsp.op_errno = EINVAL;
                 goto out;
         }
 
         gf_log ("cli", GF_LOG_INFO, "Received resp to probe");
-         if (!rsp.op_ret) {
-                switch (rsp.op_errno) {
-                        case GF_PROBE_SUCCESS:
-                                snprintf (msg, sizeof (msg),
-                                          "Probe successful");
-                                break;
-                        case GF_PROBE_LOCALHOST:
-                                snprintf (msg, sizeof (msg),
-                                          "Probe on localhost not needed");
-                                break;
-                        case GF_PROBE_FRIEND:
-                                snprintf (msg, sizeof (msg),
-                                          "Probe on host %s port %d already"
-                                          " in peer list", rsp.hostname,
-                                          rsp.port);
-                                break;
-                        default:
-                                snprintf (msg, sizeof (msg),
-                                          "Probe returned with unknown errno %d",
-                                           rsp.op_errno);
-                                break;
-                }
-         }
 
-        if (rsp.op_ret) {
-                switch (rsp.op_errno) {
-                        case GF_PROBE_ANOTHER_CLUSTER:
-                                snprintf (msg, sizeof (msg),
-                                          "%s is already part of another"
-                                          " cluster", rsp.hostname);
-                                break;
-                        case GF_PROBE_VOLUME_CONFLICT:
-                                snprintf (msg, sizeof (msg),
-                                          "Atleast one volume on %s conflicts "
-                                          "with existing volumes in the "
-                                          "cluster", rsp.hostname);
-                                break;
-                        case GF_PROBE_UNKNOWN_PEER:
-                                snprintf (msg, sizeof (msg),
-                                          "%s responded with 'unknown peer'"
-                                          " error, this could happen if %s "
-                                          "doesn't have localhost in its peer"
-                                          " database", rsp.hostname,
-                                          rsp.hostname);
-                                break;
-                        case GF_PROBE_ADD_FAILED:
-                                snprintf (msg, sizeof (msg),
-                                          "Failed to add peer information "
-                                          "on %s" , rsp.hostname);
-                                break;
-
-                        default:
-                                snprintf (msg, sizeof (msg),
-                                          "Probe unsuccessful\nProbe returned "
-                                          "with unknown errno %d",
-                                          rsp.op_errno);
-                                break;
-                }
-                gf_log ("glusterd",GF_LOG_ERROR,"Probe failed with op_ret %d"
-                        " and op_errno %d", rsp.op_ret, rsp.op_errno);
+        if (rsp.op_errstr && (strlen (rsp.op_errstr) > 0)) {
+                snprintf (msg, sizeof (msg), "%s", rsp.op_errstr);
+                if (rsp.op_ret)
+                        gf_log ("cli", GF_LOG_ERROR, "%s", msg);
         }
 
-#if (HAVE_LIB_XML)
         if (global_state->mode & GLUSTER_MODE_XML) {
-                ret = cli_xml_output_str ("peerProbe", msg, rsp.op_ret,
-                                          rsp.op_errno, NULL);
+                ret = cli_xml_output_str (NULL,
+                                          (rsp.op_ret)? NULL : msg,
+                                          rsp.op_ret, rsp.op_errno,
+                                          (rsp.op_ret)? msg : NULL);
                 if (ret)
                         gf_log ("cli", GF_LOG_ERROR,
                                 "Error outputting to xml");
                 goto out;
         }
-#endif
-        cli_out ("%s", msg);
+
+        if (!rsp.op_ret)
+                cli_out ("peer probe: success. %s", msg);
+        else
+                cli_err ("peer probe: failed: %s", msg);
+
         ret = rsp.op_ret;
 
 out:
@@ -192,71 +145,53 @@ out:
 }
 
 int
-gf_cli3_1_deprobe_cbk (struct rpc_req *req, struct iovec *iov,
+gf_cli_deprobe_cbk (struct rpc_req *req, struct iovec *iov,
                        int count, void *myframe)
 {
-        gf1_cli_deprobe_rsp    rsp   = {0,};
+        gf_cli_rsp            rsp   = {0,};
         int                   ret   = -1;
-        char                  msg[1024] = {0,};
+        char              msg[1024] = {0,};
 
         if (-1 == req->rpc_status) {
                 goto out;
         }
 
-        ret = xdr_to_generic (*iov, &rsp, (xdrproc_t)xdr_gf1_cli_deprobe_rsp);
+        ret = xdr_to_generic (*iov, &rsp, (xdrproc_t)xdr_gf_cli_rsp);
         if (ret < 0) {
-                gf_log ("", GF_LOG_ERROR, "error");
+                gf_log (((call_frame_t *) myframe)->this->name, GF_LOG_ERROR,
+                        "Failed to decode xdr response");
                 //rsp.op_ret   = -1;
                 //rsp.op_errno = EINVAL;
                 goto out;
         }
 
         gf_log ("cli", GF_LOG_INFO, "Received resp to deprobe");
+
         if (rsp.op_ret) {
-                switch (rsp.op_errno) {
-                        case GF_DEPROBE_LOCALHOST:
-                                snprintf (msg, sizeof (msg),
-                                          "%s is localhost", rsp.hostname);
-                                break;
-                        case GF_DEPROBE_NOT_FRIEND:
-                                snprintf (msg, sizeof (msg),
-                                          "%s is not part of cluster",
-                                          rsp.hostname);
-                                break;
-                        case GF_DEPROBE_BRICK_EXIST:
-                                snprintf (msg, sizeof (msg),
-                                          "Brick(s) with the peer %s exist in "
-                                          "cluster", rsp.hostname);
-                                break;
-                        case GF_DEPROBE_FRIEND_DOWN:
-                                snprintf (msg, sizeof (msg),
-                                          "One of the peers is probably down."
-                                          " Check with 'peer status'.");
-                                break;
-                        default:
-                                snprintf (msg, sizeof (msg),
-                                          "Detach unsuccessful\nDetach returned"
-                                          " with unknown errno %d",
-                                          rsp.op_errno);
-                                break;
+                if (strlen (rsp.op_errstr) > 0) {
+                        snprintf (msg, sizeof (msg), "%s", rsp.op_errstr);
+                        gf_log ("cli", GF_LOG_ERROR, "%s", rsp.op_errstr);
                 }
-                gf_log ("glusterd",GF_LOG_ERROR,"Detach failed with op_ret %d"
-                        " and op_errno %d", rsp.op_ret, rsp.op_errno);
         } else {
-                snprintf (msg, sizeof (msg), "Detach successful");
+                snprintf (msg, sizeof (msg), "success");
         }
 
-#if (HAVE_LIB_XML)
         if (global_state->mode & GLUSTER_MODE_XML) {
-                ret = cli_xml_output_str ("peerDetach", msg, rsp.op_ret,
-                                          rsp.op_errno, NULL);
+                ret = cli_xml_output_str (NULL,
+                                          (rsp.op_ret)? NULL : msg,
+                                          rsp.op_ret, rsp.op_errno,
+                                          (rsp.op_ret)? msg : NULL);
                 if (ret)
                         gf_log ("cli", GF_LOG_ERROR,
                                 "Error outputting to xml");
                 goto out;
         }
-#endif
-        cli_out ("%s", msg);
+
+        if (!rsp.op_ret)
+                cli_out ("peer detach: %s", msg);
+        else
+                cli_err ("peer detach: failed: %s", msg);
+
         ret = rsp.op_ret;
 
 out:
@@ -265,34 +200,143 @@ out:
 }
 
 int
-gf_cli3_1_list_friends_cbk (struct rpc_req *req, struct iovec *iov,
-                             int count, void *myframe)
+gf_cli_output_peer_status (dict_t *dict, int count)
 {
-        gf1_cli_peer_list_rsp      rsp   = {0,};
         int                        ret   = -1;
-        dict_t                     *dict = NULL;
         char                       *uuid_buf = NULL;
         char                       *hostname_buf = NULL;
         int32_t                    i = 1;
         char                       key[256] = {0,};
         char                       *state = NULL;
-        int32_t                    port = 0;
         int32_t                    connected = 0;
         char                       *connected_str = NULL;
 
+        cli_out ("Number of Peers: %d", count);
+        i = 1;
+        while ( i <= count) {
+                snprintf (key, 256, "friend%d.uuid", i);
+                ret = dict_get_str (dict, key, &uuid_buf);
+                if (ret)
+                        goto out;
+
+                snprintf (key, 256, "friend%d.hostname", i);
+                ret = dict_get_str (dict, key, &hostname_buf);
+                if (ret)
+                        goto out;
+
+                snprintf (key, 256, "friend%d.connected", i);
+                ret = dict_get_int32 (dict, key, &connected);
+                if (ret)
+                        goto out;
+                if (connected)
+                        connected_str = "Connected";
+                else
+                        connected_str = "Disconnected";
+
+
+                snprintf (key, 256, "friend%d.state", i);
+                ret = dict_get_str (dict, key, &state);
+                if (ret)
+                        goto out;
+
+                cli_out ("\nHostname: %s\nUuid: %s\nState: %s (%s)",
+                         hostname_buf, uuid_buf, state, connected_str);
+                i++;
+        }
+
+        ret = 0;
+out:
+        return ret;
+}
+
+int
+gf_cli_output_pool_list (dict_t *dict, int count)
+{
+        int                        ret   = -1;
+        char                       *uuid_buf = NULL;
+        char                       *hostname_buf = NULL;
+        int32_t                    i = 1;
+        char                       key[256] = {0,};
+        int32_t                    connected = 0;
+        char                       *connected_str = NULL;
+
+        if (count >= 1)
+                cli_out ("UUID\t\t\t\t\tHostname\tState");
+
+        while ( i <= count) {
+                snprintf (key, 256, "friend%d.uuid", i);
+                ret = dict_get_str (dict, key, &uuid_buf);
+                if (ret)
+                        goto out;
+
+                snprintf (key, 256, "friend%d.hostname", i);
+                ret = dict_get_str (dict, key, &hostname_buf);
+                if (ret)
+                        goto out;
+
+                snprintf (key, 256, "friend%d.connected", i);
+                ret = dict_get_int32 (dict, key, &connected);
+                if (ret)
+                        goto out;
+                if (connected)
+                        connected_str = "Connected";
+                else
+                        connected_str = "Disconnected";
+
+                cli_out ("%s\t%-9s\t%s ", uuid_buf, hostname_buf,
+                         connected_str);
+                i++;
+        }
+
+        ret = 0;
+out:
+        return ret;
+}
+
+/* function pointer for gf_cli_output_{pool_list,peer_status} */
+typedef int (*cli_friend_output_fn) (dict_t*, int);
+
+int
+gf_cli_list_friends_cbk (struct rpc_req *req, struct iovec *iov,
+                             int count, void *myframe)
+{
+        gf1_cli_peer_list_rsp      rsp   = {0,};
+        int                        ret   = -1;
+        dict_t                    *dict = NULL;
+        char                       msg[1024] = {0,};
+        char                      *cmd = NULL;
+        cli_friend_output_fn       friend_output_fn;
+        call_frame_t              *frame = NULL;
+        unsigned long              flags = 0;
+
+        frame = myframe;
+        flags = (long)frame->local;
+
+        if (flags == GF_CLI_LIST_POOL_NODES) {
+                cmd = "pool list";
+                friend_output_fn = &gf_cli_output_pool_list;
+        } else {
+                cmd = "peer status";
+                friend_output_fn = &gf_cli_output_peer_status;
+        }
+
+        /* 'free' the flags set by gf_cli_list_friends */
+        frame->local = NULL;
+
         if (-1 == req->rpc_status) {
                 goto out;
         }
 
         ret = xdr_to_generic (*iov, &rsp, (xdrproc_t)xdr_gf1_cli_peer_list_rsp);
         if (ret < 0) {
-                gf_log ("", GF_LOG_ERROR, "error");
+                gf_log (((call_frame_t *) myframe)->this->name, GF_LOG_ERROR,
+                        "Failed to decode xdr response");
                 //rsp.op_ret   = -1;
                 //rsp.op_errno = EINVAL;
                 goto out;
         }
 
-        gf_log ("cli", GF_LOG_INFO, "Received resp to list: %d",
+        gf_log ("cli", GF_LOG_DEBUG, "Received resp to list: %d",
                 rsp.op_ret);
 
         ret = rsp.op_ret;
@@ -300,7 +344,19 @@ gf_cli3_1_list_friends_cbk (struct rpc_req *req, struct iovec *iov,
         if (!rsp.op_ret) {
 
                 if (!rsp.friends.friends_len) {
-                        cli_out ("No peers present");
+                        snprintf (msg, sizeof (msg),
+                                  "%s: No peers present", cmd);
+                        if (global_state->mode & GLUSTER_MODE_XML) {
+                                ret = cli_xml_output_peer_status (dict,
+                                                                  rsp.op_ret,
+                                                                  rsp.op_errno,
+                                                                  msg);
+                                if (ret)
+                                        gf_log ("cli", GF_LOG_ERROR,
+                                                "Error outputting to xml");
+                                goto out;
+                        }
+                        cli_err ("%s", msg);
                         ret = 0;
                         goto out;
                 }
@@ -322,69 +378,34 @@ gf_cli3_1_list_friends_cbk (struct rpc_req *req, struct iovec *iov,
                         goto out;
                 }
 
-#if (HAVE_LIB_XML)
                 if (global_state->mode & GLUSTER_MODE_XML) {
-                        ret = cli_xml_output_dict ("peerStatus", dict,
-                                                   rsp.op_ret, rsp.op_errno,
-                                                   NULL);
+                        ret = cli_xml_output_peer_status (dict, rsp.op_ret,
+                                                          rsp.op_errno, msg);
                         if (ret)
                                 gf_log ("cli", GF_LOG_ERROR,
-                                        "Error ouputting to xml");
+                                        "Error outputting to xml");
                         goto out;
                 }
-#endif
-                ret = dict_get_int32 (dict, "count", &count);
 
+                ret = dict_get_int32 (dict, "count", &count);
                 if (ret) {
                         goto out;
                 }
 
-                cli_out ("Number of Peers: %d", count);
-
-                while ( i <= count) {
-                        snprintf (key, 256, "friend%d.uuid", i);
-                        ret = dict_get_str (dict, key, &uuid_buf);
-                        if (ret)
-                                goto out;
-
-                        snprintf (key, 256, "friend%d.hostname", i);
-                        ret = dict_get_str (dict, key, &hostname_buf);
-                        if (ret)
-                                goto out;
-
-                        snprintf (key, 256, "friend%d.connected", i);
-                        ret = dict_get_int32 (dict, key, &connected);
-                        if (ret)
-                                goto out;
-                        if (connected)
-                                connected_str = "Connected";
-                        else
-                                connected_str = "Disconnected";
-
-                        snprintf (key, 256, "friend%d.port", i);
-                        ret = dict_get_int32 (dict, key, &port);
-                        if (ret)
-                                goto out;
-
-                        snprintf (key, 256, "friend%d.state", i);
-                        ret = dict_get_str (dict, key, &state);
-                        if (ret)
-                                goto out;
-
-                        if (!port) {
-                                cli_out ("\nHostname: %s\nUuid: %s\nState: %s "
-                                         "(%s)",
-                                         hostname_buf, uuid_buf, state,
-                                         connected_str);
-                        } else {
-                                cli_out ("\nHostname: %s\nPort: %d\nUuid: %s\n"
-                                         "State: %s (%s)", hostname_buf, port,
-                                         uuid_buf, state, connected_str);
-                        }
-                        i++;
+                ret = friend_output_fn (dict, count);
+                if (ret) {
+                        goto out;
                 }
         } else {
-                ret = -1;
+                if (global_state->mode & GLUSTER_MODE_XML) {
+                        ret = cli_xml_output_peer_status (dict, rsp.op_ret,
+                                                          rsp.op_errno, NULL);
+                        if (ret)
+                                gf_log ("cli", GF_LOG_ERROR,
+                                        "Error outputting to xml");
+                } else {
+                        ret = -1;
+                }
                 goto out;
         }
 
@@ -394,7 +415,7 @@ gf_cli3_1_list_friends_cbk (struct rpc_req *req, struct iovec *iov,
 out:
         cli_cmd_broadcast_response (ret);
         if (ret)
-                cli_out ("Peer status unsuccessful");
+                cli_err ("%s: failed", cmd);
 
         if (dict)
                 dict_destroy (dict);
@@ -428,14 +449,37 @@ cli_out_options ( char *substr, char *optstr, char *valstr)
         cli_out ("%s: %s",ptr2 , valstr);
 }
 
+static int
+_gf_cli_output_volinfo_opts (dict_t *d, char *k,
+                             data_t *v, void *tmp)
+{
+        int     ret   = 0;
+        char   *key   = NULL;
+        char   *ptr   = NULL;
+        data_t *value = NULL;
+
+        key = tmp;
+
+        ptr = strstr (k, "option.");
+        if (ptr) {
+                value = v;
+                if (!value) {
+                        ret = -1;
+                        goto out;
+                }
+                cli_out_options (key, k, v->data);
+        }
+out:
+        return ret;
+}
+
 
 int
-gf_cli3_1_get_volume_cbk (struct rpc_req *req, struct iovec *iov,
+gf_cli_get_volume_cbk (struct rpc_req *req, struct iovec *iov,
                           int count, void *myframe)
 {
         int                        ret                  = -1;
         int                        opt_count            = 0;
-        int                        k                    = 0;
         int32_t                    i                    = 0;
         int32_t                    j                    = 1;
         int32_t                    status               = 0;
@@ -446,24 +490,27 @@ gf_cli3_1_get_volume_cbk (struct rpc_req *req, struct iovec *iov,
         int32_t                    replica_count        = 0;
         int32_t                    vol_type             = 0;
         int32_t                    transport            = 0;
-        char                      *ptr                  = NULL;
         char                      *volume_id_str        = NULL;
         char                      *brick                = NULL;
         char                      *volname              = NULL;
         dict_t                    *dict                 = NULL;
-        data_pair_t               *pairs                = NULL;
-        data_t                    *value                = NULL;
         cli_local_t               *local                = NULL;
         char                       key[1024]            = {0};
         char                       err_str[2048]        = {0};
         gf_cli_rsp                 rsp                  = {0};
+        char                      *caps                 = NULL;
+        int                        k __attribute__((unused)) = 0;
+        // snap_volume variable helps in showing whether a volume is a normal
+        //volume or a volume for the snapshot
+        int32_t                    snap_volume          = 0;
 
         if (-1 == req->rpc_status)
                 goto out;
 
         ret = xdr_to_generic (*iov, &rsp, (xdrproc_t)xdr_gf_cli_rsp);
         if (ret < 0) {
-                gf_log ("cli", GF_LOG_ERROR, "error");
+                gf_log (((call_frame_t *) myframe)->this->name, GF_LOG_ERROR,
+                        "Failed to decode xdr response");
                 goto out;
         }
 
@@ -476,11 +523,9 @@ gf_cli3_1_get_volume_cbk (struct rpc_req *req, struct iovec *iov,
         }
 
         if (!rsp.dict.dict_len) {
-#if (HAVE_LIB_XML)
                 if (global_state->mode & GLUSTER_MODE_XML)
                         goto xml_output;
-#endif
-                cli_out ("No volumes present");
+                cli_err ("No volumes present");
                 ret = 0;
                 goto out;
         }
@@ -523,20 +568,15 @@ gf_cli3_1_get_volume_cbk (struct rpc_req *req, struct iovec *iov,
                                   "Volume %s does not exist",
                                   local->get_vol.volname);
                         ret = -1;
-#if (HAVE_LIB_XML)
                         if (!(global_state->mode & GLUSTER_MODE_XML))
-#endif
-                        {
                                 goto out;
-                        }
                 }
         }
 
-#if (HAVE_LIB_XML)
 xml_output:
         if (global_state->mode & GLUSTER_MODE_XML) {
                 /* For GET_NEXT_VOLUME output is already begun in
-                 * and will also end in gf_cli3_1_get_next_volume()
+                 * and will also end in gf_cli_get_next_volume()
                  */
                 if (local->get_vol.flags == GF_CLI_GET_VOLUME) {
                         ret = cli_xml_output_vol_info_begin
@@ -566,7 +606,6 @@ xml_output:
                 }
                 goto out;
         }
-#endif
 
         while ( i < count) {
                 cli_out (" ");
@@ -585,6 +624,11 @@ xml_output:
                 if (ret)
                         goto out;
 
+                snprintf (key, sizeof (key), "volume%d.snap_volume", i);
+                ret = dict_get_int32 (dict, key, &snap_volume);
+                if (ret)
+                        goto out;
+
                 snprintf (key, 256, "volume%d.brick_count", i);
                 ret = dict_get_int32 (dict, key, &brick_count);
                 if (ret)
@@ -622,9 +666,47 @@ xml_output:
                         vol_type = type + 3;
 
                 cli_out ("Volume Name: %s", volname);
-                cli_out ("Type: %s", cli_volume_type[vol_type]);
+                cli_out ("Type: %s", cli_vol_type_str[vol_type]);
                 cli_out ("Volume ID: %s", volume_id_str);
-                cli_out ("Status: %s", cli_volume_status[status]);
+                cli_out ("Status: %s", cli_vol_status_str[status]);
+                if (snap_volume)
+                        cli_out ("Snap Volume: %s", "yes");
+                else
+                        cli_out ("Snap Volume: %s", "no");
+
+#ifdef HAVE_BD_XLATOR
+                k = 0;
+                memset (key, 0, sizeof (key));
+                snprintf (key, sizeof (key), "volume%d.xlator%d", i, k);
+                ret = dict_get_str (dict, key, &caps);
+                if (ret)
+                        goto next;
+                do {
+                        j = 0;
+                        cli_out ("Xlator %d: %s", k + 1, caps);
+                        do {
+                                memset (key, 0, sizeof (key));
+                                snprintf (key, sizeof (key),
+                                          "volume%d.xlator%d.caps%d",
+                                          i, k, j++);
+                                ret = dict_get_str (dict, key, &caps);
+                                if (ret)
+                                        break;
+                                cli_out ("Capability %d: %s", j, caps);
+                        } while (1);
+
+                        memset (key, 0, sizeof (key));
+                        snprintf (key, sizeof (key),
+                                  "volume%d.xlator%d", i, ++k);
+                        ret = dict_get_str (dict, key, &caps);
+                        if (ret)
+                                break;
+                } while (1);
+
+next:
+#else
+                caps = 0; /* Avoid compiler warnings when BD not enabled */
+#endif
 
                 if (type == GF_CLUSTER_TYPE_STRIPE_REPLICATE) {
                         cli_out ("Number of Bricks: %d x %d x %d = %d",
@@ -632,10 +714,8 @@ xml_output:
                                  stripe_count,
                                  replica_count,
                                  brick_count);
-
                 } else if (type == GF_CLUSTER_TYPE_NONE) {
                         cli_out ("Number of Bricks: %d", brick_count);
-
                 } else {
                         /* For both replicate and stripe, dist_count is
                            good enough */
@@ -663,15 +743,15 @@ xml_output:
                                 goto out;
 
                         cli_out ("Brick%d: %s", j, brick);
+#ifdef HAVE_BD_XLATOR
+                        snprintf (key, 256, "volume%d.vg%d", i, j);
+                        ret = dict_get_str (dict, key, &caps);
+                        if (!ret)
+                                cli_out ("Brick%d VG: %s", j, caps);
+#endif
                         j++;
                 }
 
-                pairs = dict->members_list;
-                if (!pairs) {
-                        ret = -1;
-                        goto out;
-                }
-
                 snprintf (key, 256, "volume%d.opt_count",i);
                 ret = dict_get_int32 (dict, key, &opt_count);
                 if (ret)
@@ -681,26 +761,12 @@ xml_output:
                         goto out;
 
                 cli_out ("Options Reconfigured:");
-                k = 0;
 
-                while (k < opt_count) {
-
-                        snprintf (key, 256, "volume%d.option.",i);
-                        while (pairs) {
-                                ptr = strstr (pairs->key, "option.");
-                                if (ptr) {
-                                        value = pairs->value;
-                                        if (!value) {
-                                                ret = -1;
-                                                goto out;
-                                        }
-                                        cli_out_options (key, pairs->key,
-                                                         value->data);
-                                }
-                                pairs = pairs->next;
-                        }
-                        k++;
-                }
+                snprintf (key, 256, "volume%d.option.",i);
+
+                ret = dict_foreach (dict, _gf_cli_output_volinfo_opts, key);
+                if (ret)
+                        goto out;
 
                 i++;
         }
@@ -710,23 +776,21 @@ xml_output:
 out:
         cli_cmd_broadcast_response (ret);
         if (ret)
-                cli_out ("%s", err_str);
+                cli_err ("%s", err_str);
 
         if (dict)
                 dict_destroy (dict);
 
-        if (rsp.dict.dict_val)
-                free (rsp.dict.dict_val);
+        free (rsp.dict.dict_val);
 
-        if (rsp.op_errstr)
-                free (rsp.op_errstr);
+        free (rsp.op_errstr);
 
         gf_log ("cli", GF_LOG_INFO, "Returning: %d", ret);
         return ret;
 }
 
 int
-gf_cli3_1_create_volume_cbk (struct rpc_req *req, struct iovec *iov,
+gf_cli_create_volume_cbk (struct rpc_req *req, struct iovec *iov,
                              int count, void *myframe)
 {
         gf_cli_rsp              rsp   = {0,};
@@ -734,61 +798,70 @@ gf_cli3_1_create_volume_cbk (struct rpc_req *req, struct iovec *iov,
         cli_local_t             *local = NULL;
         char                    *volname = NULL;
         dict_t                  *dict = NULL;
+        dict_t                  *rsp_dict = NULL;
 
         if (-1 == req->rpc_status) {
                 goto out;
         }
 
         local = ((call_frame_t *) (myframe))->local;
-        ((call_frame_t *) (myframe))->local = NULL;
 
         ret = xdr_to_generic (*iov, &rsp, (xdrproc_t)xdr_gf_cli_rsp);
         if (ret < 0) {
-                gf_log ("", GF_LOG_ERROR, "error");
+                gf_log (((call_frame_t *) myframe)->this->name, GF_LOG_ERROR,
+                        "Failed to decode xdr response");
                 goto out;
         }
 
+        gf_log ("cli", GF_LOG_INFO, "Received resp to create volume");
+
         dict = local->dict;
 
         ret = dict_get_str (dict, "volname", &volname);
+        if (ret)
+                goto out;
 
-        gf_log ("cli", GF_LOG_INFO, "Received resp to create volume");
-
-#if (HAVE_LIB_XML)
         if (global_state->mode & GLUSTER_MODE_XML) {
-                ret = cli_xml_output_dict ("volCreate", dict, rsp.op_ret,
-                                           rsp.op_errno, rsp.op_errstr);
+                if (rsp.op_ret == 0) {
+                        rsp_dict = dict_new ();
+                        ret = dict_unserialize (rsp.dict.dict_val,
+                                                rsp.dict.dict_len,
+                                                &rsp_dict);
+                        if (ret) {
+                                gf_log ("cli", GF_LOG_ERROR,
+                                        "Failed rsp_dict unserialization");
+                                goto out;
+                        }
+                }
+
+                ret = cli_xml_output_vol_create (rsp_dict, rsp.op_ret,
+                                                 rsp.op_errno, rsp.op_errstr);
                 if (ret)
                         gf_log ("cli", GF_LOG_ERROR,
                                 "Error outputting to xml");
                 goto out;
         }
-#endif
 
         if (rsp.op_ret && strcmp (rsp.op_errstr, ""))
-                cli_out ("%s", rsp.op_errstr);
+                cli_err ("volume create: %s: failed: %s", volname,
+                         rsp.op_errstr);
+        else if (rsp.op_ret)
+                cli_err ("volume create: %s: failed", volname);
         else
-                cli_out ("Creation of volume %s has been %s", volname,
-                                (rsp.op_ret) ? "unsuccessful":
-                                "successful. Please start the volume to "
-                                "access data.");
+                cli_out ("volume create: %s: success: "
+                         "please start the volume to access data", volname);
+
         ret = rsp.op_ret;
 
 out:
         cli_cmd_broadcast_response (ret);
-        if (dict)
-                dict_unref (dict);
-        if (local)
-                cli_local_wipe (local);
-        if (rsp.dict.dict_val)
-                free (rsp.dict.dict_val);
-        if (rsp.op_errstr)
-                free (rsp.op_errstr);
+        free (rsp.dict.dict_val);
+        free (rsp.op_errstr);
         return ret;
 }
 
 int
-gf_cli3_1_delete_volume_cbk (struct rpc_req *req, struct iovec *iov,
+gf_cli_delete_volume_cbk (struct rpc_req *req, struct iovec *iov,
                              int count, void *myframe)
 {
         gf_cli_rsp              rsp   = {0,};
@@ -797,47 +870,197 @@ gf_cli3_1_delete_volume_cbk (struct rpc_req *req, struct iovec *iov,
         char                    *volname = NULL;
         call_frame_t            *frame = NULL;
         dict_t                  *dict = NULL;
+        dict_t                  *rsp_dict = NULL;
 
         if (-1 == req->rpc_status) {
                 goto out;
         }
 
+        frame = myframe;
+
         ret = xdr_to_generic (*iov, &rsp, (xdrproc_t)xdr_gf_cli_rsp);
         if (ret < 0) {
-                gf_log ("", GF_LOG_ERROR, "error");
+                gf_log (frame->this->name, GF_LOG_ERROR,
+                        "Failed to decode xdr response");
                 goto out;
         }
 
-        frame = myframe;
         local = frame->local;
-        frame->local = NULL;
 
         if (local)
                 dict = local->dict;
         ret = dict_get_str (dict, "volname", &volname);
         if (ret) {
-                gf_log (THIS->name, GF_LOG_ERROR,
+                gf_log (frame->this->name, GF_LOG_ERROR,
                         "dict get failed");
                 goto out;
         }
 
         gf_log ("cli", GF_LOG_INFO, "Received resp to delete volume");
 
-#if (HAVE_LIB_XML)
         if (global_state->mode & GLUSTER_MODE_XML) {
-                ret = cli_xml_output_dict ("volDelete", dict, rsp.op_ret,
+                if (rsp.op_ret == 0) {
+                        rsp_dict = dict_new ();
+                        ret = dict_unserialize (rsp.dict.dict_val,
+                                                rsp.dict.dict_len,
+                                                &rsp_dict);
+                        if (ret) {
+                                gf_log ("cli", GF_LOG_ERROR,
+                                        "Failed rsp_dict unserialization");
+                                goto out;
+                        }
+                }
+
+                ret = cli_xml_output_generic_volume ("volDelete", rsp_dict,
+                                                     rsp.op_ret, rsp.op_errno,
+                                                     rsp.op_errstr);
+                if (ret)
+                        gf_log ("cli", GF_LOG_ERROR,
+                                "Error outputting to xml");
+                goto out;
+        }
+
+        if (rsp.op_ret && strcmp (rsp.op_errstr, ""))
+                cli_err ("volume delete: %s: failed: %s", volname,
+                         rsp.op_errstr);
+        else if (rsp.op_ret)
+                cli_err ("volume delete: %s: failed", volname);
+        else
+                cli_out ("volume delete: %s: success", volname);
+
+        ret = rsp.op_ret;
+
+out:
+        cli_cmd_broadcast_response (ret);
+        free (rsp.dict.dict_val);
+
+        gf_log ("", GF_LOG_INFO, "Returning with %d", ret);
+        return ret;
+}
+
+int
+gf_cli3_1_uuid_get_cbk (struct rpc_req *req, struct iovec *iov,
+                        int count, void *myframe)
+{
+        char                    *uuid_str = NULL;
+        gf_cli_rsp              rsp   = {0,};
+        int                     ret   = -1;
+        cli_local_t             *local = NULL;
+        call_frame_t            *frame = NULL;
+        dict_t                  *dict = NULL;
+
+        if (-1 == req->rpc_status)
+                goto out;
+
+        frame = myframe;
+
+        ret = xdr_to_generic (*iov, &rsp, (xdrproc_t)xdr_gf_cli_rsp);
+        if (ret < 0) {
+                gf_log (frame->this->name, GF_LOG_ERROR,
+                        "Failed to decode xdr response");
+                goto out;
+        }
+
+        local = frame->local;
+        frame->local = NULL;
+
+        gf_log ("cli", GF_LOG_INFO, "Received resp to uuid get");
+
+        dict = dict_new ();
+        if (!dict) {
+                ret = -1;
+                goto out;
+        }
+
+        ret = dict_unserialize (rsp.dict.dict_val, rsp.dict.dict_len,
+                                &dict);
+        if (ret) {
+                gf_log ("cli", GF_LOG_ERROR, "Failed to unserialize "
+                        "response for uuid get");
+                goto out;
+        }
+
+        ret = dict_get_str (dict, "uuid", &uuid_str);
+        if (ret) {
+                gf_log ("cli", GF_LOG_ERROR, "Failed to get uuid "
+                        "from dictionary");
+                goto out;
+        }
+
+        if (global_state->mode & GLUSTER_MODE_XML) {
+                ret = cli_xml_output_dict ("uuidGenerate", dict, rsp.op_ret,
+                                           rsp.op_errno, rsp.op_errstr);
+                if (ret)
+                        gf_log ("cli", GF_LOG_ERROR,
+                                "Error outputting to xml");
+                goto out;
+        }
+
+        if (rsp.op_ret) {
+                if (strcmp (rsp.op_errstr, "") == 0)
+                        cli_err ("Get uuid was unsuccessful");
+                else
+                        cli_err ("%s", rsp.op_errstr);
+
+        } else {
+                cli_out ("UUID: %s", uuid_str);
+
+        }
+        ret = rsp.op_ret;
+
+out:
+        cli_cmd_broadcast_response (ret);
+        cli_local_wipe (local);
+        if (rsp.dict.dict_val)
+                free (rsp.dict.dict_val);
+        if (dict)
+                dict_unref (dict);
+
+        gf_log ("", GF_LOG_DEBUG, "Returning with %d", ret);
+        return ret;
+}
+
+int
+gf_cli3_1_uuid_reset_cbk (struct rpc_req *req, struct iovec *iov,
+                             int count, void *myframe)
+{
+        gf_cli_rsp              rsp   = {0,};
+        int                     ret   = -1;
+        cli_local_t             *local = NULL;
+        call_frame_t            *frame = NULL;
+        dict_t                  *dict = NULL;
+
+        if (-1 == req->rpc_status) {
+                goto out;
+        }
+
+        frame = myframe;
+
+        ret = xdr_to_generic (*iov, &rsp, (xdrproc_t)xdr_gf_cli_rsp);
+        if (ret < 0) {
+                gf_log (frame->this->name, GF_LOG_ERROR,
+                        "Failed to decode xdr response");
+                goto out;
+        }
+
+        local = frame->local;
+        frame->local = NULL;
+
+        gf_log ("cli", GF_LOG_INFO, "Received resp to uuid reset");
+
+        if (global_state->mode & GLUSTER_MODE_XML) {
+                ret = cli_xml_output_dict ("uuidReset", dict, rsp.op_ret,
                                            rsp.op_errno, rsp.op_errstr);
                 if (ret)
                         gf_log ("cli", GF_LOG_ERROR,
                                 "Error outputting to xml");
                 goto out;
         }
-#endif
 
         if (rsp.op_ret && strcmp (rsp.op_errstr, ""))
-                cli_out ("%s", rsp.op_errstr);
+                cli_err ("%s", rsp.op_errstr);
         else
-                cli_out ("Deleting volume %s has been %s", volname,
+                cli_out ("resetting the peer uuid has been %s",
                          (rsp.op_ret) ? "unsuccessful": "successful");
         ret = rsp.op_ret;
 
@@ -854,7 +1077,7 @@ out:
 }
 
 int
-gf_cli3_1_start_volume_cbk (struct rpc_req *req, struct iovec *iov,
+gf_cli_start_volume_cbk (struct rpc_req *req, struct iovec *iov,
                              int count, void *myframe)
 {
         gf_cli_rsp              rsp   = {0,};
@@ -863,69 +1086,76 @@ gf_cli3_1_start_volume_cbk (struct rpc_req *req, struct iovec *iov,
         char                    *volname = NULL;
         call_frame_t            *frame = NULL;
         dict_t                  *dict = NULL;
+        dict_t                  *rsp_dict = NULL;
 
         if (-1 == req->rpc_status) {
                 goto out;
         }
 
+        frame = myframe;
+
         ret = xdr_to_generic (*iov, &rsp, (xdrproc_t)xdr_gf_cli_rsp);
         if (ret < 0) {
-                gf_log ("", GF_LOG_ERROR, "error");
+                gf_log (frame->this->name, GF_LOG_ERROR,
+                        "Failed to decode xdr response");
                 goto out;
         }
 
-        frame = myframe;
-
-        if (frame) {
+        if (frame)
                 local = frame->local;
-                frame->local = NULL;
-        }
 
         if (local)
                 dict = local->dict;
 
         ret = dict_get_str (dict, "volname", &volname);
         if (ret) {
-                gf_log (THIS->name, GF_LOG_ERROR, "dict get failed");
+                gf_log (frame->this->name, GF_LOG_ERROR, "dict get failed");
                 goto out;
         }
 
         gf_log ("cli", GF_LOG_INFO, "Received resp to start volume");
 
-#if (HAVE_LIB_XML)
         if (global_state->mode & GLUSTER_MODE_XML) {
-                ret = cli_xml_output_dict ("volStart", dict, rsp.op_ret,
-                                           rsp.op_errno, rsp.op_errstr);
+                if (rsp.op_ret == 0) {
+                        rsp_dict = dict_new ();
+                        ret = dict_unserialize (rsp.dict.dict_val,
+                                                rsp.dict.dict_len,
+                                                &rsp_dict);
+                        if (ret) {
+                                gf_log ("cli", GF_LOG_ERROR,
+                                        "Failed rsp_dict unserialization");
+                                goto out;
+                        }
+                }
+
+                ret = cli_xml_output_generic_volume ("volStart", rsp_dict,
+                                                     rsp.op_ret, rsp.op_errno,
+                                                     rsp.op_errstr);
                 if (ret)
                         gf_log ("cli", GF_LOG_ERROR,
                                 "Error outputting to xml");
                 goto out;
         }
-#endif
 
         if (rsp.op_ret && strcmp (rsp.op_errstr, ""))
-                cli_out ("%s", rsp.op_errstr);
+                cli_err ("volume start: %s: failed: %s", volname,
+                         rsp.op_errstr);
+        else if (rsp.op_ret)
+                cli_err ("volume start: %s: failed", volname);
         else
-                cli_out ("Starting volume %s has been %s", volname,
-                        (rsp.op_ret) ? "unsuccessful": "successful");
+                cli_out ("volume start: %s: success", volname);
 
         ret = rsp.op_ret;
 
 out:
         cli_cmd_broadcast_response (ret);
-        if (local)
-                cli_local_wipe (local);
-        if (rsp.dict.dict_val)
-                free (rsp.dict.dict_val);
-        if (rsp.op_errstr)
-                free (rsp.op_errstr);
-        if (dict)
-                dict_unref (dict);
+        free (rsp.dict.dict_val);
+        free (rsp.op_errstr);
         return ret;
 }
 
 int
-gf_cli3_1_stop_volume_cbk (struct rpc_req *req, struct iovec *iov,
+gf_cli_stop_volume_cbk (struct rpc_req *req, struct iovec *iov,
                              int count, void *myframe)
 {
         gf_cli_rsp            rsp   = {0,};
@@ -934,19 +1164,21 @@ gf_cli3_1_stop_volume_cbk (struct rpc_req *req, struct iovec *iov,
         char                  *volname = NULL;
         call_frame_t          *frame = NULL;
         dict_t                *dict = NULL;
+        dict_t                *rsp_dict = NULL;
 
         if (-1 == req->rpc_status) {
                 goto out;
         }
 
+        frame = myframe;
+
         ret = xdr_to_generic (*iov, &rsp, (xdrproc_t)xdr_gf_cli_rsp);
         if (ret < 0) {
-                gf_log ("", GF_LOG_ERROR, "error");
+                gf_log (frame->this->name, GF_LOG_ERROR,
+                        "Failed to decode xdr response");
                 goto out;
         }
 
-        frame = myframe;
-
         if (frame)
                 local = frame->local;
 
@@ -954,7 +1186,7 @@ gf_cli3_1_stop_volume_cbk (struct rpc_req *req, struct iovec *iov,
                 dict = local->dict;
                 ret = dict_get_str (dict, "volname", &volname);
                 if (ret) {
-                        gf_log (THIS->name, GF_LOG_ERROR,
+                        gf_log (frame->this->name, GF_LOG_ERROR,
                                 "Unable to get volname from dict");
                         goto out;
                 }
@@ -962,37 +1194,47 @@ gf_cli3_1_stop_volume_cbk (struct rpc_req *req, struct iovec *iov,
 
         gf_log ("cli", GF_LOG_INFO, "Received resp to stop volume");
 
-#if (HAVE_LIB_XML)
         if (global_state->mode & GLUSTER_MODE_XML) {
-                ret = cli_xml_output_dict ("volStop", dict, rsp.op_ret,
-                                           rsp.op_errno, rsp.op_errstr);
+                if (rsp.op_ret == 0) {
+                        rsp_dict = dict_new ();
+                        ret = dict_unserialize (rsp.dict.dict_val,
+                                                rsp.dict.dict_len,
+                                                &rsp_dict);
+                        if (ret) {
+                                gf_log ("cli", GF_LOG_ERROR,
+                                        "Failed rsp_dict unserialization");
+                                goto out;
+                        }
+                }
+
+                ret = cli_xml_output_generic_volume ("volStop", rsp_dict,
+                                                     rsp.op_ret, rsp.op_errno,
+                                                     rsp.op_errstr);
                 if (ret)
                         gf_log ("cli", GF_LOG_ERROR,
                                 "Error outputting to xml");
                 goto out;
         }
-#endif
 
         if (rsp.op_ret && strcmp (rsp.op_errstr, ""))
-                cli_out ("%s", rsp.op_errstr);
+                cli_err ("volume stop: %s: failed: %s", volname, rsp.op_errstr);
+        else if (rsp.op_ret)
+                cli_err ("volume stop: %s: failed", volname);
         else
-                cli_out ("Stopping volume %s has been %s", volname,
-                        (rsp.op_ret) ? "unsuccessful": "successful");
+                cli_out ("volume stop: %s: success", volname);
+
         ret = rsp.op_ret;
 
 out:
         cli_cmd_broadcast_response (ret);
-        if (rsp.op_errstr)
-                free (rsp.op_errstr);
-        if (rsp.dict.dict_val)
-                free (rsp.dict.dict_val);
-        if (local)
-                cli_local_wipe (local);
+        free (rsp.op_errstr);
+        free (rsp.dict.dict_val);
+
         return ret;
 }
 
 int
-gf_cli3_1_defrag_volume_cbk (struct rpc_req *req, struct iovec *iov,
+gf_cli_defrag_volume_cbk (struct rpc_req *req, struct iovec *iov,
                              int count, void *myframe)
 {
         gf_cli_rsp               rsp     = {0,};
@@ -1010,40 +1252,45 @@ gf_cli3_1_defrag_volume_cbk (struct rpc_req *req, struct iovec *iov,
         char                     msg[1024] = {0,};
         gf_defrag_status_t       status_rcd = GF_DEFRAG_STATUS_NOT_STARTED;
         int32_t                  counter = 0;
-        char                    *node_uuid = NULL;
+        char                    *node_name = NULL;
         char                     key[256] = {0,};
         int32_t                  i = 1;
+        uint64_t                 failures = 0;
+        uint64_t                 skipped = 0;
+        double                   elapsed = 0;
+        char                    *size_str = NULL;
+        char                    *task_id_str = NULL;
 
         if (-1 == req->rpc_status) {
                 goto out;
         }
 
+        frame = myframe;
+
         ret = xdr_to_generic (*iov, &rsp,
                               (xdrproc_t)xdr_gf_cli_rsp);
         if (ret < 0) {
-                gf_log ("", GF_LOG_ERROR, "error");
+                gf_log (frame->this->name, GF_LOG_ERROR,
+                        "Failed to decode xdr response");
                 goto out;
         }
 
-        frame = myframe;
-
         if (frame)
                 local = frame->local;
 
-        if (local) {
+        if (local)
                 local_dict = local->dict;
-        }
 
         ret = dict_get_str (local_dict, "volname", &volname);
         if (ret) {
-                gf_log (THIS->name, GF_LOG_ERROR,
+                gf_log (frame->this->name, GF_LOG_ERROR,
                         "Failed to get volname");
                 goto out;
         }
 
         ret = dict_get_int32 (local_dict, "rebalance-command", (int32_t*)&cmd);
         if (ret) {
-                gf_log (THIS->name, GF_LOG_ERROR,
+                gf_log (frame->this->name, GF_LOG_ERROR,
                         "Failed to get command");
                 goto out;
         }
@@ -1063,24 +1310,27 @@ gf_cli3_1_defrag_volume_cbk (struct rpc_req *req, struct iovec *iov,
                 }
         }
 
-        if (!((cmd == GF_DEFRAG_CMD_STOP) || (cmd == GF_DEFRAG_CMD_STATUS))) {
-                /* All other possibility is about starting a volume */
-                if (rsp.op_ret && strcmp (rsp.op_errstr, ""))
+        if (!((cmd == GF_DEFRAG_CMD_STOP) || (cmd == GF_DEFRAG_CMD_STATUS)) &&
+             !(global_state->mode & GLUSTER_MODE_XML)) {
+                /* All other possibilites are about starting a rebalance */
+                ret = dict_get_str (dict, GF_REBALANCE_TID_KEY, &task_id_str);
+                if (rsp.op_ret && strcmp (rsp.op_errstr, "")) {
                         snprintf (msg, sizeof (msg), "%s", rsp.op_errstr);
-                else
-                        snprintf (msg, sizeof (msg),
-                                  "Starting rebalance on volume %s has been %s",
-                                  volname, (rsp.op_ret) ? "unsuccessful":
-                                  "successful");
+                } else {
+                        if (!rsp.op_ret) {
+                                snprintf (msg, sizeof (msg),
+                                          "Starting rebalance on volume %s has "
+                                          "been successful.\nID: %s", volname,
+                                          task_id_str);
+                        } else {
+                                snprintf (msg, sizeof (msg),
+                                          "Starting rebalance on volume %s has "
+                                          "been unsuccessful.", volname);
+                        }
+                }
                 goto done;
         }
 
-        ret = dict_get_int32 (dict, "count", &counter);
-        if (ret) {
-                gf_log (THIS->name, GF_LOG_ERROR, "count not set");
-                goto out;
-        }
-
         if (cmd == GF_DEFRAG_CMD_STOP) {
                 if (rsp.op_ret == -1) {
                         if (strcmp (rsp.op_errstr, ""))
@@ -1093,8 +1343,12 @@ gf_cli3_1_defrag_volume_cbk (struct rpc_req *req, struct iovec *iov,
                         goto done;
                 } else {
                         snprintf (msg, sizeof (msg),
-                                 "Stopped rebalance process on volume %s \n",
-                                  volname);
+                                  "rebalance process may be in the middle of a "
+                                  "file migration.\nThe process will be fully "
+                                  "stopped once the migration of the file is "
+                                  "complete.\nPlease check rebalance process "
+                                  "for completion before doing any further "
+                                  "brick related tasks on the volume.");
                 }
         }
         if (cmd == GF_DEFRAG_CMD_STATUS) {
@@ -1109,96 +1363,118 @@ gf_cli3_1_defrag_volume_cbk (struct rpc_req *req, struct iovec *iov,
                         goto done;
                 }
         }
-        cli_out ("%40s %16s %13s %13s %14s", "Node", "Rebalanced-files",
-                 "size", "scanned", "status");
-        cli_out ("%40s %16s %13s %13s %14s", "---------", "-----------",
-                 "-----------", "-----------", "------------");
 
+        if (global_state->mode & GLUSTER_MODE_XML) {
+                ret = cli_xml_output_vol_rebalance (cmd, dict, rsp.op_ret,
+                                                    rsp.op_errno,
+                                                    rsp.op_errstr);
+                goto out;
+        }
+
+        ret = dict_get_int32 (dict, "count", &counter);
+        if (ret) {
+                gf_log (frame->this->name, GF_LOG_ERROR, "count not set");
+                goto out;
+        }
+
+        cli_out ("%40s %16s %13s %13s %13s %13s %20s %18s", "Node",
+                 "Rebalanced-files", "size", "scanned", "failures", "skipped",
+                 "status", "run time in secs");
+        cli_out ("%40s %16s %13s %13s %13s %13s %20s %18s", "---------",
+                 "-----------", "-----------", "-----------", "-----------",
+                 "-----------", "------------", "--------------");
         do {
-                snprintf (key, 256, "node-uuid-%d", i);
-                ret = dict_get_str (dict, key, &node_uuid);
+                snprintf (key, 256, "node-name-%d", i);
+                ret = dict_get_str (dict, key, &node_name);
                 if (ret)
-                        gf_log (THIS->name, GF_LOG_TRACE,
-                                "failed to get node-uuid");
+                        gf_log (frame->this->name, GF_LOG_TRACE,
+                                "failed to get node-name");
 
+                memset (key, 0, 256);
                 snprintf (key, 256, "files-%d", i);
                 ret = dict_get_uint64 (dict, key, &files);
                 if (ret)
-                        gf_log (THIS->name, GF_LOG_TRACE,
+                        gf_log (frame->this->name, GF_LOG_TRACE,
                                 "failed to get file count");
 
+                memset (key, 0, 256);
                 snprintf (key, 256, "size-%d", i);
                 ret = dict_get_uint64 (dict, key, &size);
                 if (ret)
-                        gf_log (THIS->name, GF_LOG_TRACE,
+                        gf_log (frame->this->name, GF_LOG_TRACE,
                                 "failed to get size of xfer");
 
+                memset (key, 0, 256);
                 snprintf (key, 256, "lookups-%d", i);
                 ret = dict_get_uint64 (dict, key, &lookup);
                 if (ret)
-                        gf_log (THIS->name, GF_LOG_TRACE,
+                        gf_log (frame->this->name, GF_LOG_TRACE,
                                 "failed to get lookedup file count");
 
+                memset (key, 0, 256);
                 snprintf (key, 256, "status-%d", i);
                 ret = dict_get_int32 (dict, key, (int32_t *)&status_rcd);
                 if (ret)
-                        gf_log (THIS->name, GF_LOG_TRACE,
+                        gf_log (frame->this->name, GF_LOG_TRACE,
                                 "failed to get status");
 
-                switch (status_rcd) {
-                case GF_DEFRAG_STATUS_NOT_STARTED:
-                        status = "not started";
-                        break;
-                case GF_DEFRAG_STATUS_STARTED:
-                        status = "in progress";
-                        break;
-                case GF_DEFRAG_STATUS_STOPPED:
-                        status = "stopped";
-                        break;
-                case GF_DEFRAG_STATUS_COMPLETE:
-                        status = "completed";
-                        break;
-                case GF_DEFRAG_STATUS_FAILED:
-                        status = "failed";
-                        break;
-                }
-                cli_out ("%40s %16"PRId64 "%13"PRId64 "%13"PRId64 "%14s", node_uuid, files,
-                        size, lookup, status);
+                memset (key, 0, 256);
+                snprintf (key, 256, "failures-%d", i);
+                ret = dict_get_uint64 (dict, key, &failures);
+                if (ret)
+                        gf_log (frame->this->name, GF_LOG_TRACE,
+                                "failed to get failures count");
+
+                memset (key, 0, 256);
+                snprintf (key, 256, "skipped-%d", i);
+                ret = dict_get_uint64 (dict, key, &skipped);
+                if (ret)
+                        gf_log (frame->this->name, GF_LOG_TRACE,
+                                "failed to get skipped count");
+                memset (key, 0, 256);
+                snprintf (key, 256, "run-time-%d", i);
+                ret = dict_get_double (dict, key, &elapsed);
+                if (ret)
+                        gf_log (frame->this->name, GF_LOG_TRACE,
+                                "failed to get run-time");
+
+                status = cli_vol_task_status_str[status_rcd];
+                size_str = gf_uint64_2human_readable(size);
+                cli_out ("%40s %16"PRIu64 " %13s" " %13"PRIu64 " %13"PRIu64
+                         " %13"PRIu64 " %20s %18.2f", node_name, files,
+                         size_str, lookup, failures, skipped, status, elapsed);
+                GF_FREE(size_str);
+
                 i++;
         } while (i <= counter);
 
 
 done:
-#if (HAVE_LIB_XML)
-        if (global_state->mode & GLUSTER_MODE_XML) {
-                ret = cli_xml_output_str ("volRebalance", msg, rsp.op_ret,
-                                          status_rcd, rsp.op_errstr);
-                if (ret)
-                        gf_log ("cli", GF_LOG_ERROR,
-                                "Error outputting to xml");
-                goto out;
+        if (global_state->mode & GLUSTER_MODE_XML)
+                cli_xml_output_str ("volRebalance", msg,
+                                    rsp.op_ret, rsp.op_errno,
+                                    rsp.op_errstr);
+        else {
+                if (rsp.op_ret)
+                        cli_err ("volume rebalance: %s: failed: %s", volname,
+                                 msg);
+                else
+                        cli_out ("volume rebalance: %s: success: %s", volname,
+                                 msg);
         }
-#endif
-        cli_out ("%s", msg);
         ret = rsp.op_ret;
 
 out:
-        if (rsp.op_errstr)
-                free (rsp.op_errstr); //malloced by xdr
-        if (rsp.dict.dict_val)
-                free (rsp.dict.dict_val); //malloced by xdr
+        free (rsp.op_errstr); //malloced by xdr
+        free (rsp.dict.dict_val); //malloced by xdr
         if (dict)
                 dict_unref (dict);
-        if (local_dict)
-                dict_unref (local_dict);
-        if (local)
-                cli_local_wipe (local);
         cli_cmd_broadcast_response (ret);
         return ret;
 }
 
 int
-gf_cli3_1_rename_volume_cbk (struct rpc_req *req, struct iovec *iov,
+gf_cli_rename_volume_cbk (struct rpc_req *req, struct iovec *iov,
                              int count, void *myframe)
 {
         gf_cli_rsp              rsp   = {0,};
@@ -1211,7 +1487,8 @@ gf_cli3_1_rename_volume_cbk (struct rpc_req *req, struct iovec *iov,
 
         ret = xdr_to_generic (*iov, &rsp, (xdrproc_t)xdr_gf_cli_rsp);
         if (ret < 0) {
-                gf_log ("", GF_LOG_ERROR, "error");
+                gf_log (((call_frame_t *) myframe)->this->name, GF_LOG_ERROR,
+                        "Failed to decode xdr response");
                 goto out;
         }
 
@@ -1220,7 +1497,6 @@ gf_cli3_1_rename_volume_cbk (struct rpc_req *req, struct iovec *iov,
         snprintf (msg, sizeof (msg), "Rename volume %s",
                   (rsp.op_ret) ? "unsuccessful": "successful");
 
-#if (HAVE_LIB_XML)
         if (global_state->mode & GLUSTER_MODE_XML) {
                 ret = cli_xml_output_str ("volRename", msg, rsp.op_ret,
                                           rsp.op_errno, rsp.op_errstr);
@@ -1229,9 +1505,12 @@ gf_cli3_1_rename_volume_cbk (struct rpc_req *req, struct iovec *iov,
                                 "Error outputting to xml");
                 goto out;
         }
-#endif
 
-        cli_out ("%s", msg);
+        if (rsp.op_ret)
+                cli_err ("volume rename: failed");
+        else
+                cli_out ("volume rename: success");
+
         ret = rsp.op_ret;
 
 out:
@@ -1240,7 +1519,7 @@ out:
 }
 
 int
-gf_cli3_1_reset_volume_cbk (struct rpc_req *req, struct iovec *iov,
+gf_cli_reset_volume_cbk (struct rpc_req *req, struct iovec *iov,
                              int count, void *myframe)
 {
         gf_cli_rsp           rsp   = {0,};
@@ -1253,19 +1532,19 @@ gf_cli3_1_reset_volume_cbk (struct rpc_req *req, struct iovec *iov,
 
         ret = xdr_to_generic (*iov, &rsp, (xdrproc_t)xdr_gf_cli_rsp);
         if (ret < 0) {
-                gf_log ("", GF_LOG_ERROR, "error");
+                gf_log (((call_frame_t *) myframe)->this->name, GF_LOG_ERROR,
+                        "Failed to decode xdr response");
                 goto out;
         }
 
         gf_log ("cli", GF_LOG_INFO, "Received resp to reset");
 
-        if (rsp.op_ret &&  strcmp (rsp.op_errstr, ""))
+        if (strcmp (rsp.op_errstr, ""))
                 snprintf (msg, sizeof (msg), "%s", rsp.op_errstr);
         else
                 snprintf (msg, sizeof (msg), "reset volume %s",
                           (rsp.op_ret) ? "unsuccessful": "successful");
 
-#if (HAVE_LIB_XML)
         if (global_state->mode & GLUSTER_MODE_XML) {
                 ret = cli_xml_output_str ("volReset", msg, rsp.op_ret,
                                           rsp.op_errno, rsp.op_errstr);
@@ -1274,18 +1553,68 @@ gf_cli3_1_reset_volume_cbk (struct rpc_req *req, struct iovec *iov,
                                 "Error outputting to xml");
                 goto out;
         }
-#endif
 
-        cli_out ("%s", msg);
+        if (rsp.op_ret)
+                cli_err ("volume reset: failed: %s", msg);
+        else
+                cli_out ("volume reset: success: %s", msg);
+
         ret = rsp.op_ret;
 
 out:
-                cli_cmd_broadcast_response (ret);
+        cli_cmd_broadcast_response (ret);
         return ret;
 }
 
+char *
+is_server_debug_xlator (void *myframe)
+{
+        call_frame_t         *frame        = NULL;
+        cli_local_t          *local        = NULL;
+        char                 **words       = NULL;
+        char                 *key          = NULL;
+        char                 *value        = NULL;
+        char                 *debug_xlator = NULL;
+
+        frame = myframe;
+        local = frame->local;
+        words = (char **)local->words;
+
+        while (*words != NULL) {
+                if (strstr (*words, "trace") == NULL &&
+                    strstr (*words, "error-gen") == NULL) {
+                        words++;
+                        continue;
+                }
+
+                key = *words;
+                words++;
+                value = *words;
+                if (value == NULL)
+                        break;
+                if (strstr (value, "client")) {
+                        words++;
+                        continue;
+                } else {
+                        if (!(strstr (value, "posix") || strstr (value, "acl")
+                              || strstr (value, "locks") ||
+                              strstr (value, "io-threads") ||
+                              strstr (value, "marker") ||
+                              strstr (value, "index"))) {
+                                words++;
+                                continue;
+                        } else {
+                                debug_xlator = gf_strdup (key);
+                                break;
+                        }
+                }
+        }
+
+        return debug_xlator;
+}
+
 int
-gf_cli3_1_set_volume_cbk (struct rpc_req *req, struct iovec *iov,
+gf_cli_set_volume_cbk (struct rpc_req *req, struct iovec *iov,
                              int count, void *myframe)
 {
         gf_cli_rsp           rsp   = {0,};
@@ -1293,6 +1622,8 @@ gf_cli3_1_set_volume_cbk (struct rpc_req *req, struct iovec *iov,
         dict_t               *dict = NULL;
         char                 *help_str = NULL;
         char                 msg[1024] = {0,};
+        char                 *debug_xlator = _gf_false;
+        char                 tmp_str[512] = {0,};
 
         if (-1 == req->rpc_status) {
                 goto out;
@@ -1300,7 +1631,8 @@ gf_cli3_1_set_volume_cbk (struct rpc_req *req, struct iovec *iov,
 
         ret = xdr_to_generic (*iov, &rsp, (xdrproc_t)xdr_gf_cli_rsp);
         if (ret < 0) {
-                gf_log ("", GF_LOG_ERROR, "error");
+                gf_log (((call_frame_t *) myframe)->this->name, GF_LOG_ERROR,
+                        "Failed to decode xdr response");
                 goto out;
         }
 
@@ -1315,14 +1647,22 @@ gf_cli3_1_set_volume_cbk (struct rpc_req *req, struct iovec *iov,
 
         ret = dict_unserialize (rsp.dict.dict_val, rsp.dict.dict_len, &dict);
 
-        if (ret)
-                goto out;
+        /* For brick processes graph change does not happen on the fly.
+         * The proces has to be restarted. So this is a check from the
+         * volume set option such that if debug xlators such as trace/errorgen
+         * are provided in the set command, warn the user.
+         */
+        debug_xlator = is_server_debug_xlator (myframe);
 
-        if (dict_get_str (dict, "help-str", &help_str))
+        if (dict_get_str (dict, "help-str", &help_str) && !msg[0])
                 snprintf (msg, sizeof (msg), "Set volume %s",
                           (rsp.op_ret) ? "unsuccessful": "successful");
+        if (rsp.op_ret == 0 && debug_xlator) {
+                snprintf (tmp_str, sizeof (tmp_str), "\n%s translator has been "
+                          "added to the server volume file. Please restart the"
+                          " volume for enabling the translator", debug_xlator);
+        }
 
-#if (HAVE_LIB_XML)
         if ((global_state->mode & GLUSTER_MODE_XML) && (help_str == NULL)) {
                 ret = cli_xml_output_str ("volSet", msg, rsp.op_ret,
                                           rsp.op_errno, rsp.op_errstr);
@@ -1331,22 +1671,35 @@ gf_cli3_1_set_volume_cbk (struct rpc_req *req, struct iovec *iov,
                                 "Error outputting to xml");
                 goto out;
         }
-#endif
 
-        if (rsp.op_ret &&  strcmp (rsp.op_errstr, ""))
-                cli_out ("%s", rsp.op_errstr);
-
-        cli_out ("%s", ((help_str == NULL) ? msg : help_str));
+        if (rsp.op_ret) {
+                if (strcmp (rsp.op_errstr, ""))
+                        cli_err ("volume set: failed: %s", rsp.op_errstr);
+                else
+                        cli_err ("volume set: failed");
+        } else {
+                if (help_str == NULL) {
+                        if (debug_xlator == NULL)
+                                cli_out ("volume set: success");
+                        else
+                                cli_out ("volume set: success%s", tmp_str);
+                }else {
+                        cli_out ("%s", help_str);
+                }
+        }
 
         ret = rsp.op_ret;
 
 out:
+        if (dict)
+                dict_unref (dict);
+        GF_FREE (debug_xlator);
         cli_cmd_broadcast_response (ret);
         return ret;
 }
 
 int
-gf_cli3_1_add_brick_cbk (struct rpc_req *req, struct iovec *iov,
+gf_cli_add_brick_cbk (struct rpc_req *req, struct iovec *iov,
                              int count, void *myframe)
 {
         gf_cli_rsp                  rsp   = {0,};
@@ -1359,7 +1712,8 @@ gf_cli3_1_add_brick_cbk (struct rpc_req *req, struct iovec *iov,
 
         ret = xdr_to_generic (*iov, &rsp, (xdrproc_t)xdr_gf_cli_rsp);
         if (ret < 0) {
-                gf_log ("", GF_LOG_ERROR, "error");
+                gf_log (((call_frame_t *) myframe)->this->name, GF_LOG_ERROR,
+                        "Failed to decode xdr response");
                 goto out;
         }
 
@@ -1372,7 +1726,6 @@ gf_cli3_1_add_brick_cbk (struct rpc_req *req, struct iovec *iov,
                 snprintf (msg, sizeof (msg), "Add Brick %s",
                           (rsp.op_ret) ? "unsuccessful": "successful");
 
-#if (HAVE_LIB_XML)
         if (global_state->mode & GLUSTER_MODE_XML) {
                 ret = cli_xml_output_str ("volAddBrick", msg, rsp.op_ret,
                                           rsp.op_errno, rsp.op_errstr);
@@ -1381,17 +1734,17 @@ gf_cli3_1_add_brick_cbk (struct rpc_req *req, struct iovec *iov,
                                 "Error outputting to xml");
                 goto out;
         }
-#endif
 
-        cli_out ("%s", msg);
+        if (rsp.op_ret)
+                cli_err ("volume add-brick: failed: %s", rsp.op_errstr);
+        else
+                cli_out ("volume add-brick: success");
         ret = rsp.op_ret;
 
 out:
         cli_cmd_broadcast_response (ret);
-        if (rsp.dict.dict_val)
-                free (rsp.dict.dict_val);
-        if (rsp.op_errstr)
-                free (rsp.op_errstr);
+        free (rsp.dict.dict_val);
+        free (rsp.op_errstr);
         return ret;
 }
 
@@ -1404,48 +1757,72 @@ gf_cli3_remove_brick_status_cbk (struct rpc_req *req, struct iovec *iov,
         int                      ret     = -1;
         uint64_t                 files   = 0;
         uint64_t                 size    = 0;
+        uint64_t                 lookup  = 0;
         dict_t                  *dict    = NULL;
         char                     msg[1024] = {0,};
+        char                     key[256] = {0,};
+        int32_t                  i       = 1;
+        int32_t                  counter = 0;
+        char                    *node_name = 0;
+        gf_defrag_status_t       status_rcd = GF_DEFRAG_STATUS_NOT_STARTED;
+        uint64_t                 failures = 0;
+        uint64_t                 skipped = 0;
+        double                   elapsed = 0;
+        char                    *size_str = NULL;
+        int32_t                  command = 0;
+        gf1_op_commands          cmd = GF_OP_CMD_NONE;
+        cli_local_t             *local = NULL;
+        call_frame_t            *frame = NULL;
+        char                    *cmd_str = "unknown";
 
         if (-1 == req->rpc_status) {
                 goto out;
         }
 
+        frame = myframe;
+
         ret = xdr_to_generic (*iov, &rsp,
                               (xdrproc_t)xdr_gf_cli_rsp);
         if (ret < 0) {
-                gf_log ("", GF_LOG_ERROR, "error");
+                gf_log (frame->this->name, GF_LOG_ERROR,
+                        "Failed to decode xdr response");
                 goto out;
         }
 
-        ret = rsp.op_ret;
-        if (rsp.op_ret == -1) {
-                if (strcmp (rsp.op_errstr, ""))
-                        cli_out ("%s", rsp.op_errstr);
-                else
-                        cli_out ("failed to get the status of "
-                                 "remove-brick process");
+        if (frame)
+                local = frame->local;
+        ret = dict_get_int32 (local->dict, "command", &command);
+        if (ret)
                 goto out;
-        }
+        cmd = command;
 
-        switch (rsp.op_errno) {
-        case GF_DEFRAG_STATUS_NOT_STARTED:
-                status = "not started";
-                break;
-        case GF_DEFRAG_STATUS_STARTED:
-                status = "in progress";
+        switch (cmd) {
+        case GF_OP_CMD_STOP:
+                cmd_str = "stop";
                 break;
-        case GF_DEFRAG_STATUS_STOPPED:
-                status = "stopped";
+        case GF_OP_CMD_STATUS:
+                cmd_str = "status";
                 break;
-        case GF_DEFRAG_STATUS_COMPLETE:
-                status = "completed";
-                break;
-        case GF_DEFRAG_STATUS_FAILED:
-                status = "failed";
+        default:
                 break;
         }
 
+        ret = rsp.op_ret;
+        if (rsp.op_ret == -1) {
+                if (strcmp (rsp.op_errstr, ""))
+                        snprintf (msg, sizeof (msg), "volume remove-brick %s: "
+                                  "failed: %s", cmd_str, rsp.op_errstr);
+                else
+                        snprintf (msg, sizeof (msg), "volume remove-brick %s: "
+                                  "failed", cmd_str);
+
+                if (global_state->mode & GLUSTER_MODE_XML)
+                        goto xml_output;
+
+                cli_err ("%s", msg);
+                goto out;
+        }
+
         if (rsp.dict.dict_len) {
                 /* Unserialize the dictionary */
                 dict  = dict_new ();
@@ -1454,60 +1831,146 @@ gf_cli3_remove_brick_status_cbk (struct rpc_req *req, struct iovec *iov,
                                         rsp.dict.dict_len,
                                         &dict);
                 if (ret < 0) {
-                        gf_log ("glusterd", GF_LOG_ERROR,
-                                "failed to "
-                                "unserialize req-buffer to dictionary");
+                        strncpy (msg, "failed to unserialize req-buffer to "
+                                 "dictionary", sizeof (msg));
+
+                        if (global_state->mode & GLUSTER_MODE_XML) {
+                                rsp.op_ret = -1;
+                                goto xml_output;
+                        }
+
+                        gf_log ("cli", GF_LOG_ERROR, "%s", msg);
                         goto out;
                 }
         }
 
-        ret = dict_get_uint64 (dict, "files", &files);
-        if (ret)
-                gf_log (THIS->name, GF_LOG_TRACE,
-                        "failed to get file count");
-
-        ret = dict_get_uint64 (dict, "size", &size);
-        if (ret)
-                gf_log (THIS->name, GF_LOG_TRACE,
-                        "failed to get size of xfer");
-
-        if (files && (rsp.op_errno == 1)) {
-                snprintf (msg, sizeof (msg),
-                          "remove-brick %s: fixed layout %"PRId64,
-                          status,files);
-                goto out;
-        }
-        if (files && (rsp.op_errno == 6)) {
-                snprintf (msg, sizeof (msg),
-                          "remove-brick %s: fixed layout %"PRId64,
-                          status, files);
+xml_output:
+        if (global_state->mode & GLUSTER_MODE_XML) {
+                if (strcmp (rsp.op_errstr, "")) {
+                        ret = cli_xml_output_vol_remove_brick (_gf_true, dict,
+                                                               rsp.op_ret,
+                                                               rsp.op_errno,
+                                                               rsp.op_errstr);
+                } else {
+                        ret = cli_xml_output_vol_remove_brick (_gf_true, dict,
+                                                               rsp.op_ret,
+                                                               rsp.op_errno,
+                                                               msg);
+                }
                 goto out;
         }
-        if (files) {
-                snprintf (msg, sizeof (msg),
-                          "remove-brick %s: decommissioned %"PRId64
-                          " files of size %"PRId64, status,
-                          files, size);
+
+        ret = dict_get_int32 (dict, "count", &counter);
+        if (ret) {
+                gf_log (frame->this->name, GF_LOG_ERROR, "count not set");
                 goto out;
         }
 
-        snprintf (msg, sizeof (msg), "remove-brick %s", status);
 
-#if (HAVE_LIB_XML)
-        if (global_state->mode & GLUSTER_MODE_XML) {
-                ret = cli_xml_output_str ("volRemoveBrick", msg, rsp.op_ret,
-                                          rsp.op_errno, rsp.op_errstr);
+        cli_out ("%40s %16s %13s %13s %13s %13s %14s %s", "Node",
+                 "Rebalanced-files", "size", "scanned", "failures", "skipped",
+                 "status", "run-time in secs");
+        cli_out ("%40s %16s %13s %13s %13s %13s %14s %16s", "---------",
+                 "-----------", "-----------", "-----------", "-----------",
+                  "-----------","------------", "--------------");
+
+        do {
+                snprintf (key, 256, "node-name-%d", i);
+                ret = dict_get_str (dict, key, &node_name);
                 if (ret)
-                        gf_log ("cli", GF_LOG_ERROR,
-                                "Error outputting to xml");
-                goto out;
+                        gf_log (frame->this->name, GF_LOG_TRACE,
+                                "failed to get node-name");
+
+                memset (key, 0, 256);
+                snprintf (key, 256, "files-%d", i);
+                ret = dict_get_uint64 (dict, key, &files);
+                if (ret)
+                        gf_log (frame->this->name, GF_LOG_TRACE,
+                                "failed to get file count");
+
+                memset (key, 0, 256);
+                snprintf (key, 256, "size-%d", i);
+                ret = dict_get_uint64 (dict, key, &size);
+                if (ret)
+                        gf_log (frame->this->name, GF_LOG_TRACE,
+                                "failed to get size of xfer");
+
+                memset (key, 0, 256);
+                snprintf (key, 256, "lookups-%d", i);
+                ret = dict_get_uint64 (dict, key, &lookup);
+                if (ret)
+                        gf_log (frame->this->name, GF_LOG_TRACE,
+                                "failed to get lookedup file count");
+
+                memset (key, 0, 256);
+                snprintf (key, 256, "status-%d", i);
+                ret = dict_get_int32 (dict, key, (int32_t *)&status_rcd);
+                if (ret)
+                        gf_log (frame->this->name, GF_LOG_TRACE,
+                                "failed to get status");
+
+                snprintf (key, 256, "failures-%d", i);
+                ret = dict_get_uint64 (dict, key, &failures);
+                if (ret)
+                        gf_log (frame->this->name, GF_LOG_TRACE,
+                                "Failed to get failure on files");
+
+                snprintf (key, 256, "failures-%d", i);
+                ret = dict_get_uint64 (dict, key, &skipped);
+                if (ret)
+                        gf_log (frame->this->name, GF_LOG_TRACE,
+                                "Failed to get skipped files");
+                memset (key, 0, 256);
+                snprintf (key, 256, "run-time-%d", i);
+                ret = dict_get_double (dict, key, &elapsed);
+                if (ret)
+                        gf_log (frame->this->name, GF_LOG_TRACE,
+                                "Failed to get run-time");
+
+                switch (status_rcd) {
+                case GF_DEFRAG_STATUS_NOT_STARTED:
+                        status = "not started";
+                        break;
+                case GF_DEFRAG_STATUS_STARTED:
+                        status = "in progress";
+                        break;
+                case GF_DEFRAG_STATUS_STOPPED:
+                        status = "stopped";
+                        break;
+                case GF_DEFRAG_STATUS_COMPLETE:
+                        status = "completed";
+                        break;
+                case GF_DEFRAG_STATUS_FAILED:
+                        status = "failed";
+                        break;
+                default:
+                        break;
+                }
+
+                size_str = gf_uint64_2human_readable(size);
+                
+		if (strcmp (status, "not started")) {
+			cli_out ("%40s %16"PRIu64 " %13s" " %13"PRIu64 " %13"
+				PRIu64 " %13"PRIu64 " %14s %16.2f", node_name,
+                                files, size_str, lookup, failures, skipped,
+                                status, elapsed);
+		}
+                GF_FREE(size_str);
+
+                i++;
+        } while (i <= counter);
+
+        if ((cmd == GF_OP_CMD_STOP) && (rsp.op_ret == 0)) {
+                cli_out ("'remove-brick' process may be in the middle of a "
+                         "file migration.\nThe process will be fully stopped "
+                         "once the migration of the file is complete.\nPlease "
+                         "check remove-brick process for completion before "
+                         "doing any further brick related tasks on the "
+                         "volume.");
         }
-#endif
 
-        cli_out ("%s", msg);
 out:
-        if (rsp.dict.dict_val)
-                free (rsp.dict.dict_val); //malloced by xdr
+        free (rsp.dict.dict_val); //malloced by xdr
         if (dict)
                 dict_unref (dict);
         cli_cmd_broadcast_response (ret);
@@ -1516,57 +1979,117 @@ out:
 
 
 int
-gf_cli3_1_remove_brick_cbk (struct rpc_req *req, struct iovec *iov,
+gf_cli_remove_brick_cbk (struct rpc_req *req, struct iovec *iov,
                              int count, void *myframe)
 {
         gf_cli_rsp                      rsp   = {0,};
         int                             ret   = -1;
         char                            msg[1024] = {0,};
+        gf1_op_commands                 cmd = GF_OP_CMD_NONE;
+        char                           *cmd_str = "unknown";
+        cli_local_t                    *local = NULL;
+        call_frame_t                   *frame = NULL;
+        char                           *task_id_str = NULL;
+        dict_t                         *rsp_dict = NULL;
 
         if (-1 == req->rpc_status) {
                 goto out;
         }
 
+        frame = myframe;
+        local = frame->local;
+
         ret = xdr_to_generic (*iov, &rsp, (xdrproc_t)xdr_gf_cli_rsp);
         if (ret < 0) {
-                gf_log ("", GF_LOG_ERROR, "error");
+                gf_log (frame->this->name, GF_LOG_ERROR,
+                        "Failed to decode xdr response");
                 goto out;
         }
 
+        ret = dict_get_int32 (local->dict, "command", (int32_t *)&cmd);
+        if (ret) {
+                 gf_log ("", GF_LOG_ERROR, "failed to get command");
+                 goto out;
+        }
+
+        if (rsp.dict.dict_len) {
+                rsp_dict = dict_new ();
+                if (!rsp_dict) {
+                        ret = -1;
+                        goto out;
+                }
+
+                ret = dict_unserialize (rsp.dict.dict_val, rsp.dict.dict_len,
+                                        &rsp_dict);
+                if (ret) {
+                        gf_log ("cli", GF_LOG_ERROR,
+                                "Failed to unserialize rsp_dict");
+                        goto out;
+                }
+        }
+
+        switch (cmd) {
+        case GF_OP_CMD_START:
+                cmd_str = "start";
+
+                ret = dict_get_str (rsp_dict, GF_REMOVE_BRICK_TID_KEY, &task_id_str);
+                if (ret) {
+                        gf_log ("cli", GF_LOG_ERROR,
+                                "remove-brick-id is not present in dict");
+                }
+                break;
+        case GF_OP_CMD_COMMIT:
+                cmd_str = "commit";
+                break;
+        case GF_OP_CMD_COMMIT_FORCE:
+                cmd_str = "commit force";
+                break;
+        default:
+                cmd_str = "unknown";
+                break;
+        }
+
         gf_log ("cli", GF_LOG_INFO, "Received resp to remove brick");
 
         if (rsp.op_ret && strcmp (rsp.op_errstr, ""))
                 snprintf (msg, sizeof (msg), "%s", rsp.op_errstr);
         else
-                snprintf (msg, sizeof (msg), "Remove Brick %s",
+                snprintf (msg, sizeof (msg), "Remove Brick %s %s", cmd_str,
                           (rsp.op_ret) ? "unsuccessful": "successful");
 
-#if (HAVE_LIB_XML)
         if (global_state->mode & GLUSTER_MODE_XML) {
-                ret = cli_xml_output_str ("volRemoveBrick", msg, rsp.op_ret,
-                                          rsp.op_errno, rsp.op_errstr);
+                ret = cli_xml_output_vol_remove_brick (_gf_false, rsp_dict,
+                                                       rsp.op_ret, rsp.op_errno,
+                                                       msg);
                 if (ret)
                         gf_log ("cli", GF_LOG_ERROR,
                                 "Error outputting to xml");
                 goto out;
         }
-#endif
-        cli_out ("%s", msg);
+
+        if (rsp.op_ret) {
+                cli_err ("volume remove-brick %s: failed: %s", cmd_str,
+                         msg);
+        } else {
+                cli_out ("volume remove-brick %s: success", cmd_str);
+                if (GF_OP_CMD_START == cmd && task_id_str != NULL)
+                        cli_out ("ID: %s", task_id_str);
+        }
+
         ret = rsp.op_ret;
 
 out:
         cli_cmd_broadcast_response (ret);
-        if (rsp.dict.dict_val)
-                free (rsp.dict.dict_val);
-        if (rsp.op_errstr)
-                free (rsp.op_errstr);
+        free (rsp.dict.dict_val);
+        free (rsp.op_errstr);
+
         return ret;
 }
 
 
 
 int
-gf_cli3_1_replace_brick_cbk (struct rpc_req *req, struct iovec *iov,
+gf_cli_replace_brick_cbk (struct rpc_req *req, struct iovec *iov,
                              int count, void *myframe)
 {
         gf_cli_rsp                       rsp              = {0,};
@@ -1580,7 +2103,8 @@ gf_cli3_1_replace_brick_cbk (struct rpc_req *req, struct iovec *iov,
         gf1_cli_replace_op               replace_op       = 0;
         char                            *rb_operation_str = NULL;
         dict_t                          *rsp_dict         = NULL;
-        char                             msg[1024]         = {0,};
+        char                             msg[1024]        = {0,};
+        char                            *task_id_str      = NULL;
 
         if (-1 == req->rpc_status) {
                 goto out;
@@ -1590,7 +2114,8 @@ gf_cli3_1_replace_brick_cbk (struct rpc_req *req, struct iovec *iov,
 
         ret = xdr_to_generic (*iov, &rsp, (xdrproc_t)xdr_gf_cli_rsp);
         if (ret < 0) {
-                gf_log ("", GF_LOG_ERROR, "error");
+                gf_log (frame->this->name, GF_LOG_ERROR,
+                        "Failed to decode xdr response");
                 goto out;
         }
 
@@ -1605,58 +2130,77 @@ gf_cli3_1_replace_brick_cbk (struct rpc_req *req, struct iovec *iov,
                 goto out;
         }
 
+        if (rsp.dict.dict_len) {
+                /* Unserialize the dictionary */
+                rsp_dict  = dict_new ();
+
+                ret = dict_unserialize (rsp.dict.dict_val,
+                                rsp.dict.dict_len,
+                                &rsp_dict);
+                if (ret < 0) {
+                        gf_log ("glusterd", GF_LOG_ERROR,
+                                        "failed to "
+                                        "unserialize rsp buffer to dictionary");
+                        goto out;
+                }
+        }
+
         switch (replace_op) {
         case GF_REPLACE_OP_START:
-                if (rsp.op_ret)
-                        rb_operation_str = "replace-brick failed to start";
-                else
-                        rb_operation_str = "replace-brick started successfully";
+                if (rsp.op_ret) {
+                        rb_operation_str = gf_strdup ("replace-brick failed to"
+                                                      " start");
+                } else {
+                        ret = dict_get_str (rsp_dict, GF_REPLACE_BRICK_TID_KEY,
+                                            &task_id_str);
+                        if (ret) {
+                                gf_log ("cli", GF_LOG_ERROR, "Failed to get "
+                                        "\"replace-brick-id\" from dict");
+                                goto out;
+                        }
+                        ret = gf_asprintf (&rb_operation_str,
+                                           "replace-brick started successfully"
+                                           "\nID: %s", task_id_str);
+                        if (ret < 0)
+                                goto out;
+                }
                 break;
 
         case GF_REPLACE_OP_STATUS:
 
-                if (rsp.op_ret || ret)
-                        rb_operation_str = "replace-brick status unknown";
-                else {
-                        if (rsp.dict.dict_len) {
-                                /* Unserialize the dictionary */
-                                rsp_dict  = dict_new ();
-
-                                ret = dict_unserialize (rsp.dict.dict_val,
-                                                        rsp.dict.dict_len,
-                                                        &rsp_dict);
-                                if (ret < 0) {
-                                        gf_log ("glusterd", GF_LOG_ERROR,
-                                                "failed to "
-                                                "unserialize req-buffer to dictionary");
-                                        goto out;
-                                }
-                        }
+                if (rsp.op_ret || ret) {
+                        rb_operation_str = gf_strdup ("replace-brick status "
+                                                      "unknown");
+                } else {
                         ret = dict_get_str (rsp_dict, "status-reply",
                                             &status_reply);
                         if (ret) {
-                                gf_log (THIS->name, GF_LOG_ERROR, "failed to"
+                                gf_log (frame->this->name, GF_LOG_ERROR, "failed to"
                                         "get status");
                                 goto out;
                         }
 
-                        rb_operation_str = status_reply;
+                        rb_operation_str = gf_strdup (status_reply);
                 }
 
                 break;
 
         case GF_REPLACE_OP_PAUSE:
                 if (rsp.op_ret)
-                        rb_operation_str = "replace-brick pause failed";
+                        rb_operation_str = gf_strdup ("replace-brick pause "
+                                                      "failed");
                 else
-                        rb_operation_str = "replace-brick paused successfully";
+                        rb_operation_str = gf_strdup ("replace-brick paused "
+                                                      "successfully");
                 break;
 
         case GF_REPLACE_OP_ABORT:
                 if (rsp.op_ret)
-                        rb_operation_str = "replace-brick abort failed";
+                        rb_operation_str = gf_strdup ("replace-brick abort "
+                                                      "failed");
                 else
-                        rb_operation_str = "replace-brick aborted successfully";
+                        rb_operation_str = gf_strdup ("replace-brick aborted "
+                                                      "successfully");
                 break;
 
         case GF_REPLACE_OP_COMMIT:
@@ -1677,9 +2221,11 @@ gf_cli3_1_replace_brick_cbk (struct rpc_req *req, struct iovec *iov,
 
 
                 if (rsp.op_ret || ret)
-                        rb_operation_str = "replace-brick commit failed";
+                        rb_operation_str = gf_strdup ("replace-brick commit "
+                                                      "failed");
                 else
-                        rb_operation_str = "replace-brick commit successful";
+                        rb_operation_str = gf_strdup ("replace-brick commit "
+                                                      "successful");
 
                 break;
 
@@ -1690,36 +2236,43 @@ gf_cli3_1_replace_brick_cbk (struct rpc_req *req, struct iovec *iov,
         }
 
         if (rsp.op_ret && (strcmp (rsp.op_errstr, ""))) {
-                rb_operation_str = rsp.op_errstr;
+                rb_operation_str = gf_strdup (rsp.op_errstr);
         }
 
         gf_log ("cli", GF_LOG_INFO, "Received resp to replace brick");
-        snprintf (msg,sizeof (msg), "%s",
+        snprintf (msg, sizeof (msg), "%s",
                   rb_operation_str ? rb_operation_str : "Unknown operation");
 
-#if (HAVE_LIB_XML)
         if (global_state->mode & GLUSTER_MODE_XML) {
-                ret = cli_xml_output_str ("volReplaceBrick", msg, rsp.op_ret,
-                                          rsp.op_errno, rsp.op_errstr);
+                ret = cli_xml_output_vol_replace_brick (replace_op, rsp_dict,
+                                                        rsp.op_ret,
+                                                        rsp.op_errno, msg);
                 if (ret)
                         gf_log ("cli", GF_LOG_ERROR,
                                 "Error outputting to xml");
                 goto out;
         }
-#endif
 
-        cli_out ("%s", msg);
+        if (rsp.op_ret)
+                cli_err ("volume replace-brick: failed: %s", msg);
+        else
+                cli_out ("volume replace-brick: success: %s", msg);
         ret = rsp.op_ret;
 
 out:
+        if (frame)
+                frame->local = NULL;
+
         if (local) {
                 dict_unref (local->dict);
                 cli_local_wipe (local);
         }
 
+        if (rb_operation_str)
+                GF_FREE (rb_operation_str);
+
         cli_cmd_broadcast_response (ret);
-        if (rsp.dict.dict_val)
-               free (rsp.dict.dict_val);
+        free (rsp.dict.dict_val);
         if (rsp_dict)
                 dict_unref (rsp_dict);
 
@@ -1728,7 +2281,7 @@ out:
 
 
 static int
-gf_cli3_1_log_rotate_cbk (struct rpc_req *req, struct iovec *iov,
+gf_cli_log_rotate_cbk (struct rpc_req *req, struct iovec *iov,
                           int count, void *myframe)
 {
         gf_cli_rsp             rsp   = {0,};
@@ -1741,7 +2294,8 @@ gf_cli3_1_log_rotate_cbk (struct rpc_req *req, struct iovec *iov,
 
         ret = xdr_to_generic (*iov, &rsp, (xdrproc_t)xdr_gf_cli_rsp);
         if (ret < 0) {
-                gf_log ("", GF_LOG_ERROR, "error");
+                gf_log (((call_frame_t *) myframe)->this->name, GF_LOG_ERROR,
+                        "Failed to decode xdr response");
                 goto out;
         }
 
@@ -1753,7 +2307,6 @@ gf_cli3_1_log_rotate_cbk (struct rpc_req *req, struct iovec *iov,
                 snprintf (msg, sizeof (msg), "log rotate %s",
                           (rsp.op_ret) ? "unsuccessful": "successful");
 
-#if (HAVE_LIB_XML)
         if (global_state->mode & GLUSTER_MODE_XML) {
                 ret = cli_xml_output_str ("volLogRotate", msg, rsp.op_ret,
                                           rsp.op_errno, rsp.op_errstr);
@@ -1762,21 +2315,22 @@ gf_cli3_1_log_rotate_cbk (struct rpc_req *req, struct iovec *iov,
                                 "Error outputting to xml");
                 goto out;
         }
-#endif
 
-        cli_out ("%s", msg);
+        if (rsp.op_ret)
+                cli_err ("volume log-rotate: failed: %s", msg);
+        else
+                cli_out ("volume log-rotate: success");
         ret = rsp.op_ret;
 
 out:
         cli_cmd_broadcast_response (ret);
-        if (rsp.dict.dict_val)
-                free (rsp.dict.dict_val);
+        free (rsp.dict.dict_val);
 
         return ret;
 }
 
 static int
-gf_cli3_1_sync_volume_cbk (struct rpc_req *req, struct iovec *iov,
+gf_cli_sync_volume_cbk (struct rpc_req *req, struct iovec *iov,
                            int count, void *myframe)
 {
         gf_cli_rsp                     rsp   = {0,};
@@ -1789,19 +2343,20 @@ gf_cli3_1_sync_volume_cbk (struct rpc_req *req, struct iovec *iov,
 
         ret = xdr_to_generic (*iov, &rsp, (xdrproc_t)xdr_gf_cli_rsp);
         if (ret < 0) {
-                gf_log ("", GF_LOG_ERROR, "error");
+                gf_log (((call_frame_t *) myframe)->this->name, GF_LOG_ERROR,
+                        "Failed to decode xdr response");
                 goto out;
         }
 
         gf_log ("cli", GF_LOG_DEBUG, "Received resp to sync");
 
         if (rsp.op_ret && strcmp (rsp.op_errstr, ""))
-                snprintf (msg, sizeof (msg), "%s", rsp.op_errstr);
+                snprintf (msg, sizeof (msg), "volume sync: failed: %s",
+                          rsp.op_errstr);
         else
                 snprintf (msg, sizeof (msg), "volume sync: %s",
-                         (rsp.op_ret) ? "unsuccessful": "successful");
+                          (rsp.op_ret) ? "failed": "success");
 
-#if (HAVE_LIB_XML)
         if (global_state->mode & GLUSTER_MODE_XML) {
                 ret = cli_xml_output_str ("volSync", msg, rsp.op_ret,
                                           rsp.op_errno, rsp.op_errstr);
@@ -1810,9 +2365,11 @@ gf_cli3_1_sync_volume_cbk (struct rpc_req *req, struct iovec *iov,
                                 "Error outputting to xml");
                 goto out;
         }
-#endif
 
-        cli_out ("%s", msg);
+        if (rsp.op_ret)
+                cli_err ("%s", msg);
+        else
+                cli_out ("%s", msg);
         ret = rsp.op_ret;
 
 out:
@@ -1821,12 +2378,12 @@ out:
 }
 
 int32_t
-gf_cli3_1_print_limit_list (char *volname, char *limit_list,
+gf_cli_print_limit_list (char *volname, char *limit_list,
                             char *op_errstr)
 {
         int64_t  size            = 0;
         int64_t  limit_value     = 0;
-        int32_t  i, j, k;
+        int32_t  i, j;
         int32_t  len = 0, ret    = -1;
         char     *size_str       = NULL;
         char     path [PATH_MAX] = {0, };
@@ -1834,6 +2391,7 @@ gf_cli3_1_print_limit_list (char *volname, char *limit_list,
         char     value [1024]    = {0, };
         char     mountdir []     = "/tmp/mntXXXXXX";
         char     abspath [PATH_MAX] = {0, };
+        char     *colon_ptr      = NULL;
         runner_t runner          = {0,};
 
         GF_VALIDATE_OR_GOTO ("cli", volname, out);
@@ -1844,7 +2402,7 @@ gf_cli3_1_print_limit_list (char *volname, char *limit_list,
 
         len = strlen (limit_list);
         if (len == 0) {
-                cli_out ("%s", op_errstr?op_errstr:"quota limit not set ");
+                cli_err ("%s", op_errstr?op_errstr:"quota limit not set ");
                 goto out;
         }
 
@@ -1870,7 +2428,7 @@ gf_cli3_1_print_limit_list (char *volname, char *limit_list,
 
         len = strlen (limit_list);
         if (len == 0) {
-                cli_out ("quota limit not set ");
+                cli_err ("quota limit not set ");
                 goto unmount;
         }
 
@@ -1881,19 +2439,16 @@ gf_cli3_1_print_limit_list (char *volname, char *limit_list,
                  "-----------------------");
         while (i < len) {
                 j = 0;
-                k = 0;
-
-                while (limit_list [i] != ':') {
-                        path [k++] = limit_list [i++];
-                }
-                path [k] = '\0';
-
-                i++;  //skip ':'
 
                 while (limit_list [i] != ',' && limit_list [i] != '\0') {
-                        value [j++] = limit_list[i++];
+                        path [j++] = limit_list[i++];
                 }
-                value [j] = '\0';
+                path [j] = '\0';
+                //here path[] contains both path and limit value
+
+                colon_ptr = strrchr (path, ':');
+                *colon_ptr = '\0';
+                strcpy (value, ++colon_ptr);
 
                 snprintf (abspath, sizeof (abspath), "%s/%s", mountdir, path);
 
@@ -1936,7 +2491,7 @@ out:
 }
 
 int
-gf_cli3_1_quota_cbk (struct rpc_req *req, struct iovec *iov,
+gf_cli_quota_cbk (struct rpc_req *req, struct iovec *iov,
                      int count, void *myframe)
 {
         gf_cli_rsp         rsp        = {0,};
@@ -1945,15 +2500,19 @@ gf_cli3_1_quota_cbk (struct rpc_req *req, struct iovec *iov,
         char              *volname    = NULL;
         char              *limit_list = NULL;
         int32_t            type       = 0;
-        char               msg[1024] = {0,};
+        char               msg[1024]  = {0,};
+        call_frame_t      *frame      = NULL;
 
         if (-1 == req->rpc_status) {
                 goto out;
         }
 
+        frame = myframe;
+
         ret = xdr_to_generic (*iov, &rsp, (xdrproc_t)xdr_gf_cli_rsp);
         if (ret < 0) {
-                gf_log ("", GF_LOG_ERROR, "error");
+                gf_log (frame->this->name, GF_LOG_ERROR,
+                        "Failed to decode xdr response");
                 goto out;
         }
 
@@ -1961,10 +2520,9 @@ gf_cli3_1_quota_cbk (struct rpc_req *req, struct iovec *iov,
             strcmp (rsp.op_errstr, "") == 0) {
                 snprintf (msg, sizeof (msg), "command unsuccessful %s",
                           rsp.op_errstr);
-#if (HAVE_LIB_XML)
+
                 if (global_state->mode & GLUSTER_MODE_XML)
                         goto xml_output;
-#endif
                 goto out;
         }
 
@@ -1985,21 +2543,20 @@ gf_cli3_1_quota_cbk (struct rpc_req *req, struct iovec *iov,
 
         ret = dict_get_str (dict, "volname", &volname);
         if (ret)
-                gf_log (THIS->name, GF_LOG_TRACE,
+                gf_log (frame->this->name, GF_LOG_TRACE,
                         "failed to get volname");
 
         ret = dict_get_str (dict, "limit_list", &limit_list);
         if (ret)
-                gf_log (THIS->name, GF_LOG_TRACE,
+                gf_log (frame->this->name, GF_LOG_TRACE,
                         "failed to get limit_list");
 
         ret = dict_get_int32 (dict, "type", &type);
         if (ret)
-                gf_log (THIS->name, GF_LOG_TRACE,
+                gf_log (frame->this->name, GF_LOG_TRACE,
                         "failed to get type");
 
         if (type == GF_QUOTA_OPTION_TYPE_LIST) {
-#if (HAVE_LIB_XML)
                 if (global_state->mode & GLUSTER_MODE_XML) {
                         ret = cli_xml_output_vol_quota_limit_list
                                 (volname, limit_list, rsp.op_ret,
@@ -2010,9 +2567,9 @@ gf_cli3_1_quota_cbk (struct rpc_req *req, struct iovec *iov,
                         goto out;
 
                 }
-#endif
+
                 if (limit_list) {
-                        gf_cli3_1_print_limit_list (volname,
+                        gf_cli_print_limit_list (volname,
                                                     limit_list,
                                                     rsp.op_errstr);
                 } else {
@@ -2030,7 +2587,6 @@ gf_cli3_1_quota_cbk (struct rpc_req *req, struct iovec *iov,
                         snprintf (msg, sizeof (msg), "successful");
         }
 
-#if (HAVE_LIB_XML)
 xml_output:
         if (global_state->mode & GLUSTER_MODE_XML) {
                 ret = cli_xml_output_str ("volQuota", msg, rsp.op_ret,
@@ -2040,36 +2596,50 @@ xml_output:
                                 "Error outputting to xml");
                 goto out;
         }
-#endif
 
-        if (strlen (msg) > 0)
-                cli_out ("%s", msg);
+        if (strlen (msg) > 0) {
+                if (rsp.op_ret)
+                        cli_err ("%s", msg);
+                else
+                        cli_out ("%s", msg);
+        }
 
         ret = rsp.op_ret;
 out:
         cli_cmd_broadcast_response (ret);
+        if (dict)
+                dict_unref (dict);
 
-        if (rsp.dict.dict_val)
-                free (rsp.dict.dict_val);
+        free (rsp.dict.dict_val);
 
         return ret;
 }
 
 int
-gf_cli3_1_getspec_cbk (struct rpc_req *req, struct iovec *iov,
+gf_cli_getspec_cbk (struct rpc_req *req, struct iovec *iov,
                        int count, void *myframe)
 {
         gf_getspec_rsp          rsp   = {0,};
         int                     ret   = -1;
         char                   *spec  = NULL;
+        call_frame_t           *frame = NULL;
 
         if (-1 == req->rpc_status) {
                 goto out;
         }
 
+        frame = myframe;
+
         ret = xdr_to_generic (*iov, &rsp, (xdrproc_t)xdr_gf_getspec_rsp);
-        if (ret < 0 || rsp.op_ret == -1) {
-                gf_log ("", GF_LOG_ERROR, "error");
+        if (ret < 0) {
+                gf_log (frame->this->name, GF_LOG_ERROR,
+                        "Failed to decode xdr response");
+                goto out;
+        }
+
+        if (rsp.op_ret == -1) {
+                gf_log (frame->this->name, GF_LOG_ERROR,
+                        "getspec failed");
                 goto out;
         }
 
@@ -2093,20 +2663,30 @@ out:
 }
 
 int
-gf_cli3_1_pmap_b2p_cbk (struct rpc_req *req, struct iovec *iov,
+gf_cli_pmap_b2p_cbk (struct rpc_req *req, struct iovec *iov,
                         int count, void *myframe)
 {
         pmap_port_by_brick_rsp rsp = {0,};
         int                     ret   = -1;
         char                   *spec  = NULL;
+        call_frame_t           *frame = NULL;
 
         if (-1 == req->rpc_status) {
                 goto out;
         }
 
+        frame = myframe;
+
         ret = xdr_to_generic (*iov, &rsp, (xdrproc_t)xdr_pmap_port_by_brick_rsp);
-        if (ret < 0 || rsp.op_ret == -1) {
-                gf_log ("", GF_LOG_ERROR, "error");
+        if (ret < 0) {
+                gf_log (frame->this->name, GF_LOG_ERROR,
+                        "Failed to decode xdr response");
+                goto out;
+        }
+
+        if (rsp.op_ret == -1) {
+                gf_log (frame->this->name, GF_LOG_ERROR,
+                        "pump_b2p failed");
                 goto out;
         }
 
@@ -2124,13 +2704,12 @@ out:
 
 
 int32_t
-gf_cli3_1_probe (call_frame_t *frame, xlator_t *this,
+gf_cli_probe (call_frame_t *frame, xlator_t *this,
                  void *data)
 {
-        gf1_cli_probe_req  req      = {0,};
+        gf_cli_req         req      = {{0,},};
         int                ret      = 0;
         dict_t            *dict     = NULL;
-        char              *hostname = NULL;
         int                port     = 0;
 
         if (!frame || !this ||  !data) {
@@ -2139,35 +2718,32 @@ gf_cli3_1_probe (call_frame_t *frame, xlator_t *this,
         }
 
         dict = data;
-        ret = dict_get_str (dict, "hostname", &hostname);
-        if (ret)
-                goto out;
 
         ret = dict_get_int32 (dict, "port", &port);
-        if (ret)
-                port = CLI_GLUSTERD_PORT;
-
-        req.hostname = hostname;
-        req.port     = port;
+        if (ret) {
+                ret = dict_set_int32 (dict, "port", CLI_GLUSTERD_PORT);
+                if (ret)
+                        goto out;
+        }
 
-        ret = cli_cmd_submit (&req, frame, cli_rpc_prog,
-                              GLUSTER_CLI_PROBE, NULL,
-                              this, gf_cli3_1_probe_cbk,
-                              (xdrproc_t)xdr_gf1_cli_probe_req);
+        ret = cli_to_glusterd (&req, frame, gf_cli_probe_cbk,
+                               (xdrproc_t) xdr_gf_cli_req, dict,
+                               GLUSTER_CLI_PROBE, this, cli_rpc_prog, NULL);
 
 out:
+        GF_FREE (req.dict.dict_val);
         gf_log ("cli", GF_LOG_DEBUG, "Returning %d", ret);
+
         return ret;
 }
 
 int32_t
-gf_cli3_1_deprobe (call_frame_t *frame, xlator_t *this,
+gf_cli_deprobe (call_frame_t *frame, xlator_t *this,
                    void *data)
 {
-        gf1_cli_deprobe_req  req      = {0,};
+        gf_cli_req           req      = {{0,},};
         int                  ret      = 0;
         dict_t              *dict     = NULL;
-        char                *hostname = NULL;
         int                  port     = 0;
         int                  flags    = 0;
 
@@ -2177,57 +2753,70 @@ gf_cli3_1_deprobe (call_frame_t *frame, xlator_t *this,
         }
 
         dict = data;
-        ret = dict_get_str (dict, "hostname", &hostname);
-        if (ret)
-                goto out;
-
         ret = dict_get_int32 (dict, "port", &port);
-        if (ret)
-                port = CLI_GLUSTERD_PORT;
+        if (ret) {
+                ret = dict_set_int32 (dict, "port", CLI_GLUSTERD_PORT);
+                if (ret)
+                        goto out;
+        }
 
         ret = dict_get_int32 (dict, "flags", &flags);
-        if (ret)
-                flags = 0;
+        if (ret) {
+                ret = dict_set_int32 (dict, "flags", 0);
+                if (ret)
+                        goto out;
+        }
 
-        req.hostname = hostname;
-        req.port     = port;
-        req.flags    = flags;
-        ret = cli_cmd_submit (&req, frame, cli_rpc_prog,
-                              GLUSTER_CLI_DEPROBE, NULL,
-                              this, gf_cli3_1_deprobe_cbk,
-                              (xdrproc_t)xdr_gf1_cli_deprobe_req);
+        ret = cli_to_glusterd (&req, frame, gf_cli_deprobe_cbk,
+                               (xdrproc_t)xdr_gf_cli_req, dict,
+                              GLUSTER_CLI_DEPROBE, this, cli_rpc_prog, NULL);
 
 out:
+        GF_FREE (req.dict.dict_val);
         gf_log ("cli", GF_LOG_DEBUG, "Returning %d", ret);
+
         return ret;
 }
 
 int32_t
-gf_cli3_1_list_friends (call_frame_t *frame, xlator_t *this,
-                        void *data)
+gf_cli_list_friends (call_frame_t *frame, xlator_t *this,
+                     void *data)
 {
         gf1_cli_peer_list_req   req = {0,};
         int                     ret = 0;
+        unsigned long           flags = 0;
 
         if (!frame || !this) {
                 ret = -1;
                 goto out;
         }
 
-        req.flags = GF_CLI_LIST_ALL;
+        GF_ASSERT (frame->local == NULL);
 
+        flags = (long)data;
+        req.flags = flags;
+        frame->local = (void*)flags;
         ret = cli_cmd_submit (&req, frame, cli_rpc_prog,
                               GLUSTER_CLI_LIST_FRIENDS, NULL,
-                              this, gf_cli3_1_list_friends_cbk,
+                              this, gf_cli_list_friends_cbk,
                               (xdrproc_t) xdr_gf1_cli_peer_list_req);
 
 out:
+        if (ret) {
+                /*
+                 * If everything goes fine, gf_cli_list_friends_cbk()
+                 * [invoked through cli_cmd_submit()]resets the
+                 * frame->local to NULL. In case cli_cmd_submit()
+                 * fails in between, RESET frame->local here.
+                 */
+                frame->local = NULL;
+        }
         gf_log ("cli", GF_LOG_DEBUG, "Returning %d", ret);
         return ret;
 }
 
 int32_t
-gf_cli3_1_get_next_volume (call_frame_t *frame, xlator_t *this,
+gf_cli_get_next_volume (call_frame_t *frame, xlator_t *this,
                            void *data)
 {
 
@@ -2243,7 +2832,6 @@ gf_cli3_1_get_next_volume (call_frame_t *frame, xlator_t *this,
         ctx = data;
         local = frame->local;
 
-#if (HAVE_LIB_XML)
         if (global_state->mode & GLUSTER_MODE_XML) {
                 ret = cli_xml_output_vol_info_begin (local, 0, 0, "");
                 if (ret) {
@@ -2251,17 +2839,15 @@ gf_cli3_1_get_next_volume (call_frame_t *frame, xlator_t *this,
                         goto out;
                 }
         }
-#endif
 
-        ret = gf_cli3_1_get_volume (frame, this, data);
+        ret = gf_cli_get_volume (frame, this, data);
 
 
         if (!local || !local->get_vol.volname) {
-#if (HAVE_LIB_XML)
                 if ((global_state->mode & GLUSTER_MODE_XML))
                         goto end_xml;
-#endif
-                cli_out ("No volumes present");
+
+                cli_err ("No volumes present");
                 goto out;
         }
 
@@ -2269,20 +2855,18 @@ gf_cli3_1_get_next_volume (call_frame_t *frame, xlator_t *this,
         ctx->volname = local->get_vol.volname;
 
         while (ctx->volname) {
-                ret = gf_cli3_1_get_volume (frame, this, ctx);
+                ret = gf_cli_get_volume (frame, this, ctx);
                 if (ret)
                         goto out;
                 ctx->volname = local->get_vol.volname;
         }
 
-#if (HAVE_LIB_XML)
 end_xml:
         if (global_state->mode & GLUSTER_MODE_XML) {
                 ret = cli_xml_output_vol_info_end (local);
                 if (ret)
                         gf_log ("cli", GF_LOG_ERROR, "Error outputting to xml");
         }
-#endif
 
 out:
         gf_log ("cli", GF_LOG_DEBUG, "Returning %d", ret);
@@ -2290,7 +2874,7 @@ out:
 }
 
 int32_t
-gf_cli3_1_get_volume (call_frame_t *frame, xlator_t *this,
+gf_cli_get_volume (call_frame_t *frame, xlator_t *this,
                       void *data)
 {
         gf_cli_req                      req = {{0,}};
@@ -2319,90 +2903,108 @@ gf_cli3_1_get_volume (call_frame_t *frame, xlator_t *this,
         flags = ctx->flags;
         ret = dict_set_int32 (dict, "flags", flags);
         if (ret) {
-                gf_log (THIS->name, GF_LOG_ERROR, "failed to set flags");
+                gf_log (frame->this->name, GF_LOG_ERROR, "failed to set flags");
                 goto out;
         }
 
-        ret = dict_allocate_and_serialize (dict,
-                                           &req.dict.dict_val,
-                                           (size_t *)&req.dict.dict_len);
+        ret = dict_allocate_and_serialize (dict, &req.dict.dict_val,
+                                           &req.dict.dict_len);
 
         ret = cli_cmd_submit (&req, frame, cli_rpc_prog,
                               GLUSTER_CLI_GET_VOLUME, NULL,
-                              this, gf_cli3_1_get_volume_cbk,
+                              this, gf_cli_get_volume_cbk,
                               (xdrproc_t) xdr_gf_cli_req);
 
 out:
         if (dict)
                 dict_unref (dict);
 
-        if (req.dict.dict_val)
-                GF_FREE (req.dict.dict_val);
+        GF_FREE (req.dict.dict_val);
 
         gf_log ("cli", GF_LOG_DEBUG, "Returning %d", ret);
         return ret;
 }
 
-
 int32_t
-gf_cli3_1_create_volume (call_frame_t *frame, xlator_t *this,
-                         void *data)
+gf_cli3_1_uuid_get (call_frame_t *frame, xlator_t *this,
+                      void *data)
 {
-        gf_cli_req              req = {{0,}};
-        int                     ret = 0;
-        dict_t                  *dict = NULL;
-        cli_local_t             *local = NULL;
+        gf_cli_req                      req = {{0,}};
+        int                             ret = 0;
+        dict_t                          *dict = NULL;
 
-        if (!frame || !this ||  !data) {
+        if (!frame || !this || !data) {
                 ret = -1;
                 goto out;
         }
 
-        dict = dict_ref ((dict_t *)data);
+        dict = data;
+        ret = cli_to_glusterd (&req, frame, gf_cli3_1_uuid_get_cbk,
+                               (xdrproc_t)xdr_gf_cli_req, dict,
+                               GLUSTER_CLI_UUID_GET, this, cli_rpc_prog,
+                               NULL);
+out:
+        gf_log ("cli", GF_LOG_DEBUG, "Returning %d", ret);
+        return ret;
+}
 
-        ret = dict_allocate_and_serialize (dict,
-                                           &req.dict.dict_val,
-                                           (size_t *)&req.dict.dict_len);
-        if (ret < 0) {
-                gf_log (this->name, GF_LOG_DEBUG,
-                        "failed to get serialized length of dict");
+int32_t
+gf_cli3_1_uuid_reset (call_frame_t *frame, xlator_t *this,
+                      void *data)
+{
+        gf_cli_req                      req = {{0,}};
+        int                             ret = 0;
+        dict_t                          *dict = NULL;
+
+        if (!frame || !this || !data) {
+                ret = -1;
                 goto out;
         }
 
-        local = cli_local_get ();
+        dict = data;
+        ret = cli_to_glusterd (&req, frame, gf_cli3_1_uuid_reset_cbk,
+                               (xdrproc_t)xdr_gf_cli_req, dict,
+                               GLUSTER_CLI_UUID_RESET, this, cli_rpc_prog,
+                               NULL);
+out:
+        gf_log ("cli", GF_LOG_DEBUG, "Returning %d", ret);
+        return ret;
+}
 
-        if (local) {
-                local->dict = dict_ref (dict);
-                frame->local = local;
-        }
+int32_t
+gf_cli_create_volume (call_frame_t *frame, xlator_t *this,
+                         void *data)
+{
+        gf_cli_req              req = {{0,}};
+        int                     ret = 0;
+        dict_t                  *dict = NULL;
 
-        ret = cli_cmd_submit (&req, frame, cli_rpc_prog,
-                              GLUSTER_CLI_CREATE_VOLUME, NULL,
-                              this, gf_cli3_1_create_volume_cbk,
-                              (xdrproc_t) xdr_gf_cli_req);
+        if (!frame || !this ||  !data) {
+                ret = -1;
+                goto out;
+        }
 
+        dict = data;
 
+        ret = cli_to_glusterd (&req, frame, gf_cli_create_volume_cbk,
+                               (xdrproc_t) xdr_gf_cli_req, dict,
+                               GLUSTER_CLI_CREATE_VOLUME, this, cli_rpc_prog,
+                               NULL);
 
 out:
         gf_log ("cli", GF_LOG_DEBUG, "Returning %d", ret);
 
-        if (dict)
-                dict_unref (dict);
-
-        if (req.dict.dict_val) {
-                GF_FREE (req.dict.dict_val);
-        }
+        GF_FREE (req.dict.dict_val);
 
         return ret;
 }
 
 int32_t
-gf_cli3_1_delete_volume (call_frame_t *frame, xlator_t *this,
+gf_cli_delete_volume (call_frame_t *frame, xlator_t *this,
                          void *data)
 {
         gf_cli_req              req = {{0,}};
         int                     ret = 0;
-        cli_local_t             *local = NULL;
         dict_t                  *dict = NULL;
 
         if (!frame || !this ||  !data) {
@@ -2410,50 +3012,26 @@ gf_cli3_1_delete_volume (call_frame_t *frame, xlator_t *this,
                 goto out;
         }
 
-        local = cli_local_get ();
-
-        dict = dict_new ();
-        ret = dict_set_str (dict, "volname", data);
-        if (ret) {
-                gf_log (THIS->name, GF_LOG_WARNING, "dict set failed");
-                goto out;
-        }
-        if (local) {
-                local->dict = dict_ref (dict);
-                frame->local = local;
-        }
-
-        ret = dict_allocate_and_serialize (dict,
-                                           &req.dict.dict_val,
-                                           (size_t *)&req.dict.dict_len);
-        if (ret < 0) {
-                gf_log (this->name, GF_LOG_ERROR,
-                        "failed to get serialize dict");
-                goto out;
-        }
+        dict = data;
 
-        ret = cli_cmd_submit (&req, frame, cli_rpc_prog,
-                              GLUSTER_CLI_DELETE_VOLUME, NULL,
-                              this, gf_cli3_1_delete_volume_cbk,
-                              (xdrproc_t)xdr_gf_cli_req);
+        ret = cli_to_glusterd (&req, frame, gf_cli_delete_volume_cbk,
+                               (xdrproc_t) xdr_gf_cli_req, dict,
+                               GLUSTER_CLI_DELETE_VOLUME, this, cli_rpc_prog,
+                               NULL);
 
 out:
-        if (dict)
-                dict_unref (dict);
-        if (req.dict.dict_val)
-                GF_FREE (req.dict.dict_val);
+        GF_FREE (req.dict.dict_val);
         gf_log ("cli", GF_LOG_DEBUG, "Returning %d", ret);
 
         return ret;
 }
 
 int32_t
-gf_cli3_1_start_volume (call_frame_t *frame, xlator_t *this,
+gf_cli_start_volume (call_frame_t *frame, xlator_t *this,
                          void *data)
 {
         gf_cli_req              req = {{0,}};
         int                     ret = 0;
-        cli_local_t             *local = NULL;
         dict_t                  *dict = NULL;
 
         if (!frame || !this ||  !data) {
@@ -2462,27 +3040,11 @@ gf_cli3_1_start_volume (call_frame_t *frame, xlator_t *this,
         }
 
         dict = data;
-        local = cli_local_get ();
 
-        ret = dict_allocate_and_serialize (dict,
-                                           &req.dict.dict_val,
-                                           (size_t *)&req.dict.dict_len);
-        if (ret < 0) {
-                gf_log (this->name, GF_LOG_ERROR,
-                        "failed to serialize dict");
-                goto out;
-        }
-
-
-        if (local) {
-                local->dict = dict_ref (dict);
-                frame->local = local;
-        }
-
-        ret = cli_cmd_submit (&req, frame, cli_rpc_prog,
-                              GLUSTER_CLI_START_VOLUME, NULL,
-                              this, gf_cli3_1_start_volume_cbk,
-                              (xdrproc_t) xdr_gf_cli_req);
+        ret = cli_to_glusterd (&req, frame, gf_cli_start_volume_cbk,
+                               (xdrproc_t) xdr_gf_cli_req, dict,
+                               GLUSTER_CLI_START_VOLUME, this, cli_rpc_prog,
+                               NULL);
 
 out:
         gf_log ("cli", GF_LOG_DEBUG, "Returning %d", ret);
@@ -2491,12 +3053,11 @@ out:
 }
 
 int32_t
-gf_cli3_1_stop_volume (call_frame_t *frame, xlator_t *this,
+gf_cli_stop_volume (call_frame_t *frame, xlator_t *this,
                          void *data)
 {
         gf_cli_req             req = {{0,}};
         int                    ret = 0;
-        cli_local_t            *local = NULL;
         dict_t                 *dict = data;
 
         if (!frame || !this ||  !data) {
@@ -2504,28 +3065,12 @@ gf_cli3_1_stop_volume (call_frame_t *frame, xlator_t *this,
                 goto out;
         }
 
-        local = cli_local_get ();
         dict = data;
 
-        if (local) {
-                local->dict = dict_ref (dict);
-                frame->local = local;
-        }
-
-        ret = dict_allocate_and_serialize (dict,
-                                           &req.dict.dict_val,
-                                           (size_t *) &req.dict.dict_len);
-        if (ret < 0) {
-                gf_log (this->name, GF_LOG_ERROR,
-                        "failed to serialize the data");
-
-                goto out;
-        }
-
-        ret = cli_cmd_submit (&req, frame, cli_rpc_prog,
-                              GLUSTER_CLI_STOP_VOLUME, NULL,
-                              this, gf_cli3_1_stop_volume_cbk,
-                              (xdrproc_t) xdr_gf_cli_req);
+        ret = cli_to_glusterd (&req, frame, gf_cli_stop_volume_cbk,
+                               (xdrproc_t) xdr_gf_cli_req, dict,
+                               GLUSTER_CLI_STOP_VOLUME, this, cli_rpc_prog,
+                               NULL);
 
 out:
         gf_log ("cli", GF_LOG_DEBUG, "Returning %d", ret);
@@ -2534,17 +3079,12 @@ out:
 }
 
 int32_t
-gf_cli3_1_defrag_volume (call_frame_t *frame, xlator_t *this,
+gf_cli_defrag_volume (call_frame_t *frame, xlator_t *this,
                          void *data)
 {
         gf_cli_req              req     =  {{0,}};
         int                     ret     = 0;
-        cli_local_t            *local   = NULL;
-        char                   *volname = NULL;
-        char                   *cmd_str = NULL;
         dict_t                 *dict    = NULL;
-        gf_cli_defrag_type      cmd     = 0;
-        dict_t                  *req_dict = NULL;
 
         if (!frame || !this ||  !data) {
                 ret = -1;
@@ -2553,81 +3093,10 @@ gf_cli3_1_defrag_volume (call_frame_t *frame, xlator_t *this,
 
         dict = data;
 
-        ret = dict_get_str (dict, "volname", &volname);
-        if (ret)
-                gf_log ("", GF_LOG_DEBUG, "error");
-
-        ret = dict_get_str (dict, "command", &cmd_str);
-        if (ret) {
-                gf_log ("", GF_LOG_DEBUG, "error");
-                goto out;
-        }
-
-        if (strcmp (cmd_str, "start") == 0) {
-                cmd = GF_DEFRAG_CMD_START;
-                ret = dict_get_str (dict, "option", &cmd_str);
-                if (!ret) {
-                        if (strcmp (cmd_str, "force") == 0) {
-                                cmd = GF_DEFRAG_CMD_START_FORCE;
-                        }
-                }
-                goto done;
-        }
-
-        if (strcmp (cmd_str, "fix-layout") == 0) {
-                cmd = GF_DEFRAG_CMD_START_LAYOUT_FIX;
-                goto done;
-        }
-        if (strcmp (cmd_str, "stop") == 0) {
-                cmd = GF_DEFRAG_CMD_STOP;
-                goto done;
-        }
-        if (strcmp (cmd_str, "status") == 0) {
-                cmd = GF_DEFRAG_CMD_STATUS;
-        }
-
-done:
-        local = cli_local_get ();
-
-        req_dict = dict_new ();
-        if (!req_dict) {
-                ret = -1;
-                goto out;
-        }
-
-        ret = dict_set_str (req_dict, "volname", volname);
-        if (ret) {
-                gf_log (THIS->name, GF_LOG_ERROR,
-                        "Failed to set dict");
-                goto out;
-        }
-
-        ret = dict_set_int32 (req_dict, "rebalance-command", (int32_t) cmd);
-        if (ret) {
-                gf_log (THIS->name, GF_LOG_ERROR,
-                        "Failed to set dict");
-                goto out;
-        }
-
-        if (local) {
-                local->dict = dict_ref (req_dict);
-                frame->local = local;
-        }
-
-        ret = dict_allocate_and_serialize (req_dict,
-                                           &req.dict.dict_val,
-                                           (size_t *) &req.dict.dict_len);
-        if (ret < 0) {
-                gf_log (this->name, GF_LOG_ERROR,
-                        "failed to serialize the data");
-
-                goto out;
-        }
-
-        ret = cli_cmd_submit (&req, frame, cli_rpc_prog,
-                              GLUSTER_CLI_DEFRAG_VOLUME, NULL,
-                              this, gf_cli3_1_defrag_volume_cbk,
-                              (xdrproc_t) xdr_gf_cli_req);
+        ret = cli_to_glusterd (&req, frame, gf_cli_defrag_volume_cbk,
+                               (xdrproc_t) xdr_gf_cli_req, dict,
+                               GLUSTER_CLI_DEFRAG_VOLUME, this, cli_rpc_prog,
+                               NULL);
 
 out:
         gf_log ("cli", GF_LOG_DEBUG, "Returning %d", ret);
@@ -2636,7 +3105,7 @@ out:
 }
 
 int32_t
-gf_cli3_1_rename_volume (call_frame_t *frame, xlator_t *this,
+gf_cli_rename_volume (call_frame_t *frame, xlator_t *this,
                          void *data)
 {
         gf_cli_req              req = {{0,}};
@@ -2650,9 +3119,8 @@ gf_cli3_1_rename_volume (call_frame_t *frame, xlator_t *this,
 
         dict = data;
 
-        ret = dict_allocate_and_serialize (dict,
-                                           &req.dict.dict_val,
-                                           (size_t *) &req.dict.dict_len);
+        ret = dict_allocate_and_serialize (dict, &req.dict.dict_val,
+                                           &req.dict.dict_len);
         if (ret < 0) {
                 gf_log (this->name, GF_LOG_ERROR,
                         "failed to serialize the data");
@@ -2663,7 +3131,7 @@ gf_cli3_1_rename_volume (call_frame_t *frame, xlator_t *this,
 
         ret = cli_cmd_submit (&req, frame, cli_rpc_prog,
                               GLUSTER_CLI_RENAME_VOLUME, NULL,
-                              this, gf_cli3_1_rename_volume_cbk,
+                              this, gf_cli_rename_volume_cbk,
                               (xdrproc_t) xdr_gf_cli_req);
 
 out:
@@ -2673,7 +3141,7 @@ out:
 }
 
 int32_t
-gf_cli3_1_reset_volume (call_frame_t *frame, xlator_t *this,
+gf_cli_reset_volume (call_frame_t *frame, xlator_t *this,
                         void *data)
 {
         gf_cli_req              req =  {{0,}};
@@ -2687,28 +3155,18 @@ gf_cli3_1_reset_volume (call_frame_t *frame, xlator_t *this,
 
         dict = data;
 
-        ret = dict_allocate_and_serialize (dict,
-                                           &req.dict.dict_val,
-                                           (size_t *)&req.dict.dict_len);
-        if (ret < 0) {
-                gf_log (this->name, GF_LOG_ERROR,
-                        "failed to get serialized length of dict");
-                goto out;
-        }
-
-        ret = cli_cmd_submit (&req, frame, cli_rpc_prog,
-                               GLUSTER_CLI_RESET_VOLUME, NULL,
-                               this, gf_cli3_1_reset_volume_cbk,
-                               (xdrproc_t) xdr_gf_cli_req);
+        ret = cli_to_glusterd (&req, frame, gf_cli_reset_volume_cbk,
+                               (xdrproc_t) xdr_gf_cli_req, dict,
+                               GLUSTER_CLI_RESET_VOLUME, this, cli_rpc_prog,
+                               NULL);
 
 out:
-                gf_log ("cli", GF_LOG_DEBUG, "Returning %d", ret);
-
+        gf_log ("cli", GF_LOG_DEBUG, "Returning %d", ret);
         return ret;
 }
 
 int32_t
-gf_cli3_1_set_volume (call_frame_t *frame, xlator_t *this,
+gf_cli_set_volume (call_frame_t *frame, xlator_t *this,
                          void *data)
 {
         gf_cli_req              req =  {{0,}};
@@ -2722,19 +3180,10 @@ gf_cli3_1_set_volume (call_frame_t *frame, xlator_t *this,
 
         dict = data;
 
-        ret = dict_allocate_and_serialize (dict,
-                                           &req.dict.dict_val,
-                                           (size_t *)&req.dict.dict_len);
-        if (ret < 0) {
-                gf_log (this->name, GF_LOG_DEBUG,
-                        "failed to get serialized length of dict");
-                goto out;
-        }
-
-        ret = cli_cmd_submit (&req, frame, cli_rpc_prog,
-                              GLUSTER_CLI_SET_VOLUME, NULL,
-                              this, gf_cli3_1_set_volume_cbk,
-                              (xdrproc_t) xdr_gf_cli_req);
+        ret = cli_to_glusterd (&req, frame, gf_cli_set_volume_cbk,
+                               (xdrproc_t) xdr_gf_cli_req, dict,
+                               GLUSTER_CLI_SET_VOLUME, this, cli_rpc_prog,
+                               NULL);
 
 out:
         gf_log ("cli", GF_LOG_DEBUG, "Returning %d", ret);
@@ -2743,7 +3192,7 @@ out:
 }
 
 int32_t
-gf_cli3_1_add_brick (call_frame_t *frame, xlator_t *this,
+gf_cli_add_brick (call_frame_t *frame, xlator_t *this,
                          void *data)
 {
         gf_cli_req              req =  {{0,}};
@@ -2768,33 +3217,20 @@ gf_cli3_1_add_brick (call_frame_t *frame, xlator_t *this,
         if (ret)
                 goto out;
 
-
-        ret = dict_allocate_and_serialize (dict,
-                                           &req.dict.dict_val,
-                                           (size_t *)&req.dict.dict_len);
-        if (ret < 0) {
-                gf_log (this->name, GF_LOG_DEBUG,
-                        "failed to get serialized length of dict");
-                goto out;
-        }
-
-        ret = cli_cmd_submit (&req, frame, cli_rpc_prog,
-                              GLUSTER_CLI_ADD_BRICK, NULL,
-                              this, gf_cli3_1_add_brick_cbk,
-                              (xdrproc_t) xdr_gf_cli_req);
+        ret = cli_to_glusterd (&req, frame, gf_cli_add_brick_cbk,
+                               (xdrproc_t) xdr_gf_cli_req, dict,
+                               GLUSTER_CLI_ADD_BRICK, this, cli_rpc_prog, NULL);
 
 out:
         gf_log ("cli", GF_LOG_DEBUG, "Returning %d", ret);
 
-        if (req.dict.dict_val) {
-                GF_FREE (req.dict.dict_val);
-        }
+        GF_FREE (req.dict.dict_val);
 
         return ret;
 }
 
 int32_t
-gf_cli3_1_remove_brick (call_frame_t *frame, xlator_t *this,
+gf_cli_remove_brick (call_frame_t *frame, xlator_t *this,
                          void *data)
 {
         gf_cli_req                req =  {{0,}};;
@@ -2821,23 +3257,16 @@ gf_cli3_1_remove_brick (call_frame_t *frame, xlator_t *this,
         if (ret)
                 goto out;
 
-        if (command != GF_OP_CMD_STATUS) {
+        if ((command != GF_OP_CMD_STATUS) &&
+            (command != GF_OP_CMD_STOP)) {
 
-                ret = dict_allocate_and_serialize (dict,
-                                                   &req.dict.dict_val,
-                                                   (size_t *)&req.dict.dict_len);
-                if (ret < 0) {
-                        gf_log (this->name, GF_LOG_DEBUG,
-                                "failed to get serialized length of dict");
-                        goto out;
-                }
 
-                ret = cli_cmd_submit (&req, frame, cli_rpc_prog,
-                                      GLUSTER_CLI_REMOVE_BRICK, NULL,
-                                      this, gf_cli3_1_remove_brick_cbk,
-                                      (xdrproc_t) xdr_gf_cli_req);
+                ret = cli_to_glusterd (&req, frame, gf_cli_remove_brick_cbk,
+                                       (xdrproc_t) xdr_gf_cli_req, dict,
+                                       GLUSTER_CLI_REMOVE_BRICK, this,
+                                       cli_rpc_prog, NULL);
         } else {
-                /* Need rebalance status to e sent :-) */
+                /* Need rebalance status to be sent :-) */
                 req_dict = dict_new ();
                 if (!req_dict) {
                         ret = -1;
@@ -2846,60 +3275,49 @@ gf_cli3_1_remove_brick (call_frame_t *frame, xlator_t *this,
 
                 ret = dict_set_str (req_dict, "volname", volname);
                 if (ret) {
-                        gf_log (THIS->name, GF_LOG_ERROR,
+                        gf_log (this->name, GF_LOG_ERROR,
                                 "Failed to set dict");
                         goto out;
                 }
 
-                cmd |= GF_DEFRAG_CMD_STATUS;
+                if (command == GF_OP_CMD_STATUS)
+                        cmd |= GF_DEFRAG_CMD_STATUS;
+                else
+                        cmd |= GF_DEFRAG_CMD_STOP;
 
                 ret = dict_set_int32 (req_dict, "rebalance-command", (int32_t) cmd);
                 if (ret) {
-                        gf_log (THIS->name, GF_LOG_ERROR,
+                        gf_log (this->name, GF_LOG_ERROR,
                                 "Failed to set dict");
                         goto out;
                 }
 
-                ret = dict_allocate_and_serialize (req_dict,
-                                                   &status_req.dict.dict_val,
-                                                   (size_t *) &status_req.dict.dict_len);
-                if (ret < 0) {
-                        gf_log (this->name, GF_LOG_ERROR,
-                                "failed to serialize the data");
+                ret = cli_to_glusterd (&status_req, frame,
+                                       gf_cli3_remove_brick_status_cbk,
+                                       (xdrproc_t) xdr_gf_cli_req, req_dict,
+                                       GLUSTER_CLI_DEFRAG_VOLUME, this,
+                                       cli_rpc_prog, NULL);
 
-                        goto out;
                 }
 
-                ret = cli_cmd_submit (&status_req, frame, cli_rpc_prog,
-                                      GLUSTER_CLI_DEFRAG_VOLUME, NULL,
-                                      this, gf_cli3_remove_brick_status_cbk,
-                                      (xdrproc_t) xdr_gf_cli_req);
-
-        }
-
 out:
+        if (req_dict)
+                dict_unref (req_dict);
         gf_log ("cli", GF_LOG_DEBUG, "Returning %d", ret);
 
-        if (req.dict.dict_val) {
-                GF_FREE (req.dict.dict_val);
-        }
-
-        if (status_req.dict.dict_val)
-                GF_FREE (status_req.dict.dict_val);
+        GF_FREE (req.dict.dict_val);
 
-        if (req_dict)
-                dict_unref (req_dict);
+        GF_FREE (status_req.dict.dict_val);
 
         return ret;
 }
 
 int32_t
-gf_cli3_1_replace_brick (call_frame_t *frame, xlator_t *this,
+gf_cli_replace_brick (call_frame_t *frame, xlator_t *this,
                          void *data)
 {
         gf_cli_req                  req        =  {{0,}};
         int                         ret        = 0;
-        cli_local_t                *local      = NULL;
         dict_t                     *dict       = NULL;
         char                       *src_brick  = NULL;
         char                       *dst_brick  = NULL;
@@ -2913,17 +3331,6 @@ gf_cli3_1_replace_brick (call_frame_t *frame, xlator_t *this,
 
         dict = data;
 
-        local = cli_local_get ();
-        if (!local) {
-                ret = -1;
-                gf_log (this->name, GF_LOG_ERROR,
-                        "Out of memory");
-                goto out;
-        }
-
-        local->dict                 = dict_ref (dict);
-        frame->local                = local;
-
         ret = dict_get_int32 (dict, "operation", &op);
         if (ret) {
                 gf_log (this->name, GF_LOG_DEBUG,
@@ -2956,34 +3363,22 @@ gf_cli3_1_replace_brick (call_frame_t *frame, xlator_t *this,
                 "%s with operation=%d", src_brick,
                 dst_brick, op);
 
-
-        ret = dict_allocate_and_serialize (dict,
-                                           &req.dict.dict_val,
-                                           (size_t *)&req.dict.dict_len);
-        if (ret < 0) {
-                gf_log (this->name, GF_LOG_DEBUG,
-                        "failed to get serialized length of dict");
-                goto out;
-        }
-
-        ret = cli_cmd_submit (&req, frame, cli_rpc_prog,
-                              GLUSTER_CLI_REPLACE_BRICK, NULL,
-                              this, gf_cli3_1_replace_brick_cbk,
-                              (xdrproc_t) xdr_gf_cli_req);
+        ret = cli_to_glusterd (&req, frame, gf_cli_replace_brick_cbk,
+                               (xdrproc_t) xdr_gf_cli_req, dict,
+                               GLUSTER_CLI_REPLACE_BRICK, this, cli_rpc_prog,
+                               NULL);
 
 out:
         gf_log ("cli", GF_LOG_DEBUG, "Returning %d", ret);
 
-        if (req.dict.dict_val) {
-                GF_FREE (req.dict.dict_val);
-        }
+        GF_FREE (req.dict.dict_val);
 
         return ret;
 }
 
 
 int32_t
-gf_cli3_1_log_rotate (call_frame_t *frame, xlator_t *this,
+gf_cli_log_rotate (call_frame_t *frame, xlator_t *this,
                       void *data)
 {
         gf_cli_req                req = {{0,}};
@@ -2997,31 +3392,20 @@ gf_cli3_1_log_rotate (call_frame_t *frame, xlator_t *this,
 
         dict = data;
 
-        ret = dict_allocate_and_serialize (dict,
-                                           &req.dict.dict_val,
-                                           (size_t *)&req.dict.dict_len);
-
-        if (ret < 0) {
-                gf_log (THIS->name, GF_LOG_ERROR, "failed to serialize dict");
-                goto out;
-        }
-
-        ret = cli_cmd_submit (&req, frame, cli_rpc_prog,
-                              GLUSTER_CLI_LOG_ROTATE, NULL,
-                              this, gf_cli3_1_log_rotate_cbk,
-                              (xdrproc_t) xdr_gf_cli_req);
-
+        ret = cli_to_glusterd (&req, frame, gf_cli_log_rotate_cbk,
+                               (xdrproc_t) xdr_gf_cli_req, dict,
+                               GLUSTER_CLI_LOG_ROTATE, this, cli_rpc_prog,
+                               NULL);
 
 out:
         gf_log ("cli", GF_LOG_DEBUG, "Returning %d", ret);
 
-        if (req.dict.dict_val)
-                GF_FREE (req.dict.dict_val);
+        GF_FREE (req.dict.dict_val);
         return ret;
 }
 
 int32_t
-gf_cli3_1_sync_volume (call_frame_t *frame, xlator_t *this,
+gf_cli_sync_volume (call_frame_t *frame, xlator_t *this,
                        void *data)
 {
         int               ret = 0;
@@ -3034,35 +3418,27 @@ gf_cli3_1_sync_volume (call_frame_t *frame, xlator_t *this,
         }
 
         dict = data;
-        ret = dict_allocate_and_serialize (dict,
-                                           &req.dict.dict_val,
-                                           (size_t *)&req.dict.dict_len);
-
-        if (ret < 0) {
-                gf_log (THIS->name, GF_LOG_ERROR, "failed to serialize dict");
-                goto out;
-        }
 
-        ret = cli_cmd_submit (&req, frame,
-                              cli_rpc_prog, GLUSTER_CLI_SYNC_VOLUME,
-                              NULL, this, gf_cli3_1_sync_volume_cbk,
-                              (xdrproc_t) xdr_gf_cli_req);
+        ret = cli_to_glusterd (&req, frame, gf_cli_sync_volume_cbk,
+                               (xdrproc_t) xdr_gf_cli_req, dict,
+                               GLUSTER_CLI_SYNC_VOLUME, this, cli_rpc_prog,
+                               NULL);
 
 out:
         gf_log ("cli", GF_LOG_DEBUG, "Returning %d", ret);
-        if (req.dict.dict_val)
-                GF_FREE (req.dict.dict_val);
+        GF_FREE (req.dict.dict_val);
 
         return ret;
 }
 
 int32_t
-gf_cli3_1_getspec (call_frame_t *frame, xlator_t *this,
+gf_cli_getspec (call_frame_t *frame, xlator_t *this,
                          void *data)
 {
         gf_getspec_req          req = {0,};
         int                     ret = 0;
         dict_t                  *dict = NULL;
+        dict_t                  *op_dict = NULL;
 
         if (!frame || !this ||  !data) {
                 ret = -1;
@@ -3075,19 +3451,52 @@ gf_cli3_1_getspec (call_frame_t *frame, xlator_t *this,
         if (ret)
                 goto out;
 
+        op_dict = dict_new ();
+        if (!dict) {
+                ret = -1;
+                goto out;
+        }
+
+        // Set the supported min and max op-versions, so glusterd can make a
+        // decision
+        ret = dict_set_int32 (op_dict, "min-op-version", GD_OP_VERSION_MIN);
+        if (ret) {
+                gf_log (THIS->name, GF_LOG_ERROR, "Failed to set min-op-version"
+                        " in request dict");
+                goto out;
+        }
+
+        ret = dict_set_int32 (op_dict, "max-op-version", GD_OP_VERSION_MAX);
+        if (ret) {
+                gf_log (THIS->name, GF_LOG_ERROR, "Failed to set max-op-version"
+                        " in request dict");
+                goto out;
+        }
+
+        ret = dict_allocate_and_serialize (op_dict, &req.xdata.xdata_val,
+                                           &req.xdata.xdata_len);
+        if (ret < 0) {
+                gf_log (THIS->name, GF_LOG_ERROR,
+                        "Failed to serialize dictionary");
+                goto out;
+        }
+
         ret = cli_cmd_submit (&req, frame, &cli_handshake_prog,
                               GF_HNDSK_GETSPEC, NULL,
-                              this, gf_cli3_1_getspec_cbk,
+                              this, gf_cli_getspec_cbk,
                               (xdrproc_t) xdr_gf_getspec_req);
 
 out:
+        if (op_dict) {
+                dict_unref(op_dict);
+        }
         gf_log ("cli", GF_LOG_DEBUG, "Returning %d", ret);
 
         return ret;
 }
 
 int32_t
-gf_cli3_1_quota (call_frame_t *frame, xlator_t *this,
+gf_cli_quota (call_frame_t *frame, xlator_t *this,
                  void *data)
 {
         gf_cli_req          req = {{0,}};
@@ -3101,27 +3510,18 @@ gf_cli3_1_quota (call_frame_t *frame, xlator_t *this,
 
         dict = data;
 
-        ret = dict_allocate_and_serialize (dict,
-                                           &req.dict.dict_val,
-                                           (size_t *)&req.dict.dict_len);
-        if (ret < 0) {
-                gf_log (this->name, GF_LOG_ERROR,
-                        "failed to get serialized length of dict");
-                goto out;
-        }
+        ret = cli_to_glusterd (&req, frame, gf_cli_quota_cbk,
+                               (xdrproc_t) xdr_gf_cli_req, dict,
+                               GLUSTER_CLI_QUOTA, this, cli_rpc_prog, NULL);
 
-        ret = cli_cmd_submit (&req, frame, cli_rpc_prog,
-                              GLUSTER_CLI_QUOTA, NULL,
-                              this, gf_cli3_1_quota_cbk,
-                              (xdrproc_t) xdr_gf_cli_req);
-
-        GF_FREE (req.dict.dict_val);
 out:
+        GF_FREE (req.dict.dict_val);
+
         return ret;
 }
 
 int32_t
-gf_cli3_1_pmap_b2p (call_frame_t *frame, xlator_t *this, void *data)
+gf_cli_pmap_b2p (call_frame_t *frame, xlator_t *this, void *data)
 {
         pmap_port_by_brick_req  req = {0,};
         int                     ret = 0;
@@ -3140,7 +3540,7 @@ gf_cli3_1_pmap_b2p (call_frame_t *frame, xlator_t *this, void *data)
 
         ret = cli_cmd_submit (&req, frame, &cli_pmap_prog,
                               GF_PMAP_PORTBYBRICK, NULL,
-                              this, gf_cli3_1_pmap_b2p_cbk,
+                              this, gf_cli_pmap_b2p_cbk,
                               (xdrproc_t) xdr_pmap_port_by_brick_req);
 
 out:
@@ -3150,7 +3550,7 @@ out:
 }
 
 static int
-gf_cli3_1_fsm_log_cbk (struct rpc_req *req, struct iovec *iov,
+gf_cli_fsm_log_cbk (struct rpc_req *req, struct iovec *iov,
                        int count, void *myframe)
 {
         gf1_cli_fsm_log_rsp        rsp   = {0,};
@@ -3170,14 +3570,15 @@ gf_cli3_1_fsm_log_cbk (struct rpc_req *req, struct iovec *iov,
 
         ret = xdr_to_generic (*iov, &rsp, (xdrproc_t)xdr_gf1_cli_fsm_log_rsp);
         if (ret < 0) {
-                gf_log ("", GF_LOG_ERROR, "error");
+                gf_log (((call_frame_t *) myframe)->this->name, GF_LOG_ERROR,
+                        "Failed to decode xdr response");
                 goto out;
         }
 
         if (rsp.op_ret) {
                 if (strcmp (rsp.op_errstr, ""))
-                        cli_out ("%s", rsp.op_errstr);
-                cli_out ("fsm log unsuccessful");
+                        cli_err ("%s", rsp.op_errstr);
+                cli_err ("fsm log unsuccessful");
                 ret = rsp.op_ret;
                 goto out;
         }
@@ -3193,7 +3594,7 @@ gf_cli3_1_fsm_log_cbk (struct rpc_req *req, struct iovec *iov,
                                 &dict);
 
         if (ret) {
-                cli_out ("bad response");
+                cli_err ("bad response");
                 goto out;
         }
 
@@ -3201,7 +3602,7 @@ gf_cli3_1_fsm_log_cbk (struct rpc_req *req, struct iovec *iov,
         if (tr_count)
                 cli_out("number of transitions: %d", tr_count);
         else
-                cli_out("No transitions");
+                cli_err("No transitions");
         for (i = 0; i < tr_count; i++) {
                 memset (key, 0, sizeof (key));
                 snprintf (key, sizeof (key), "log%d-old-state", i);
@@ -3240,7 +3641,7 @@ out:
 }
 
 int32_t
-gf_cli3_1_fsm_log (call_frame_t *frame, xlator_t *this, void *data)
+gf_cli_fsm_log (call_frame_t *frame, xlator_t *this, void *data)
 {
         int                        ret = -1;
         gf1_cli_fsm_log_req        req = {0,};
@@ -3254,7 +3655,7 @@ gf_cli3_1_fsm_log (call_frame_t *frame, xlator_t *this, void *data)
         req.name = data;
         ret = cli_cmd_submit (&req, frame, cli_rpc_prog,
                               GLUSTER_CLI_FSM_LOG, NULL,
-                              this, gf_cli3_1_fsm_log_cbk,
+                              this, gf_cli_fsm_log_cbk,
                               (xdrproc_t) xdr_gf1_cli_fsm_log_req);
 
 out:
@@ -3264,14 +3665,17 @@ out:
 }
 
 int
-gf_cli3_1_gsync_config_command (dict_t *dict)
+gf_cli_gsync_config_command (dict_t *dict)
 {
         runner_t runner     = {0,};
         char *subop         = NULL;
         char *gwd           = NULL;
         char *slave         = NULL;
+        char *confpath      = NULL;
         char *master        = NULL;
         char *op_name       = NULL;
+        int   ret           = -1;
+        char  conf_path[PATH_MAX] = "";
 
         if (dict_get_str (dict, "subop", &subop) != 0)
                 return -1;
@@ -3290,9 +3694,17 @@ gf_cli3_1_gsync_config_command (dict_t *dict)
         if (dict_get_str (dict, "op_name", &op_name) != 0)
                 op_name = NULL;
 
+        ret = dict_get_str (dict, "conf_path", &confpath);
+        if (!confpath) {
+                ret = snprintf (conf_path, sizeof(conf_path) - 1,
+                                "%s/"GEOREP"/gsyncd_template.conf", gwd);
+                conf_path[ret] = '\0';
+                confpath = conf_path;
+        }
+
         runinit (&runner);
         runner_add_args (&runner, GSYNCD_PREFIX"/gsyncd", "-c", NULL);
-        runner_argprintf (&runner, "%s/"GSYNC_CONF, gwd);
+        runner_argprintf (&runner, "%s", confpath);
         if (master)
                 runner_argprintf (&runner, ":%s", master);
         runner_add_arg (&runner, slave);
@@ -3304,64 +3716,643 @@ gf_cli3_1_gsync_config_command (dict_t *dict)
 }
 
 int
-gf_cli3_1_gsync_out_status (dict_t *dict)
+gf_cli_fetch_gsyncd_status_values (char *status,
+                                   gf_cli_gsync_status_t *sts_val)
 {
-        int              gsync_count = 0;
-        int              i = 0;
-        int              ret = 0;
-        char             mst[PATH_MAX] = {0, };
-        char             slv[PATH_MAX]= {0, };
-        char             sts[PATH_MAX] = {0, };
-        char             hyphens[81] = {0, };
-        char             *mst_val = NULL;
-        char             *slv_val = NULL;
-        char             *sts_val = NULL;
-
-        cli_out ("%-20s %-50s %-10s", "MASTER", "SLAVE", "STATUS");
-
-        for (i=0; i<sizeof(hyphens)-1; i++)
-                hyphens[i] = '-';
+        int32_t   ret      = -1;
+        char     *tmp      = NULL;
+        char     *save_ptr = NULL;
+        char     *key      = NULL;
+        char     *value    = NULL;
 
-        cli_out ("%s", hyphens);
+        if (!status || !sts_val) {
+                gf_log ("", GF_LOG_ERROR, "status or sts_val is null");
+                goto out;
+        }
 
+        tmp = strtok_r (status, "\n", &save_ptr);
 
-        ret = dict_get_int32 (dict, "gsync-count", &gsync_count);
+        if (tmp)
+                sts_val->health = gf_strdup (tmp);
+
+        while (tmp) {
+                key = strtok_r (tmp, "=", &value);
+
+                if ((key) && (!strcmp(key, "Uptime")))
+                        sts_val->uptime = gf_strdup (value);
+
+                if ((key) && (!strcmp(key, "FilesSyncd")))
+                        sts_val->files_syncd = gf_strdup (value);
+
+                if ((key) && (!strcmp(key, "FilesPending")))
+                        sts_val->files_pending = gf_strdup (value);
+
+                if ((key) && (!strcmp(key, "BytesPending"))) {
+                        value = gf_uint64_2human_readable(atol(value));
+                        sts_val->bytes_pending = gf_strdup (value);
+                }
+
+                if ((key) && (!strcmp(key, "DeletesPending")))
+                        sts_val->deletes_pending = gf_strdup (value);
+
+                tmp = strtok_r (NULL, ";", &save_ptr);
+        }
+
+        if (sts_val->health)
+                ret = 0;
+
+        if (!sts_val->uptime)
+                sts_val->uptime = gf_strdup ("N/A");
+
+        if (!sts_val->files_syncd)
+                sts_val->files_syncd = gf_strdup ("N/A");
+
+        if (!sts_val->files_pending)
+                sts_val->files_pending = gf_strdup ("N/A");
+
+        if (!sts_val->bytes_pending)
+                sts_val->bytes_pending = gf_strdup ("N/A");
+
+        if (!sts_val->deletes_pending)
+                sts_val->deletes_pending = gf_strdup ("N/A");
+
+out:
+        gf_log ("", GF_LOG_DEBUG, "Returning %d.", ret);
+        return ret;
+}
+
+char*
+get_struct_variable (int mem_num, gf_cli_gsync_status_t *sts_val)
+{
+        switch (mem_num) {
+        case 0: return (sts_val->node);
+        case 1: return (sts_val->master);
+        case 2: return (sts_val->slave);
+        case 3: return (sts_val->health);
+        case 4: return (sts_val->uptime);
+        case 5: return (sts_val->files_syncd);
+        case 6: return (sts_val->files_pending);
+        case 7: return (sts_val->bytes_pending);
+        case 8: return (sts_val->deletes_pending);
+        default:
+                 goto out;
+        }
+
+out:
+        return NULL;
+}
+
+int
+gf_cli_print_status (char **title_values,
+                     gf_cli_gsync_status_t **sts_vals,
+                     int *spacing, int gsync_count,
+                     int number_of_fields, int is_detail)
+{
+        int     indents                  = 0;
+        int     i                        = 0;
+        int     j                        = 0;
+        int     ret                      = 0;
+        int     total_spacing            = 0;
+        char  **output_values            = NULL;
+        char   *tmp                      = NULL;
+        char   *hyphens                  = NULL;
+        char    heading[PATH_MAX]        = {0, };
+        char    indent_spaces[PATH_MAX]  = {0, };
+
+        /* calculating spacing for hyphens */
+        for (i = 0; i < number_of_fields; i++) {
+                /* Suppressing master and slave output for status detail */
+                if ((is_detail) && ((i == 1) || (i == 2))) {
+                        total_spacing++;
+                        continue;
+                } else if ((!is_detail) && (i > 4)) {
+                       /* Suppressing detailed output for
+                        * status */
+                        continue;
+                }
+                spacing[i] += 3; /* Adding extra space to
+                                    distinguish between fields */
+                total_spacing += spacing[i];
+        }
+        total_spacing += 4; /* For the spacing between the fields */
+
+        /* char pointers for each field */
+        output_values = GF_CALLOC (number_of_fields, sizeof (char *),
+                                   gf_common_mt_char);
+        if (!output_values) {
+                ret = -1;
+                goto out;
+        }
+        for (i = 0; i < number_of_fields; i++) {
+                output_values[i] = GF_CALLOC (spacing[i] + 1, sizeof (char),
+                                              gf_common_mt_char);
+                if (!output_values[i]) {
+                        ret = -1;
+                        goto out;
+                }
+        }
+
+        hyphens = GF_CALLOC (total_spacing + 1, sizeof (char),
+                             gf_common_mt_char);
+        if (!hyphens) {
+                ret = -1;
+                goto out;
+        }
+
+        ret = snprintf(heading, sizeof(heading), "MASTER: %s  SLAVE: %s",
+                       sts_vals[0]->master, sts_vals[0]->slave);
         if (ret) {
-                gf_log ("cli", GF_LOG_INFO, "No active geo-replication sessions"
-                        "present for the selected");
+                if (ret < sizeof(heading))
+                        heading[ret] = '\0';
+                else
+                        heading[sizeof(heading) - 1] = '\0';
                 ret = 0;
+        } else {
+                ret = -1;
                 goto out;
         }
 
-        for (i = 1; i <= gsync_count; i++) {
-                snprintf (mst, sizeof(mst), "master%d", i);
-                snprintf (slv, sizeof(slv), "slave%d", i);
-                snprintf (sts, sizeof(sts), "status%d", i);
+        if (is_detail) {
+                cli_out (" ");
+                if (strlen(heading) > total_spacing)
+                        cli_out ("%s", heading);
+                else {
+                        /* Printing the heading with centre justification */
+                        indents = (total_spacing - strlen(heading)) / 2;
+                        memset (indent_spaces, ' ', indents);
+                        indent_spaces[indents] = '\0';
+                        ret = snprintf (hyphens, total_spacing, "%s%s",
+                                        indent_spaces, heading);
+                        if (ret) {
+                                hyphens[ret] = '\0';
+                                cli_out ("%s", hyphens);
+                                ret = 0;
+                        } else {
+                                ret = -1;
+                                goto out;
+                        }
+                }
+                cli_out (" ");
+        }
 
-                ret = dict_get_str (dict, mst, &mst_val);
+        /* setting the title "NODE", "MASTER", etc. from title_values[]
+           and printing the same */
+        for (j = 0; j < number_of_fields; j++) {
+                /* Suppressing master and slave output for status detail */
+                if ((is_detail) && ((j == 1) || (j == 2))) {
+                        output_values[j][0] = '\0';
+                        continue;
+                } else if ((!is_detail) && (j > 4)) {
+                       /* Suppressing detailed output for
+                        * status */
+                       output_values[j][0] = '\0';
+                        continue;
+                }
+                memset (output_values[j], ' ', spacing[j]);
+                memcpy (output_values[j], title_values[j],
+                        strlen(title_values[j]));
+                output_values[j][spacing[j]] = '\0';
+        }
+        cli_out ("%s %s %s %s %s %s %s %s %s", output_values[0],
+                 output_values[1], output_values[2], output_values[3],
+                 output_values[4], output_values[5], output_values[6],
+                 output_values[7], output_values[8]);
+
+        /* setting and printing the hyphens */
+        memset (hyphens, '-', total_spacing);
+        hyphens[total_spacing] = '\0';
+        cli_out ("%s", hyphens);
+
+        for (i = 0; i < gsync_count; i++) {
+                for (j = 0; j < number_of_fields; j++) {
+                        /* Suppressing master and slave output for
+                         * status detail */
+                        if ((is_detail) && ((j == 1) || (j == 2))) {
+                                output_values[j][0] = '\0';
+                                continue;
+                        }  else if ((!is_detail) && (j > 4)) {
+                                /* Suppressing detailed output for
+                                 * status */
+                                output_values[j][0] = '\0';
+                                continue;
+                        }
+                        tmp = get_struct_variable(j, sts_vals[i]);
+                        if (!tmp) {
+                                gf_log ("", GF_LOG_ERROR,
+                                        "struct member empty.");
+                                ret = -1;
+                                goto out;
+                        }
+                        memset (output_values[j], ' ', spacing[j]);
+                        memcpy (output_values[j], tmp, strlen (tmp));
+                        output_values[j][spacing[j]] = '\0';
+                }
+
+                cli_out ("%s %s %s %s %s %s %s %s %s", output_values[0],
+                         output_values[1], output_values[2], output_values[3],
+                         output_values[4], output_values[5], output_values[6],
+                         output_values[7], output_values[8]);
+        }
+
+out:
+        if (output_values) {
+                for (i = 0; i < number_of_fields; i++) {
+                        if (output_values[i])
+                                GF_FREE (output_values[i]);
+                }
+                GF_FREE (output_values);
+        }
+
+        if (hyphens)
+                GF_FREE (hyphens);
+
+        return ret;
+}
+
+int
+gf_cli_read_status_data (dict_t *dict,
+                         gf_cli_gsync_status_t **sts_vals,
+                         int *spacing, int gsync_count,
+                         int number_of_fields)
+{
+        int     ret            = 0;
+        int     i              = 0;
+        int     j              = 0;
+        char    mst[PATH_MAX]  = {0, };
+        char    slv[PATH_MAX]  = {0, };
+        char    sts[PATH_MAX]  = {0, };
+        char    nds[PATH_MAX]  = {0, };
+        char   *status         = NULL;
+        char   *tmp            = NULL;
+
+        /* Storing per node status info in each object */
+        for (i = 0; i < gsync_count; i++) {
+                snprintf (nds, sizeof(nds), "node%d", i + 1);
+                snprintf (mst, sizeof(mst), "master%d", i + 1);
+                snprintf (slv, sizeof(slv), "slave%d", i + 1);
+                snprintf (sts, sizeof(sts), "status%d", i + 1);
+
+                /* Fetching the values from dict, and calculating
+                   the max length for each field */
+                ret = dict_get_str (dict, nds, &(sts_vals[i]->node));
                 if (ret)
                         goto out;
 
-                ret = dict_get_str (dict, slv, &slv_val);
+                ret = dict_get_str (dict, mst, &(sts_vals[i]->master));
                 if (ret)
                         goto out;
 
-                ret = dict_get_str (dict, sts, &sts_val);
+                ret = dict_get_str (dict, slv, &(sts_vals[i]->slave));
                 if (ret)
                         goto out;
 
-                cli_out ("%-20s %-50s %-10s", mst_val,
-                         slv_val, sts_val);
+                ret = dict_get_str (dict, sts, &status);
+                if (ret)
+                        goto out;
 
+                /* Fetching health and uptime from sts_val */
+                ret = gf_cli_fetch_gsyncd_status_values (status, sts_vals[i]);
+                if (ret)
+                        goto out;
+
+                for (j = 0; j < number_of_fields; j++) {
+                        tmp = get_struct_variable(j, sts_vals[i]);
+                        if (!tmp) {
+                                gf_log ("", GF_LOG_ERROR,
+                                        "struct member empty.");
+                                ret = -1;
+                                goto out;
+                        }
+                        if (strlen (tmp) > spacing[j])
+                                spacing[j] = strlen (tmp);
+                }
+        }
+
+out:
+        return ret;
+}
+
+int
+gf_cli_gsync_status_output (dict_t *dict, int status_detail)
+{
+        int                     gsync_count    = 0;
+        int                     i              = 0;
+        int                     j              = 0;
+        int                     ret            = 0;
+        int                     spacing[10]    = {0};
+        int                     num_of_fields  = 9;
+        char                    errmsg[1024]   = "";
+        char                   *master         = NULL;
+        char                   *slave          = NULL;
+        char                   *tmp            = NULL;
+        char                   *title_values[] = {"NODE", "MASTER", "SLAVE",
+                                                  "HEALTH", "UPTIME",
+                                                  "FILES SYNCD",
+                                                  "FILES PENDING",
+                                                  "BYTES PENDING",
+                                                  "DELETES PENDING"};
+        gf_cli_gsync_status_t **sts_vals = NULL;
+
+        /* Checks if any session is active or not */
+        ret = dict_get_int32 (dict, "gsync-count", &gsync_count);
+        if (ret) {
+                ret = dict_get_str (dict, "master", &master);
+
+                ret = dict_get_str (dict, "slave", &slave);
+
+                if (master) {
+                        if (slave)
+                                snprintf (errmsg, sizeof(errmsg), "No active "
+                                          "geo-replication sessions between %s"
+                                          " and %s", master, slave);
+                        else
+                                snprintf (errmsg, sizeof(errmsg), "No active "
+                                          "geo-replication sessions for %s",
+                                          master);
+                } else
+                        snprintf (errmsg, sizeof(errmsg), "No active "
+                                  "geo-replication sessions");
+
+                gf_log ("cli", GF_LOG_INFO, "%s", errmsg);
+                cli_out ("%s", errmsg);
+                ret = 0;
+                goto out;
+        }
+
+        for (i = 0; i < num_of_fields; i++)
+                spacing[i] = strlen(title_values[i]);
+
+        /* gsync_count = number of nodes reporting output.
+           each sts_val object will store output of each
+           node */
+        sts_vals = GF_CALLOC (gsync_count, sizeof (gf_cli_gsync_status_t *),
+                              gf_common_mt_char);
+        if (!sts_vals) {
+                ret = -1;
+                goto out;
+        }
+        for (i = 0; i < gsync_count; i++) {
+                sts_vals[i] = GF_CALLOC (1, sizeof (gf_cli_gsync_status_t),
+                                         gf_common_mt_char);
+                if (!sts_vals[i]) {
+                        ret = -1;
+                        goto out;
+                }
+        }
+
+        ret = gf_cli_read_status_data (dict, sts_vals, spacing,
+                                       gsync_count, num_of_fields);
+        if (ret) {
+                gf_log ("", GF_LOG_ERROR, "Unable to read status data");
+                goto out;
+        }
+
+        ret = gf_cli_print_status (title_values, sts_vals, spacing, gsync_count,
+                                   num_of_fields, status_detail);
+        if (ret) {
+                gf_log ("", GF_LOG_ERROR, "Unable to print status output");
+                goto out;
+        }
+
+out:
+        if (sts_vals) {
+                for (i = 0; i < gsync_count; i++) {
+                        for (j = 3; j < num_of_fields; j++) {
+                                tmp = get_struct_variable(j, sts_vals[i]);
+                                if (tmp)
+                                        GF_FREE (tmp);
+                        }
+                }
+                GF_FREE (sts_vals);
         }
 
- out:
         return ret;
+}
 
+static int32_t
+write_contents_to_common_pem_file (dict_t *dict, int output_count)
+{
+        char            *workdir                   = NULL;
+        char             common_pem_file[PATH_MAX] = "";
+        char            *output                    = NULL;
+        char             output_name[PATH_MAX]     = "";
+        int              bytes_writen              = 0;
+        int              fd                        = -1;
+        int              ret                       = -1;
+        int              i                         = -1;
+
+        ret = dict_get_str (dict, "glusterd_workdir", &workdir);
+        if (ret || !workdir) {
+                gf_log ("", GF_LOG_ERROR, "Unable to fetch workdir");
+                ret = -1;
+                goto out;
+        }
+
+        snprintf (common_pem_file, sizeof(common_pem_file),
+                  "%s/geo-replication/common_secret.pem.pub",
+                  workdir);
+
+        unlink (common_pem_file);
+
+        fd = open (common_pem_file, O_WRONLY | O_CREAT, 0600);
+        if (fd == -1) {
+                gf_log ("", GF_LOG_ERROR, "Failed to open %s"
+                        " Error : %s", common_pem_file,
+                        strerror (errno));
+                ret = -1;
+                goto out;
+        }
+
+        for (i = 1; i <= output_count; i++) {
+                memset (output_name, '\0', sizeof (output_name));
+                snprintf (output_name, sizeof (output_name),
+                          "output_%d", i);
+                ret = dict_get_str (dict, output_name, &output);
+                if (ret) {
+                        gf_log ("", GF_LOG_ERROR, "Failed to get %s.",
+                                output_name);
+                        cli_out ("Unable to fetch output.");
+                }
+                if (output) {
+                        bytes_writen = write (fd, output, strlen(output));
+                        if (bytes_writen != strlen(output)) {
+                                gf_log ("", GF_LOG_ERROR, "Failed to write "
+                                        "to %s", common_pem_file);
+                                ret = -1;
+                                goto out;
+                        }
+                        /* Adding the new line character */
+                        bytes_writen = write (fd, "\n", strlen("\n"));
+                        if (bytes_writen != strlen("\n")) {
+                                gf_log ("", GF_LOG_ERROR,
+                                        "Failed to add new line char");
+                                ret = -1;
+                                goto out;
+                        }
+                        output = NULL;
+                }
+        }
+
+        cli_out ("Common secret pub file present at %s", common_pem_file);
+        ret = 0;
+out:
+        if (fd)
+                close (fd);
+
+        gf_log ("", GF_LOG_DEBUG, "Returning %d", ret);
+        return ret;
 }
 
 int
-gf_cli3_1_gsync_set_cbk (struct rpc_req *req, struct iovec *iov,
+gf_cli_sys_exec_cbk (struct rpc_req *req, struct iovec *iov,
+                     int count, void *myframe)
+{
+        int                     ret     = -1;
+        int                     output_count     = -1;
+        int                     i     = -1;
+        char                   *output  = NULL;
+        char                   *command = NULL;
+        char                    output_name[PATH_MAX] = "";
+        gf_cli_rsp              rsp     = {0, };
+        dict_t                  *dict   = NULL;
+        call_frame_t            *frame  = NULL;
+
+        if (req->rpc_status == -1) {
+                ret = -1;
+                goto out;
+        }
+
+        frame = myframe;
+
+        ret = xdr_to_generic (*iov, &rsp, (xdrproc_t)xdr_gf_cli_rsp);
+        if (ret < 0) {
+                gf_log (frame->this->name, GF_LOG_ERROR,
+                        "Failed to decode xdr response");
+                goto out;
+       }
+
+        dict = dict_new ();
+
+        if (!dict) {
+                ret = -1;
+                goto out;
+        }
+
+        ret = dict_unserialize (rsp.dict.dict_val, rsp.dict.dict_len, &dict);
+
+        if (ret)
+                goto out;
+
+        if (rsp.op_ret) {
+                cli_err ("%s", rsp.op_errstr ? rsp.op_errstr :
+                         "Command failed.");
+                ret = rsp.op_ret;
+                goto out;
+        }
+
+        ret = dict_get_int32 (dict, "output_count", &output_count);
+        if (ret) {
+                cli_out ("Command executed successfully.");
+                ret = 0;
+                goto out;
+        }
+
+        ret = dict_get_str (dict, "command", &command);
+        if (ret) {
+                gf_log ("", GF_LOG_ERROR,
+                        "Unable to get command from dict");
+                goto out;
+        }
+
+        if (!strcmp (command, "gsec_create")) {
+                ret = write_contents_to_common_pem_file (dict, output_count);
+                if (!ret)
+                        goto out;
+        }
+
+        for (i = 1; i <= output_count; i++) {
+                memset (output_name, '\0', sizeof (output_name));
+                snprintf (output_name, sizeof (output_name),
+                          "output_%d", i);
+                ret = dict_get_str (dict, output_name, &output);
+                if (ret) {
+                        gf_log ("", GF_LOG_ERROR, "Failed to get %s.",
+                                output_name);
+                        cli_out ("Unable to fetch output.");
+                }
+                if (output) {
+                        cli_out ("%s", output);
+                        output = NULL;
+                }
+        }
+
+        ret = 0;
+out:
+        if (dict)
+                dict_unref (dict);
+        cli_cmd_broadcast_response (ret);
+
+        free (rsp.dict.dict_val);
+
+        return ret;
+}
+
+int
+gf_cli_copy_file_cbk (struct rpc_req *req, struct iovec *iov,
+                      int count, void *myframe)
+{
+        int                     ret     = -1;
+        gf_cli_rsp              rsp     = {0, };
+        dict_t                  *dict   = NULL;
+        call_frame_t            *frame  = NULL;
+
+        if (req->rpc_status == -1) {
+                ret = -1;
+                goto out;
+        }
+
+        frame = myframe;
+
+        ret = xdr_to_generic (*iov, &rsp, (xdrproc_t)xdr_gf_cli_rsp);
+        if (ret < 0) {
+                gf_log (frame->this->name, GF_LOG_ERROR,
+                        "Failed to decode xdr response");
+                goto out;
+       }
+
+        dict = dict_new ();
+
+        if (!dict) {
+                ret = -1;
+                goto out;
+        }
+
+        ret = dict_unserialize (rsp.dict.dict_val, rsp.dict.dict_len, &dict);
+
+       if (ret)
+                goto out;
+
+        if (rsp.op_ret) {
+                cli_err ("%s", rsp.op_errstr ? rsp.op_errstr :
+                         "Copy unsuccessful");
+                ret = rsp.op_ret;
+                goto out;
+        }
+
+        cli_out ("Successfully copied file.");
+
+out:
+        if (dict)
+                dict_unref (dict);
+        cli_cmd_broadcast_response (ret);
+
+        free (rsp.dict.dict_val);
+
+        return ret;
+}
+
+int
+gf_cli_gsync_set_cbk (struct rpc_req *req, struct iovec *iov,
                          int count, void *myframe)
 {
         int                     ret     = -1;
@@ -3370,17 +4361,22 @@ gf_cli3_1_gsync_set_cbk (struct rpc_req *req, struct iovec *iov,
         char                    *gsync_status = NULL;
         char                    *master = NULL;
         char                    *slave  = NULL;
-        int32_t                  type    = 0;
+        int32_t                  type   = 0;
+        call_frame_t            *frame  = NULL;
+        gf_boolean_t             status_detail = _gf_false;
+
 
         if (req->rpc_status == -1) {
                 ret = -1;
                 goto out;
         }
 
+        frame = myframe;
+
         ret = xdr_to_generic (*iov, &rsp, (xdrproc_t)xdr_gf_cli_rsp);
         if (ret < 0) {
-                gf_log ("", GF_LOG_ERROR,
-                        "Unable to get response structure");
+                gf_log (frame->this->name, GF_LOG_ERROR,
+                        "Failed to decode xdr response");
                 goto out;
         }
 
@@ -3396,19 +4392,17 @@ gf_cli3_1_gsync_set_cbk (struct rpc_req *req, struct iovec *iov,
         if (ret)
                 goto out;
 
-#if (HAVE_LIB_XML)
         if (global_state->mode & GLUSTER_MODE_XML) {
-                ret = cli_xml_output_dict ("volGeoRep", dict, rsp.op_ret,
-                                           rsp.op_errno, rsp.op_errstr);
+                ret = cli_xml_output_vol_gsync (dict, rsp.op_ret, rsp.op_errno,
+                                                rsp.op_errstr);
                 if (ret)
                         gf_log ("cli", GF_LOG_ERROR,
                                 "Error outputting to xml");
                 goto out;
         }
-#endif
 
         if (rsp.op_ret) {
-                cli_out ("%s", rsp.op_errstr ? rsp.op_errstr :
+                cli_err ("%s", rsp.op_errstr ? rsp.op_errstr :
                          GEOREP" command unsuccessful");
                 ret = rsp.op_ret;
                 goto out;
@@ -3422,7 +4416,7 @@ gf_cli3_1_gsync_set_cbk (struct rpc_req *req, struct iovec *iov,
 
         ret = dict_get_int32 (dict, "type", &type);
         if (ret) {
-                gf_log (THIS->name, GF_LOG_ERROR, "failed to get type");
+                gf_log (frame->this->name, GF_LOG_ERROR, "failed to get type");
                 goto out;
         }
 
@@ -3442,29 +4436,50 @@ gf_cli3_1_gsync_set_cbk (struct rpc_req *req, struct iovec *iov,
                 break;
 
                 case GF_GSYNC_OPTION_TYPE_CONFIG:
-                        ret = gf_cli3_1_gsync_config_command (dict);
+                        ret = gf_cli_gsync_config_command (dict);
                 break;
 
                 case GF_GSYNC_OPTION_TYPE_STATUS:
-                        ret = gf_cli3_1_gsync_out_status (dict);
-                        goto out;
+                        status_detail = dict_get_str_boolean (dict,
+                                                              "status-detail",
+                                                              _gf_false);
+                        ret = gf_cli_gsync_status_output (dict, status_detail);
+                break;
+
+                case GF_GSYNC_OPTION_TYPE_DELETE:
+                        if (dict_get_str (dict, "master", &master) != 0)
+                                master = "???";
+                        if (dict_get_str (dict, "slave", &slave) != 0)
+                                slave = "???";
+                        cli_out ("Deleting " GEOREP " session between %s & %s"
+                                 " has been successful", master, slave);
+                break;
+
+                case GF_GSYNC_OPTION_TYPE_CREATE:
+                        if (dict_get_str (dict, "master", &master) != 0)
+                                master = "???";
+                        if (dict_get_str (dict, "slave", &slave) != 0)
+                                slave = "???";
+                        cli_out ("Creating " GEOREP " session between %s & %s"
+                                 " has been successful", master, slave);
+                break;
+
                 default:
                         cli_out (GEOREP" command executed successfully");
         }
 
 out:
-
+        if (dict)
+                dict_unref (dict);
         cli_cmd_broadcast_response (ret);
 
-        if (rsp.dict.dict_val)
-                free (rsp.dict.dict_val);
+        free (rsp.dict.dict_val);
 
         return ret;
 }
 
 int32_t
-gf_cli3_1_gsync_set (call_frame_t *frame, xlator_t *this,
-                     void *data)
+gf_cli_sys_exec (call_frame_t *frame, xlator_t *this, void *data)
 {
         int                      ret    = 0;
         dict_t                  *dict   = NULL;
@@ -3472,29 +4487,67 @@ gf_cli3_1_gsync_set (call_frame_t *frame, xlator_t *this,
 
         if (!frame || !this || !data) {
                 ret = -1;
+                gf_log ("cli", GF_LOG_ERROR, "Invalid data");
                 goto out;
         }
 
         dict = data;
 
-        ret = dict_allocate_and_serialize (dict,
-                                           &req.dict.dict_val,
-                                           (size_t *) &req.dict.dict_len);
-        if (ret < 0) {
-                gf_log (this->name, GF_LOG_ERROR,
-                        "failed to serialize the data");
+        ret = cli_to_glusterd (&req, frame, gf_cli_sys_exec_cbk,
+                               (xdrproc_t) xdr_gf_cli_req, dict,
+                               GLUSTER_CLI_SYS_EXEC, this, cli_rpc_prog,
+                               NULL);
+out:
+        GF_FREE (req.dict.dict_val);
+        return ret;
+}
+
+int32_t
+gf_cli_copy_file (call_frame_t *frame, xlator_t *this, void *data)
+{
+        int                      ret    = 0;
+        dict_t                  *dict   = NULL;
+        gf_cli_req               req = {{0,}};
 
+        if (!frame || !this || !data) {
+                ret = -1;
+                gf_log ("cli", GF_LOG_ERROR, "Invalid data");
                 goto out;
         }
 
-        ret = cli_cmd_submit (&req, frame, cli_rpc_prog,
-                              GLUSTER_CLI_GSYNC_SET, NULL,
-                              this, gf_cli3_1_gsync_set_cbk,
-                              (xdrproc_t) xdr_gf_cli_req);
+        dict = data;
+
+        ret = cli_to_glusterd (&req, frame, gf_cli_copy_file_cbk,
+                               (xdrproc_t) xdr_gf_cli_req, dict,
+                               GLUSTER_CLI_COPY_FILE, this, cli_rpc_prog,
+                               NULL);
+out:
+        GF_FREE (req.dict.dict_val);
+        return ret;
+}
+
+int32_t
+gf_cli_gsync_set (call_frame_t *frame, xlator_t *this,
+                     void *data)
+{
+        int                      ret    = 0;
+        dict_t                  *dict   = NULL;
+        gf_cli_req               req = {{0,}};
+
+        if (!frame || !this || !data) {
+                ret = -1;
+                goto out;
+        }
+
+        dict = data;
+
+        ret = cli_to_glusterd (&req, frame, gf_cli_gsync_set_cbk,
+                               (xdrproc_t) xdr_gf_cli_req, dict,
+                               GLUSTER_CLI_GSYNC_SET, this, cli_rpc_prog,
+                               NULL);
 
 out:
-        if (req.dict.dict_val)
-                GF_FREE (req.dict.dict_val);
+        GF_FREE (req.dict.dict_val);
 
         return ret;
 }
@@ -3573,7 +4626,7 @@ cmd_profile_volume_brick_out (dict_t *dict, int count, int interval)
                 snprintf (key, sizeof (key), "%d-%d-%d-maxlatency", count,
                           interval, i);
                 ret = dict_get_double (dict, key, &profile_info[i].max_latency);
-                profile_info[i].fop_name = gf_fop_list[i];
+                profile_info[i].fop_name = (char *)gf_fop_list[i];
 
                 total_percentage_latency +=
                        (profile_info[i].fop_hits * profile_info[i].avg_latency);
@@ -3685,7 +4738,7 @@ cmd_profile_volume_brick_out (dict_t *dict, int count, int interval)
 }
 
 int32_t
-gf_cli3_1_profile_volume_cbk (struct rpc_req *req, struct iovec *iov,
+gf_cli_profile_volume_cbk (struct rpc_req *req, struct iovec *iov,
                               int count, void *myframe)
 {
         gf_cli_rsp                        rsp   = {0,};
@@ -3707,7 +4760,8 @@ gf_cli3_1_profile_volume_cbk (struct rpc_req *req, struct iovec *iov,
         gf_log ("cli", GF_LOG_DEBUG, "Received resp to profile");
         ret = xdr_to_generic (*iov, &rsp, (xdrproc_t)xdr_gf_cli_rsp);
         if (ret < 0) {
-                gf_log ("", GF_LOG_ERROR, "error");
+                gf_log (((call_frame_t *) myframe)->this->name, GF_LOG_ERROR,
+                        "Failed to decode xdr response");
                 goto out;
         }
 
@@ -3730,7 +4784,6 @@ gf_cli3_1_profile_volume_cbk (struct rpc_req *req, struct iovec *iov,
                 dict->extra_stdfree = rsp.dict.dict_val;
         }
 
-#if (HAVE_LIB_XML)
         if (global_state->mode & GLUSTER_MODE_XML) {
                 ret = cli_xml_output_vol_profile (dict, rsp.op_ret,
                                                   rsp.op_errno,
@@ -3740,7 +4793,6 @@ gf_cli3_1_profile_volume_cbk (struct rpc_req *req, struct iovec *iov,
                                 "Error outputting to xml");
                 goto out;
         }
-#endif
 
         ret = dict_get_str (dict, "volname", &volname);
         if (ret)
@@ -3751,7 +4803,7 @@ gf_cli3_1_profile_volume_cbk (struct rpc_req *req, struct iovec *iov,
                 goto out;
 
         if (rsp.op_ret && strcmp (rsp.op_errstr, "")) {
-                cli_out ("%s", rsp.op_errstr);
+                cli_err ("%s", rsp.op_errstr);
         } else {
                 switch (op) {
                 case GF_CLI_STATS_START:
@@ -3789,7 +4841,7 @@ gf_cli3_1_profile_volume_cbk (struct rpc_req *req, struct iovec *iov,
                 goto out;
 
         if (!brick_count) {
-                cli_out ("All bricks of volume %s are down.", volname);
+                cli_err ("All bricks of volume %s are down.", volname);
                 goto out;
         }
 
@@ -3828,14 +4880,13 @@ gf_cli3_1_profile_volume_cbk (struct rpc_req *req, struct iovec *iov,
 out:
         if (dict)
                 dict_unref (dict);
-        if (rsp.op_errstr)
-                free (rsp.op_errstr);
+        free (rsp.op_errstr);
         cli_cmd_broadcast_response (ret);
         return ret;
 }
 
 int32_t
-gf_cli3_1_profile_volume (call_frame_t *frame, xlator_t *this, void *data)
+gf_cli_profile_volume (call_frame_t *frame, xlator_t *this, void *data)
 {
         int                        ret   = -1;
         gf_cli_req                 req   = {{0,}};
@@ -3849,58 +4900,47 @@ gf_cli3_1_profile_volume (call_frame_t *frame, xlator_t *this, void *data)
                 goto out;
         dict = data;
 
-        ret = dict_allocate_and_serialize (dict,
-                                           &req.dict.dict_val,
-                                           (size_t *)&req.dict.dict_len);
-
-        if (ret < 0) {
-                gf_log (this->name, GF_LOG_ERROR,
-                        "failed to serialize the data");
-
-                goto out;
-        }
-
-
-        ret = cli_cmd_submit (&req, frame, cli_rpc_prog,
-                              GLUSTER_CLI_PROFILE_VOLUME, NULL,
-                              this, gf_cli3_1_profile_volume_cbk,
-                              (xdrproc_t) xdr_gf_cli_req);
+        ret = cli_to_glusterd (&req, frame, gf_cli_profile_volume_cbk,
+                               (xdrproc_t) xdr_gf_cli_req, dict,
+                               GLUSTER_CLI_PROFILE_VOLUME, this, cli_rpc_prog,
+                               NULL);
 
 out:
         gf_log ("cli", GF_LOG_DEBUG, "Returning %d", ret);
 
-        if (req.dict.dict_val)
-                GF_FREE (req.dict.dict_val);
+        GF_FREE (req.dict.dict_val);
         return ret;
 }
 
 int32_t
-gf_cli3_1_top_volume_cbk (struct rpc_req *req, struct iovec *iov,
+gf_cli_top_volume_cbk (struct rpc_req *req, struct iovec *iov,
                               int count, void *myframe)
 {
         gf_cli_rsp                        rsp   = {0,};
         int                               ret   = -1;
-        dict_t                            *dict = NULL;
-        gf1_cli_stats_op                op = GF_CLI_STATS_NONE;
+        dict_t                           *dict = NULL;
+        gf1_cli_stats_op                  op = GF_CLI_STATS_NONE;
         char                              key[256] = {0};
         int                               i = 0;
         int32_t                           brick_count = 0;
         char                              brick[1024];
         int32_t                           members = 0;
-        char                              *filename;
-        char                              *bricks;
-        uint64_t                           value = 0;
+        char                             *filename;
+        char                             *bricks;
+        uint64_t                          value = 0;
         int32_t                           j = 0;
         gf1_cli_top_op                    top_op = GF_CLI_TOP_NONE;
         uint64_t                          nr_open = 0;
         uint64_t                          max_nr_open = 0;
         double                            throughput = 0;
         double                            time = 0;
-        long int                          time_sec = 0;
-        long int                           time_usec = 0;
-        struct tm                         *tm = NULL;
+        int32_t                           time_sec = 0;
+        long int                          time_usec = 0;
         char                              timestr[256] = {0, };
-        char                              *openfd_str = NULL;
+        char                             *openfd_str = NULL;
+        gf_boolean_t                      nfs = _gf_false;
+        gf_boolean_t                      clear_stats = _gf_false;
+        int                               stats_cleared = 0;
 
         if (-1 == req->rpc_status) {
                 goto out;
@@ -3909,14 +4949,15 @@ gf_cli3_1_top_volume_cbk (struct rpc_req *req, struct iovec *iov,
         gf_log ("cli", GF_LOG_DEBUG, "Received resp to top");
         ret = xdr_to_generic (*iov, &rsp, (xdrproc_t)xdr_gf_cli_rsp);
         if (ret < 0) {
-                gf_log ("", GF_LOG_ERROR, "Unable to decode response");
+                gf_log (((call_frame_t *) myframe)->this->name, GF_LOG_ERROR,
+                        "Failed to decode xdr response");
                 goto out;
         }
 
         if (rsp.op_ret) {
                 if (strcmp (rsp.op_errstr, ""))
-                        cli_out ("%s", rsp.op_errstr);
-                cli_out ("volume top unsuccessful");
+                        cli_err ("%s", rsp.op_errstr);
+                cli_err ("volume top unsuccessful");
                 ret = rsp.op_ret;
                 goto out;
         }
@@ -3945,7 +4986,6 @@ gf_cli3_1_top_volume_cbk (struct rpc_req *req, struct iovec *iov,
                 goto out;
         }
 
-#if (HAVE_LIB_XML)
         if (global_state->mode & GLUSTER_MODE_XML) {
                 ret = cli_xml_output_vol_top (dict, rsp.op_ret,
                                               rsp.op_errno,
@@ -3956,7 +4996,6 @@ gf_cli3_1_top_volume_cbk (struct rpc_req *req, struct iovec *iov,
                 }
                 goto out;
         }
-#endif
 
         ret = dict_get_int32 (dict, "count", &brick_count);
         if (ret)
@@ -3965,14 +5004,31 @@ gf_cli3_1_top_volume_cbk (struct rpc_req *req, struct iovec *iov,
         ret = dict_get_int32 (dict, key, (int32_t*)&top_op);
         if (ret)
                 goto out;
+
+        clear_stats = dict_get_str_boolean (dict, "clear-stats", _gf_false);
+
         while (i < brick_count) {
                 i++;
                 snprintf (brick, sizeof (brick), "%d-brick", i);
                 ret = dict_get_str (dict, brick, &bricks);
                 if (ret)
                         goto out;
-                ret = dict_get_str_boolean (dict, "nfs", _gf_false);
-                if (ret)
+
+                nfs = dict_get_str_boolean (dict, "nfs", _gf_false);
+
+                if (clear_stats) {
+                        memset (key, 0, sizeof (key));
+                        snprintf (key, sizeof (key), "%d-stats-cleared", i);
+                        ret = dict_get_int32 (dict, key, &stats_cleared);
+                        if (ret)
+                                goto out;
+                        cli_out (stats_cleared ? "Cleared stats for %s %s" :
+                                 "Failed to clear stats for %s %s",
+                                 nfs ? "NFS server on" : "brick", bricks);
+                        continue;
+                }
+
+                if (nfs)
                         cli_out ("NFS Server : %s", bricks);
                 else
                         cli_out ("Brick: %s", bricks);
@@ -4053,11 +5109,9 @@ gf_cli3_1_top_volume_cbk (struct rpc_req *req, struct iovec *iov,
                                 ret = dict_get_int32 (dict, key, (int32_t *)&time_usec);
                                 if (ret)
                                         goto out;
-                                tm    = localtime (&time_sec);
-                                if (!tm)
-                                        goto out;
-                                strftime (timestr, 256, "%Y-%m-%d %H:%M:%S", tm);
-                                snprintf (timestr + strlen (timestr), 256 - strlen (timestr),
+                                gf_time_fmt (timestr, sizeof timestr,
+                                             time_sec, gf_timefmt_FT);
+                                snprintf (timestr + strlen (timestr), sizeof timestr - strlen (timestr),
                                   ".%"GF_PRI_SUSECONDS, time_usec);
                                 if (strlen (filename) < VOL_TOP_PERF_FILENAME_DEF_WIDTH)
                                         cli_out ("%*"PRIu64" %-*s %-*s",
@@ -4089,13 +5143,12 @@ out:
         if (dict)
                 dict_unref (dict);
 
-        if (rsp.dict.dict_val)
-                free (rsp.dict.dict_val);
+        free (rsp.dict.dict_val);
         return ret;
 }
 
 int32_t
-gf_cli3_1_top_volume (call_frame_t *frame, xlator_t *this, void *data)
+gf_cli_top_volume (call_frame_t *frame, xlator_t *this, void *data)
 {
         int                        ret   = -1;
         gf_cli_req                 req   = {{0,}};
@@ -4109,32 +5162,20 @@ gf_cli3_1_top_volume (call_frame_t *frame, xlator_t *this, void *data)
                 goto out;
         dict = data;
 
-        ret = dict_allocate_and_serialize (dict,
-                                           &req.dict.dict_val,
-                                           (size_t *)&req.dict.dict_len);
-        if (ret < 0) {
-                gf_log (this->name, GF_LOG_ERROR,
-                        "failed to serialize the data");
-
-                goto out;
-        }
-
-
-        ret = cli_cmd_submit (&req, frame, cli_rpc_prog,
-                              GLUSTER_CLI_PROFILE_VOLUME, NULL,
-                              this, gf_cli3_1_top_volume_cbk,
-                              (xdrproc_t) xdr_gf_cli_req);
+        ret = cli_to_glusterd (&req, frame, gf_cli_top_volume_cbk,
+                               (xdrproc_t) xdr_gf_cli_req, dict,
+                               GLUSTER_CLI_PROFILE_VOLUME, this, cli_rpc_prog,
+                               NULL);
 
 out:
         gf_log ("cli", GF_LOG_DEBUG, "Returning %d", ret);
-        if (req.dict.dict_val)
-                GF_FREE (req.dict.dict_val);
+        GF_FREE (req.dict.dict_val);
         return ret;
 }
 
 
 int
-gf_cli3_1_getwd_cbk (struct rpc_req *req, struct iovec *iov,
+gf_cli_getwd_cbk (struct rpc_req *req, struct iovec *iov,
                        int count, void *myframe)
 {
         gf1_cli_getwd_rsp rsp   = {0,};
@@ -4145,8 +5186,15 @@ gf_cli3_1_getwd_cbk (struct rpc_req *req, struct iovec *iov,
         }
 
         ret = xdr_to_generic (*iov, &rsp, (xdrproc_t)xdr_gf1_cli_getwd_rsp);
-        if (ret < 0 || rsp.op_ret == -1) {
-                gf_log ("", GF_LOG_ERROR, "error");
+        if (ret < 0) {
+                gf_log (((call_frame_t *) myframe)->this->name, GF_LOG_ERROR,
+                        "Failed to decode xdr response");
+                goto out;
+        }
+
+        if (rsp.op_ret == -1) {
+                cli_err ("getwd failed");
+                ret = rsp.op_ret;
                 goto out;
         }
 
@@ -4162,7 +5210,7 @@ out:
 }
 
 int32_t
-gf_cli3_1_getwd (call_frame_t *frame, xlator_t *this, void *data)
+gf_cli_getwd (call_frame_t *frame, xlator_t *this, void *data)
 {
         int                      ret = -1;
         gf1_cli_getwd_req        req = {0,};
@@ -4175,7 +5223,7 @@ gf_cli3_1_getwd (call_frame_t *frame, xlator_t *this, void *data)
 
         ret = cli_cmd_submit (&req, frame, cli_rpc_prog,
                               GLUSTER_CLI_GETWD, NULL,
-                              this, gf_cli3_1_getwd_cbk,
+                              this, gf_cli_getwd_cbk,
                               (xdrproc_t) xdr_gf1_cli_getwd_req);
 
 out:
@@ -4278,7 +5326,7 @@ out:
 }
 
 void
-cli_print_volume_status_mem (dict_t *dict, gf_boolean_t nfs)
+cli_print_volume_status_mem (dict_t *dict, gf_boolean_t notbrick)
 {
         int             ret = -1;
         char            *volname = NULL;
@@ -4286,7 +5334,9 @@ cli_print_volume_status_mem (dict_t *dict, gf_boolean_t nfs)
         char            *path = NULL;
         int             online = -1;
         char            key[1024] = {0,};
-        int             brick_count = 0;
+        int             brick_index_max = -1;
+        int             other_count = 0;
+        int             index_max = 0;
         int             val = 0;
         int             i = 0;
 
@@ -4297,24 +5347,29 @@ cli_print_volume_status_mem (dict_t *dict, gf_boolean_t nfs)
                 goto out;
         cli_out ("Memory status for volume : %s", volname);
 
-        ret = dict_get_int32 (dict, "count", &brick_count);
+        ret = dict_get_int32 (dict, "brick-index-max", &brick_index_max);
         if (ret)
                 goto out;
+        ret = dict_get_int32 (dict, "other-count", &other_count);
+        if (ret)
+                goto out;
+
+        index_max = brick_index_max + other_count;
 
-        for (i = 0; i < brick_count; i++) {
+        for (i = 0; i <= index_max; i++) {
                 cli_out ("----------------------------------------------");
 
                 memset (key, 0, sizeof (key));
                 snprintf (key, sizeof (key), "brick%d.hostname", i);
                 ret = dict_get_str (dict, key, &hostname);
                 if (ret)
-                        goto out;
+                        continue;
                 memset (key, 0, sizeof (key));
                 snprintf (key, sizeof (key), "brick%d.path", i);
                 ret = dict_get_str (dict, key, &path);
                 if (ret)
-                        goto out;
-                if (nfs)
+                        continue;
+                if (notbrick)
                         cli_out ("%s : %s", hostname, path);
                 else
                         cli_out ("Brick : %s:%s", hostname, path);
@@ -4325,7 +5380,10 @@ cli_print_volume_status_mem (dict_t *dict, gf_boolean_t nfs)
                 if (ret)
                         goto out;
                 if (!online) {
-                        cli_out ("Brick is offline");
+                        if (notbrick)
+                                cli_out ("%s is offline", hostname);
+                        else
+                                cli_out ("Brick is offline");
                         continue;
                 }
 
@@ -4412,11 +5470,13 @@ out:
 }
 
 void
-cli_print_volume_status_clients (dict_t *dict, gf_boolean_t nfs)
+cli_print_volume_status_clients (dict_t *dict, gf_boolean_t notbrick)
 {
         int             ret = -1;
         char            *volname = NULL;
-        int             brick_count = 0;
+        int             brick_index_max = -1;
+        int             other_count = 0;
+        int             index_max = 0;
         char            *hostname = NULL;
         char            *path = NULL;
         int             online = -1;
@@ -4435,11 +5495,16 @@ cli_print_volume_status_clients (dict_t *dict, gf_boolean_t nfs)
                 goto out;
         cli_out ("Client connections for volume %s", volname);
 
-        ret = dict_get_int32 (dict, "count", &brick_count);
+        ret = dict_get_int32 (dict, "brick-index-max", &brick_index_max);
+        if (ret)
+                goto out;
+        ret = dict_get_int32 (dict, "other-count", &other_count);
         if (ret)
                 goto out;
 
-        for ( i = 0; i < brick_count; i++) {
+        index_max = brick_index_max + other_count;
+
+        for (i = 0; i <= index_max; i++) {
                 cli_out ("----------------------------------------------");
 
                 memset (key, 0, sizeof (key));
@@ -4453,7 +5518,7 @@ cli_print_volume_status_clients (dict_t *dict, gf_boolean_t nfs)
                 if (ret)
                         goto out;
 
-                if (nfs)
+                if (notbrick)
                         cli_out ("%s : %s", hostname, path);
                 else
                         cli_out ("Brick : %s:%s", hostname, path);
@@ -4464,7 +5529,10 @@ cli_print_volume_status_clients (dict_t *dict, gf_boolean_t nfs)
                 if (ret)
                         goto out;
                 if (!online) {
-                        cli_out ("Brick is offline");
+                        if (notbrick)
+                                cli_out ("%s is offline", hostname);
+                        else
+                                cli_out ("Brick is offline");
                         continue;
                 }
 
@@ -4475,6 +5543,9 @@ cli_print_volume_status_clients (dict_t *dict, gf_boolean_t nfs)
                         goto out;
 
                 cli_out ("Clients connected : %d", client_count);
+                if (client_count == 0)
+                        continue;
+
                 cli_out ("%-48s %15s %15s", "Hostname", "BytesRead",
                          "BytesWritten");
                 cli_out ("%-48s %15s %15s", "--------", "---------",
@@ -4657,11 +5728,13 @@ out:
 }
 
 void
-cli_print_volume_status_inode (dict_t *dict, gf_boolean_t nfs)
+cli_print_volume_status_inode (dict_t *dict, gf_boolean_t notbrick)
 {
         int             ret = -1;
         char            *volname = NULL;
-        int             brick_count = 0;
+        int             brick_index_max = -1;
+        int             other_count = 0;
+        int             index_max = 0;
         char            *hostname = NULL;
         char            *path = NULL;
         int             online = -1;
@@ -4677,11 +5750,16 @@ cli_print_volume_status_inode (dict_t *dict, gf_boolean_t nfs)
                 goto out;
         cli_out ("Inode tables for volume %s", volname);
 
-        ret = dict_get_int32 (dict, "count", &brick_count);
+        ret = dict_get_int32 (dict, "brick-index-max", &brick_index_max);
+        if (ret)
+                goto out;
+        ret = dict_get_int32 (dict, "other-count", &other_count);
         if (ret)
                 goto out;
 
-        for (i = 0; i < brick_count; i++) {
+        index_max = brick_index_max + other_count;
+
+        for ( i = 0; i <= index_max; i++) {
                 cli_out ("----------------------------------------------");
 
                 memset (key, 0, sizeof (key));
@@ -4694,7 +5772,7 @@ cli_print_volume_status_inode (dict_t *dict, gf_boolean_t nfs)
                 ret = dict_get_str (dict, key, &path);
                 if (ret)
                         goto out;
-                if (nfs)
+                if (notbrick)
                         cli_out ("%s : %s", hostname, path);
                 else
                         cli_out ("Brick : %s:%s", hostname, path);
@@ -4705,7 +5783,10 @@ cli_print_volume_status_inode (dict_t *dict, gf_boolean_t nfs)
                 if (ret)
                         goto out;
                 if (!online) {
-                        cli_out ("Brick is offline");
+                        if (notbrick)
+                                cli_out ("%s is offline", hostname);
+                        else
+                                cli_out ("Brick is offline");
                         continue;
                 }
 
@@ -4775,7 +5856,7 @@ cli_print_volume_status_fdtable (dict_t *dict, char *prefix)
         if (ret)
                 goto out;
         if (0 == openfds) {
-                cli_out ("No open fds");
+                cli_err ("No open fds");
                 goto out;
         }
 
@@ -4813,11 +5894,13 @@ out:
 }
 
 void
-cli_print_volume_status_fd (dict_t *dict, gf_boolean_t nfs)
+cli_print_volume_status_fd (dict_t *dict, gf_boolean_t notbrick)
 {
         int             ret = -1;
         char            *volname = NULL;
-        int             brick_count = 0;
+        int             brick_index_max = -1;
+        int             other_count = 0;
+        int             index_max = 0;
         char            *hostname = NULL;
         char            *path = NULL;
         int             online = -1;
@@ -4833,11 +5916,16 @@ cli_print_volume_status_fd (dict_t *dict, gf_boolean_t nfs)
                 goto out;
         cli_out ("FD tables for volume %s", volname);
 
-        ret = dict_get_int32 (dict, "count", &brick_count);
+        ret = dict_get_int32 (dict, "brick-index-max", &brick_index_max);
+        if (ret)
+                goto out;
+        ret = dict_get_int32 (dict, "other-count", &other_count);
         if (ret)
                 goto out;
 
-        for (i = 0; i < brick_count; i++) {
+        index_max = brick_index_max + other_count;
+
+        for (i = 0; i <= index_max; i++) {
                 cli_out ("----------------------------------------------");
 
                 memset (key, 0, sizeof (key));
@@ -4851,7 +5939,7 @@ cli_print_volume_status_fd (dict_t *dict, gf_boolean_t nfs)
                 if (ret)
                         goto out;
 
-                if (nfs)
+                if (notbrick)
                         cli_out ("%s : %s", hostname, path);
                 else
                         cli_out ("Brick : %s:%s", hostname, path);
@@ -4862,7 +5950,10 @@ cli_print_volume_status_fd (dict_t *dict, gf_boolean_t nfs)
                 if (ret)
                         goto out;
                 if (!online) {
-                        cli_out ("Brick is offline");
+                        if (notbrick)
+                                cli_out ("%s is offline", hostname);
+                        else
+                                cli_out ("Brick is offline");
                         continue;
                 }
 
@@ -5031,11 +6122,13 @@ cli_print_volume_status_call_stack (dict_t *dict, char *prefix)
 }
 
 void
-cli_print_volume_status_callpool (dict_t *dict, gf_boolean_t nfs)
+cli_print_volume_status_callpool (dict_t *dict, gf_boolean_t notbrick)
 {
         int             ret = -1;
         char            *volname = NULL;
-        int             brick_count = 0;
+        int             brick_index_max = -1;
+        int             other_count = 0;
+        int             index_max = 0;
         char            *hostname = NULL;
         char            *path = NULL;
         int             online = -1;
@@ -5051,11 +6144,16 @@ cli_print_volume_status_callpool (dict_t *dict, gf_boolean_t nfs)
                 goto out;
         cli_out ("Pending calls for volume %s", volname);
 
-        ret = dict_get_int32 (dict, "count", &brick_count);
+        ret = dict_get_int32 (dict, "brick-index-max", &brick_index_max);
+        if (ret)
+                goto out;
+        ret = dict_get_int32 (dict, "other-count", &other_count);
         if (ret)
                 goto out;
 
-        for (i = 0; i < brick_count; i++) {
+        index_max = brick_index_max + other_count;
+
+        for (i = 0; i <= index_max; i++) {
                 cli_out ("----------------------------------------------");
 
                 memset (key, 0, sizeof (key));
@@ -5069,7 +6167,7 @@ cli_print_volume_status_callpool (dict_t *dict, gf_boolean_t nfs)
                 if (ret)
                         goto out;
 
-                if (nfs)
+                if (notbrick)
                         cli_out ("%s : %s", hostname, path);
                 else
                         cli_out ("Brick : %s:%s", hostname, path);
@@ -5080,7 +6178,10 @@ cli_print_volume_status_callpool (dict_t *dict, gf_boolean_t nfs)
                 if (ret)
                         goto out;
                 if (!online) {
-                        cli_out ("Brick is offline");
+                        if (notbrick)
+                                cli_out ("%s is offline", hostname);
+                        else
+                                cli_out ("Brick is offline");
                         continue;
                 }
 
@@ -5109,38 +6210,194 @@ out:
         return;
 }
 
+static void
+cli_print_volume_status_tasks (dict_t *dict)
+{
+        int             ret         = -1;
+        int             i           = 0;
+        int             j           = 0;
+        int             count       = 0;
+        int             task_count  = 0;
+        int             status      = 0;
+        char           *op          = NULL;
+        char           *task_id_str = NULL;
+        char           *volname     = NULL;
+        char            key[1024]   = {0,};
+        char            task[1024]  = {0,};
+        char           *brick       = NULL;
+        char           *src_brick   = NULL;
+        char           *dest_brick  = NULL;
+
+        ret = dict_get_str (dict, "volname", &volname);
+        if (ret)
+                goto out;
+
+        ret = dict_get_int32 (dict, "tasks", &task_count);
+        if (ret) {
+                gf_log ("cli", GF_LOG_ERROR, "Failed to get tasks count");
+                return;
+        }
+
+        cli_out ("Task Status of Volume %s", volname);
+        cli_print_line (CLI_BRICK_STATUS_LINE_LEN);
+
+        if (task_count == 0) {
+                cli_out ("There are no active volume tasks");
+                cli_out (" ");
+                return;
+        }
+
+        for (i = 0; i < task_count; i++) {
+                memset (key, 0, sizeof (key));
+                snprintf (key, sizeof (key), "task%d.type", i);
+                ret = dict_get_str(dict, key, &op);
+                if (ret)
+                        return;
+                cli_out ("%-20s : %-20s", "Task", op);
+
+                memset (key, 0, sizeof (key));
+                snprintf (key, sizeof (key), "task%d.id", i);
+                ret = dict_get_str (dict, key, &task_id_str);
+                if (ret)
+                        return;
+                cli_out ("%-20s : %-20s", "ID", task_id_str);
+
+                memset (key, 0, sizeof (key));
+                snprintf (key, sizeof (key), "task%d.status", i);
+                ret = dict_get_int32 (dict, key, &status);
+                if (ret)
+                        return;
+
+                snprintf (task, sizeof (task), "task%d", i);
+
+                /*
+                   Replace brick only has two states - In progress and Complete
+                   Ref: xlators/mgmt/glusterd/src/glusterd-replace-brick.c
+                */
+
+                if (!strcmp (op, "Replace brick")) {
+                        if (status)
+                                status = GF_DEFRAG_STATUS_COMPLETE;
+                        else
+                                status = GF_DEFRAG_STATUS_STARTED;
+
+                        memset (key, 0, sizeof (key));
+                        snprintf (key, sizeof (key), "%s.src-brick", task);
+                        ret = dict_get_str (dict, key, &src_brick);
+                        if (ret)
+                                goto out;
+
+                        cli_out ("%-20s : %-20s", "Source Brick", src_brick);
+
+                        memset (key, 0, sizeof (key));
+                        snprintf (key, sizeof (key), "%s.dst-brick", task);
+                        ret = dict_get_str (dict, key, &dest_brick);
+                        if (ret)
+                                goto out;
+
+                        cli_out ("%-20s : %-20s", "Destination Brick",
+                                 dest_brick);
+
+                } else if (!strcmp (op, "Remove brick")) {
+                        memset (key, 0, sizeof (key));
+                        snprintf (key, sizeof (key), "%s.count", task);
+                        ret = dict_get_int32 (dict, key, &count);
+                        if (ret)
+                                goto out;
+
+                        cli_out ("%-20s", "Removed bricks:");
+
+                        for (j = 1; j <= count; j++) {
+                                memset (key, 0, sizeof (key));
+                                snprintf (key, sizeof (key),"%s.brick%d",
+                                          task, j);
+                                ret = dict_get_str (dict, key, &brick);
+                                if (ret)
+                                        goto out;
+
+                                cli_out ("%-20s", brick);
+                        }
+                }
+                cli_out ("%-20s : %-20s", "Status",
+                         cli_vol_task_status_str[status]);
+                cli_out (" ");
+        }
+
+out:
+        return;
+}
+
 static int
-gf_cli3_1_status_cbk (struct rpc_req *req, struct iovec *iov,
+gf_cli_status_cbk (struct rpc_req *req, struct iovec *iov,
                       int count, void *myframe)
 {
-        int                             ret            = -1;
-        int                             i              = 0;
-        int                             pid            = -1;
-        uint32_t                        cmd            = 0;
-        gf_boolean_t                    nfs            = _gf_false;
-        char                            key[1024]      = {0,};
-        char                           *hostname       = NULL;
-        char                           *path           = NULL;
-        char                           *volname        = NULL;
-        dict_t                         *dict           = NULL;
-        gf_cli_rsp                      rsp            = {0,};
-        cli_volume_status_t             status         = {0};
+        int                             ret             = -1;
+        int                             brick_index_max = -1;
+        int                             other_count     = 0;
+        int                             index_max       = 0;
+        int                             i               = 0;
+        int                             pid             = -1;
+        uint32_t                        cmd             = 0;
+        gf_boolean_t                    notbrick        = _gf_false;
+        char                            key[1024]       = {0,};
+        char                           *hostname        = NULL;
+        char                           *path            = NULL;
+        char                           *volname         = NULL;
+        dict_t                         *dict            = NULL;
+        gf_cli_rsp                      rsp             = {0,};
+        cli_volume_status_t             status          = {0};
+        cli_local_t                    *local           = NULL;
+        gf_boolean_t                    wipe_local      = _gf_false;
+        char                            msg[1024]       = {0,};
 
         if (req->rpc_status == -1)
                 goto out;
 
         ret = xdr_to_generic (*iov, &rsp, (xdrproc_t)xdr_gf_cli_rsp);
         if (ret < 0) {
-                gf_log ("cli", GF_LOG_ERROR, "Volume status response error");
+                gf_log (((call_frame_t *) myframe)->this->name, GF_LOG_ERROR,
+                        "Failed to decode xdr response");
                 goto out;
         }
 
         gf_log ("cli", GF_LOG_DEBUG, "Received response to status cmd");
 
-        if (rsp.op_ret && strcmp (rsp.op_errstr, ""))
-                cli_out ("%s", rsp.op_errstr);
-        else if (rsp.op_ret)
-                cli_out ("Unable to obtain volume status information.");
+        local = ((call_frame_t *)myframe)->local;
+        if (!local) {
+                local = cli_local_get ();
+                if (!local) {
+                        ret = -1;
+                        gf_log ("cli", GF_LOG_ERROR, "Failed to get local");
+                        goto out;
+                }
+                wipe_local = _gf_true;
+        }
+
+        if (rsp.op_ret) {
+                if (strcmp (rsp.op_errstr, ""))
+                        snprintf (msg, sizeof (msg), "%s", rsp.op_errstr);
+                else
+                        snprintf (msg, sizeof (msg), "Unable to obtain volume "
+                                  "status information.");
+
+                if (global_state->mode & GLUSTER_MODE_XML) {
+                        if (!local->all)
+                                cli_xml_output_str ("volStatus", msg,
+                                                    rsp.op_ret, rsp.op_errno,
+                                                    rsp.op_errstr);
+                        ret = 0;
+                        goto out;
+                }
+
+                cli_err ("%s", msg);
+                if (local && local->all) {
+                        ret = 0;
+                        cli_out (" ");
+                } else
+                        ret = -1;
+
+                goto out;
+        }
 
         dict = dict_new ();
         if (!dict)
@@ -5157,55 +6414,84 @@ gf_cli3_1_status_cbk (struct rpc_req *req, struct iovec *iov,
                 goto out;
 
         if ((cmd & GF_CLI_STATUS_ALL)) {
-                ((call_frame_t *)myframe)->local = dict;
-                ret = 0;
+                if (local && local->dict) {
+                        dict_ref (dict);
+                        ret = dict_set_static_ptr (local->dict, "rsp-dict", dict);
+                        ret = 0;
+                } else {
+                        gf_log ("cli", GF_LOG_ERROR, "local not found");
+                        ret = -1;
+                }
                 goto out;
         }
 
-        if (cmd & GF_CLI_STATUS_NFS)
-                nfs = _gf_true;
+        if ((cmd & GF_CLI_STATUS_NFS) || (cmd & GF_CLI_STATUS_SHD))
+                notbrick = _gf_true;
 
-        ret = dict_get_int32 (dict, "count", &count);
-        if (ret)
-                goto out;
-        if (count == 0) {
-                ret = -1;
-                goto out;
-        }
-
-#if (HAVE_LIB_XML)
         if (global_state->mode & GLUSTER_MODE_XML) {
-                ret = cli_xml_output_vol_status (dict, rsp.op_ret,
-                                                 rsp.op_errno, rsp.op_errstr);
-                if (ret) {
-                        gf_log ("cli", GF_LOG_ERROR,
-                                "Error outputting to xml");
+                if (!local->all) {
+                        ret = cli_xml_output_vol_status_begin (local,
+                                                               rsp.op_ret,
+                                                               rsp.op_errno,
+                                                               rsp.op_errstr);
+                        if (ret) {
+                                gf_log ("cli", GF_LOG_ERROR,
+                                        "Error outputting to xml");
+                                goto out;
+                        }
+                }
+                if (cmd & GF_CLI_STATUS_TASKS) {
+                        ret = cli_xml_output_vol_status_tasks_detail (local,
+                                                                      dict);
+                        if (ret) {
+                                gf_log ("cli", GF_LOG_ERROR,"Error outputting "
+                                        "to xml");
+                                goto out;
+                        }
+                } else {
+                        ret = cli_xml_output_vol_status (local, dict);
+                        if (ret) {
+                                gf_log ("cli", GF_LOG_ERROR,
+                                        "Error outputting to xml");
+                                goto out;
+                        }
+                }
+
+                if (!local->all) {
+                        ret = cli_xml_output_vol_status_end (local);
+                        if (ret) {
+                                gf_log ("cli", GF_LOG_ERROR,
+                                        "Error outputting to xml");
+                        }
                 }
                 goto out;
         }
-#endif
 
         status.brick = GF_CALLOC (1, PATH_MAX + 256, gf_common_mt_strdup);
 
         switch (cmd & GF_CLI_STATUS_MASK) {
                 case GF_CLI_STATUS_MEM:
-                        cli_print_volume_status_mem (dict, nfs);
+                        cli_print_volume_status_mem (dict, notbrick);
                         goto cont;
                         break;
                 case GF_CLI_STATUS_CLIENTS:
-                        cli_print_volume_status_clients (dict, nfs);
+                        cli_print_volume_status_clients (dict, notbrick);
                         goto cont;
                         break;
                 case GF_CLI_STATUS_INODE:
-                        cli_print_volume_status_inode (dict, nfs);
+                        cli_print_volume_status_inode (dict, notbrick);
                         goto cont;
                         break;
                 case GF_CLI_STATUS_FD:
-                        cli_print_volume_status_fd (dict, nfs);
+                        cli_print_volume_status_fd (dict, notbrick);
                         goto cont;
                         break;
                 case GF_CLI_STATUS_CALLPOOL:
-                        cli_print_volume_status_callpool (dict, nfs);
+                        cli_print_volume_status_callpool (dict, notbrick);
+                        goto cont;
+                        break;
+                case GF_CLI_STATUS_TASKS:
+                        cli_print_volume_status_tasks (dict);
                         goto cont;
                         break;
                 default:
@@ -5216,48 +6502,68 @@ gf_cli3_1_status_cbk (struct rpc_req *req, struct iovec *iov,
         if (ret)
                 goto out;
 
-        cli_out ("\nStatus of volume: %s", volname);
+        ret = dict_get_int32 (dict, "brick-index-max", &brick_index_max);
+        if (ret)
+                goto out;
+
+        ret = dict_get_int32 (dict, "other-count", &other_count);
+        if (ret)
+                goto out;
 
-        if ((cmd & GF_CLI_STATUS_DETAIL) == 0)
-                cli_out ("Brick\t\t\t\t\t\t\tPort\tOnline\tPid");
+        index_max = brick_index_max + other_count;
 
-        for (i = 0; i < count; i++) {
 
+        cli_out ("Status of volume: %s", volname);
+
+        if ((cmd & GF_CLI_STATUS_DETAIL) == 0) {
+                cli_out ("Gluster process\t\t\t\t\t\tPort\tOnline\tPid");
                 cli_print_line (CLI_BRICK_STATUS_LINE_LEN);
+        }
+
+        for (i = 0; i <= index_max; i++) {
+
 
                 memset (key, 0, sizeof (key));
                 snprintf (key, sizeof (key), "brick%d.hostname", i);
                 ret = dict_get_str (dict, key, &hostname);
                 if (ret)
-                        goto out;
+                        continue;
 
                 memset (key, 0, sizeof (key));
                 snprintf (key, sizeof (key), "brick%d.path", i);
                 ret = dict_get_str (dict, key, &path);
                 if (ret)
-                        goto out;
+                        continue;
 
+                /* Brick/not-brick is handled seperately here as all
+                 * types of nodes are contained in the default output
+                 */
                 memset (status.brick, 0, PATH_MAX + 255);
-                snprintf (status.brick, PATH_MAX + 255, "%s:%s",
-                          hostname, path);
+                if (!strcmp (hostname, "NFS Server") ||
+                    !strcmp (hostname, "Self-heal Daemon"))
+                        snprintf (status.brick, PATH_MAX + 255, "%s on %s",
+                                  hostname, path);
+                else
+                        snprintf (status.brick, PATH_MAX + 255, "Brick %s:%s",
+                                  hostname, path);
 
                 memset (key, 0, sizeof (key));
                 snprintf (key, sizeof (key), "brick%d.port", i);
                 ret = dict_get_int32 (dict, key, &(status.port));
                 if (ret)
-                        goto out;
+                        continue;
 
                 memset (key, 0, sizeof (key));
                 snprintf (key, sizeof (key), "brick%d.status", i);
                 ret = dict_get_int32 (dict, key, &(status.online));
                 if (ret)
-                        goto out;
+                        continue;
 
                 memset (key, 0, sizeof (key));
                 snprintf (key, sizeof (key), "brick%d.pid", i);
                 ret = dict_get_int32 (dict, key, &pid);
                 if (ret)
-                        goto out;
+                        continue;
                 if (pid == -1)
                         ret = gf_asprintf (&(status.pid_str), "%s", "N/A");
                 else
@@ -5270,26 +6576,34 @@ gf_cli3_1_status_cbk (struct rpc_req *req, struct iovec *iov,
                         ret = cli_get_detail_status (dict, i, &status);
                         if (ret)
                                 goto out;
+                        cli_print_line (CLI_BRICK_STATUS_LINE_LEN);
                         cli_print_detailed_status (&status);
 
                 } else {
                         cli_print_brick_status (&status);
                 }
         }
+        cli_out (" ");
 
+        if ((cmd & GF_CLI_STATUS_MASK) == GF_CLI_STATUS_NONE)
+                cli_print_volume_status_tasks (dict);
 cont:
         ret = rsp.op_ret;
 
 out:
-        if (status.brick)
-                GF_FREE (status.brick);
+        if (dict)
+                dict_unref (dict);
+        GF_FREE (status.brick);
+        if (local && wipe_local) {
+                cli_local_wipe (local);
+        }
 
         cli_cmd_broadcast_response (ret);
         return ret;
 }
 
 int32_t
-gf_cli3_1_status_volume (call_frame_t *frame, xlator_t *this,
+gf_cli_status_volume (call_frame_t *frame, xlator_t *this,
                          void *data)
 {
         gf_cli_req                      req  = {{0,}};
@@ -5301,20 +6615,10 @@ gf_cli3_1_status_volume (call_frame_t *frame, xlator_t *this,
 
         dict = data;
 
-        ret = dict_allocate_and_serialize (dict,
-                                           &req.dict.dict_val,
-                                           (size_t *)&req.dict.dict_len);
-        if (ret < 0) {
-                gf_log ("cli", GF_LOG_ERROR,
-                        "failed to serialize the data");
-
-                goto out;
-        }
-        ret = cli_cmd_submit (&req, frame, cli_rpc_prog,
-                              GLUSTER_CLI_STATUS_VOLUME, NULL,
-                              this, gf_cli3_1_status_cbk,
-                              (xdrproc_t)xdr_gf_cli_req);
-
+        ret = cli_to_glusterd (&req, frame, gf_cli_status_cbk,
+                               (xdrproc_t) xdr_gf_cli_req, dict,
+                               GLUSTER_CLI_STATUS_VOLUME, this, cli_rpc_prog,
+                               NULL);
  out:
         gf_log ("cli", GF_LOG_DEBUG, "Returning: %d", ret);
         return ret;
@@ -5329,29 +6633,32 @@ gf_cli_status_volume_all (call_frame_t *frame, xlator_t *this, void *data)
         uint32_t         cmd          = 0;
         char             key[1024]    = {0};
         char            *volname      = NULL;
-        dict_t          *vol_dict     = NULL;
+        void            *vol_dict     = NULL;
         dict_t          *dict         = NULL;
+        cli_local_t     *local        = NULL;
 
-        dict = (dict_t *)data;
-        ret = dict_get_uint32 (dict, "cmd", &cmd);
-        if (ret)
+        if (frame->local) {
+                local = frame->local;
+                local->all = _gf_true;
+        } else
                 goto out;
 
-        ret = gf_cli3_1_status_volume (frame, this, data);
+        ret = dict_get_uint32 (local->dict, "cmd", &cmd);
         if (ret)
                 goto out;
 
-        vol_dict =  (dict_t *)(frame->local);
 
-        ret = dict_get_int32 (vol_dict, "vol_count", &vol_count);
-        if (ret) {
-                cli_out ("Failed to get names of volumes");
+        ret = gf_cli_status_volume (frame, this, data);
+        if (ret)
                 goto out;
-        }
 
-        if (vol_count == 0) {
-                cli_out ("No volumes present");
-                ret = 0;
+        ret = dict_get_ptr (local->dict, "rsp-dict", &vol_dict);
+        if (ret)
+                goto out;
+
+        ret = dict_get_int32 ((dict_t *)vol_dict, "vol_count", &vol_count);
+        if (ret) {
+                cli_err ("Failed to get names of volumes");
                 goto out;
         }
 
@@ -5359,6 +6666,22 @@ gf_cli_status_volume_all (call_frame_t *frame, xlator_t *this, void *data)
         cmd &= ~GF_CLI_STATUS_ALL;
         cmd |= GF_CLI_STATUS_VOL;
 
+        if (global_state->mode & GLUSTER_MODE_XML) {
+                //TODO: Pass proper op_* values
+                ret = cli_xml_output_vol_status_begin (local, 0,0, NULL);
+                if (ret) {
+                        gf_log ("cli", GF_LOG_ERROR,
+                                "Error outputting to xml");
+                        goto out;
+                }
+        }
+
+        if (vol_count == 0 && !(global_state->mode & GLUSTER_MODE_XML)) {
+                cli_err ("No volumes present");
+                ret = 0;
+                goto out;
+        }
+
         for (i = 0; i < vol_count; i++) {
 
                 dict = dict_new ();
@@ -5371,7 +6694,7 @@ gf_cli_status_volume_all (call_frame_t *frame, xlator_t *this, void *data)
                 if (ret)
                         goto out;
 
-                ret = dict_set_dynstr (dict, "volname", volname);
+                ret = dict_set_str (dict, "volname", volname);
                 if (ret)
                         goto out;
 
@@ -5379,23 +6702,38 @@ gf_cli_status_volume_all (call_frame_t *frame, xlator_t *this, void *data)
                 if (ret)
                         goto out;
 
-                ret = gf_cli3_1_status_volume (frame, this, dict);
+                ret = gf_cli_status_volume (frame, this, dict);
                 if (ret)
                         goto out;
 
                 dict_unref (dict);
         }
 
+        if (global_state->mode & GLUSTER_MODE_XML) {
+                ret = cli_xml_output_vol_status_end (local);
+        }
+
  out:
         if (ret)
                 gf_log ("cli", GF_LOG_ERROR, "status all failed");
+
+        if (vol_dict)
+                dict_unref (vol_dict);
+
         if (ret && dict)
                 dict_unref (dict);
+
+        if (local)
+                cli_local_wipe (local);
+
+        if (frame)
+                frame->local = NULL;
+
         return ret;
 }
 
 static int
-gf_cli3_1_mount_cbk (struct rpc_req *req, struct iovec *iov,
+gf_cli_mount_cbk (struct rpc_req *req, struct iovec *iov,
                   int count, void *myframe)
 {
         gf1_cli_mount_rsp rsp   = {0,};
@@ -5407,7 +6745,8 @@ gf_cli3_1_mount_cbk (struct rpc_req *req, struct iovec *iov,
 
         ret = xdr_to_generic (*iov, &rsp, (xdrproc_t)xdr_gf1_cli_mount_rsp);
         if (ret < 0) {
-                gf_log ("", GF_LOG_ERROR, "error");
+                gf_log (((call_frame_t *) myframe)->this->name, GF_LOG_ERROR,
+                        "Failed to decode xdr response");
                 goto out;
         }
 
@@ -5418,9 +6757,9 @@ gf_cli3_1_mount_cbk (struct rpc_req *req, struct iovec *iov,
                 cli_out ("%s", rsp.path);
         } else {
                 /* weird sounding but easy to parse... */
-                cli_out ("%d : failed with this errno (%s)",
+                cli_err ("%d : failed with this errno (%s)",
                          rsp.op_errno, strerror (rsp.op_errno));
-                ret = 1;
+                ret = -1;
         }
 
 out:
@@ -5429,7 +6768,7 @@ out:
 }
 
 int32_t
-gf_cli3_1_mount (call_frame_t *frame, xlator_t *this, void *data)
+gf_cli_mount (call_frame_t *frame, xlator_t *this, void *data)
 {
         gf1_cli_mount_req  req  = {0,};
         int                ret  = -1;
@@ -5445,7 +6784,7 @@ gf_cli3_1_mount (call_frame_t *frame, xlator_t *this, void *data)
 
         req.label = label;
         ret = dict_allocate_and_serialize (dict, &req.dict.dict_val,
-                                           (size_t *)&req.dict.dict_len);
+                                           &req.dict.dict_len);
         if (ret) {
                 ret = -1;
                 goto out;
@@ -5453,7 +6792,7 @@ gf_cli3_1_mount (call_frame_t *frame, xlator_t *this, void *data)
 
         ret = cli_cmd_submit (&req, frame, cli_rpc_prog,
                               GLUSTER_CLI_MOUNT, NULL,
-                              this, gf_cli3_1_mount_cbk,
+                              this, gf_cli_mount_cbk,
                               (xdrproc_t)xdr_gf1_cli_mount_req);
 
 out:
@@ -5462,7 +6801,7 @@ out:
 }
 
 static int
-gf_cli3_1_umount_cbk (struct rpc_req *req, struct iovec *iov,
+gf_cli_umount_cbk (struct rpc_req *req, struct iovec *iov,
                    int count, void *myframe)
 {
         gf1_cli_umount_rsp rsp   = {0,};
@@ -5474,7 +6813,8 @@ gf_cli3_1_umount_cbk (struct rpc_req *req, struct iovec *iov,
 
         ret = xdr_to_generic (*iov, &rsp, (xdrproc_t)xdr_gf1_cli_umount_rsp);
         if (ret < 0) {
-                gf_log ("", GF_LOG_ERROR, "error");
+                gf_log (((call_frame_t *) myframe)->this->name, GF_LOG_ERROR,
+                        "Failed to decode xdr response");
                 goto out;
         }
 
@@ -5483,8 +6823,8 @@ gf_cli3_1_umount_cbk (struct rpc_req *req, struct iovec *iov,
         if (rsp.op_ret == 0)
                 ret = 0;
         else {
-                cli_out ("umount failed");
-                ret = 1;
+                cli_err ("umount failed");
+                ret = -1;
         }
 
 out:
@@ -5493,7 +6833,7 @@ out:
 }
 
 int32_t
-gf_cli3_1_umount (call_frame_t *frame, xlator_t *this, void *data)
+gf_cli_umount (call_frame_t *frame, xlator_t *this, void *data)
 {
         gf1_cli_umount_req  req  = {0,};
         int                ret  = -1;
@@ -5515,7 +6855,7 @@ gf_cli3_1_umount (call_frame_t *frame, xlator_t *this, void *data)
 
         ret = cli_cmd_submit (&req, frame, cli_rpc_prog,
                               GLUSTER_CLI_UMOUNT, NULL,
-                              this, gf_cli3_1_umount_cbk,
+                              this, gf_cli_umount_cbk,
                               (xdrproc_t)xdr_gf1_cli_umount_req);
 
  out:
@@ -5524,39 +6864,211 @@ gf_cli3_1_umount (call_frame_t *frame, xlator_t *this, void *data)
 }
 
 void
-cmd_heal_volume_brick_out (dict_t *dict, int brick)
+cmd_heal_volume_statistics_out (dict_t *dict, int  brick)
 {
+
         uint64_t        num_entries = 0;
         int             ret = 0;
         char            key[256] = {0};
         char            *hostname = NULL;
-        char            *path = NULL;
         uint64_t        i = 0;
+        uint64_t        healed_count = 0;
+        uint64_t        split_brain_count = 0;
+        uint64_t        heal_failed_count = 0;
+        char            *start_time_str = NULL;
+        char            *end_time_str = NULL;
+        char            *crawl_type = NULL;
+        int             progress = -1;
+
+        snprintf (key, sizeof key, "%d-hostname", brick);
+        ret = dict_get_str (dict, key, &hostname);
+        if (ret)
+                goto out;
+        cli_out ("------------------------------------------------");
+        cli_out ("\nCrawl statistics for brick no %d", brick);
+        cli_out ("Hostname of brick %s", hostname);
+
+        snprintf (key, sizeof key, "statistics-%d-count", brick);
+        ret = dict_get_uint64 (dict, key, &num_entries);
+        if (ret)
+                goto out;
+
+        for (i = 0; i < num_entries; i++)
+        {
+                snprintf (key, sizeof key, "statistics_crawl_type-%d-%"PRIu64,
+                          brick, i);
+                ret = dict_get_str (dict, key, &crawl_type);
+                if (ret)
+                        goto out;
+
+                snprintf (key, sizeof key, "statistics_healed_cnt-%d-%"PRIu64,
+                          brick,i);
+                ret = dict_get_uint64 (dict, key, &healed_count);
+                if (ret)
+                        goto out;
 
-        snprintf (key, sizeof (key), "%d-hostname", brick);
+                snprintf (key, sizeof key, "statistics_sb_cnt-%d-%"PRIu64,
+                          brick, i);
+                ret = dict_get_uint64 (dict, key, &split_brain_count);
+                if (ret)
+                        goto out;
+                snprintf (key, sizeof key, "statistics_heal_failed_cnt-%d-%"PRIu64,
+                          brick, i);
+                ret = dict_get_uint64 (dict, key, &heal_failed_count);
+                if (ret)
+                        goto out;
+                snprintf (key, sizeof key, "statistics_strt_time-%d-%"PRIu64,
+                          brick, i);
+                ret = dict_get_str (dict, key,  &start_time_str);
+                if (ret)
+                        goto out;
+                snprintf (key, sizeof key, "statistics_end_time-%d-%"PRIu64,
+                          brick, i);
+                ret = dict_get_str (dict, key, &end_time_str);
+                if (ret)
+                        goto out;
+                snprintf (key, sizeof key, "statistics_inprogress-%d-%"PRIu64,
+                          brick, i);
+                ret = dict_get_int32 (dict, key, &progress);
+                if (ret)
+                        goto out;
+
+                cli_out ("\nStarting time of crawl: %s", start_time_str);
+                if (progress == 1)
+                        cli_out ("Crawl is in progress");
+                else
+                        cli_out ("Ending time of crawl: %s", end_time_str);
+
+                cli_out ("Type of crawl: %s", crawl_type);
+                cli_out ("No. of entries healed: %"PRIu64,
+                         healed_count);
+                cli_out ("No. of entries in split-brain: %"PRIu64,
+                        split_brain_count);
+                cli_out ("No. of heal failed entries: %"PRIu64,
+                         heal_failed_count);
+
+        }
+
+
+out:
+        return;
+}
+
+void
+cmd_heal_volume_brick_out (dict_t *dict, int brick)
+{
+        uint64_t        num_entries = 0;
+        int             ret = 0;
+        char            key[256] = {0};
+        char           *hostname = NULL;
+        char           *path = NULL;
+        char           *status = NULL;
+        uint64_t        i = 0;
+        uint32_t        time = 0;
+        char            timestr[32] = {0};
+        char            *shd_status = NULL;
+
+        snprintf (key, sizeof key, "%d-hostname", brick);
         ret = dict_get_str (dict, key, &hostname);
         if (ret)
                 goto out;
-        snprintf (key, sizeof (key), "%d-path", brick);
+        snprintf (key, sizeof key, "%d-path", brick);
         ret = dict_get_str (dict, key, &path);
         if (ret)
                 goto out;
-        snprintf (key, sizeof (key), "%d-count", brick);
-        ret = dict_get_uint64 (dict, key, &num_entries);
-        cli_out ("\nEntries on %s:%s %"PRIu64, hostname, path, num_entries);
-        for (i = 0; i < num_entries; i++) {
-                snprintf (key, sizeof (key), "%d-%"PRIu64, brick, i);
-                ret = dict_get_str (dict, key, &path);
+        cli_out ("\nBrick %s:%s", hostname, path);
+
+        snprintf (key, sizeof key, "%d-status", brick);
+        ret = dict_get_str (dict, key, &status);
+        if (status && strlen (status))
+                cli_out ("Status: %s", status);
+
+        snprintf (key, sizeof key, "%d-shd-status",brick);
+        ret = dict_get_str (dict, key, &shd_status);
+
+        if(!shd_status)
+        {
+                snprintf (key, sizeof key, "%d-count", brick);
+                ret = dict_get_uint64 (dict, key, &num_entries);
+                cli_out ("Number of entries: %"PRIu64, num_entries);
+
+
+                for (i = 0; i < num_entries; i++) {
+                        snprintf (key, sizeof key, "%d-%"PRIu64, brick, i);
+                        ret = dict_get_str (dict, key, &path);
+                        if (ret)
+                                continue;
+                        time = 0;
+                        snprintf (key, sizeof key, "%d-%"PRIu64"-time",
+                                  brick, i);
+                        ret = dict_get_uint32 (dict, key, &time);
+                        if (!time) {
+                                cli_out ("%s", path);
+                        } else {
+                                gf_time_fmt (timestr, sizeof timestr,
+                                             time, gf_timefmt_FT);
+                                if (i == 0) {
+                                cli_out ("at                    path on brick");
+                                cli_out ("-----------------------------------");
+                                }
+                                cli_out ("%s %s", timestr, path);
+                        }
+                }
+        }
+
+out:
+        return;
+}
+
+
+void
+cmd_heal_volume_statistics_heal_count_out (dict_t *dict, int brick)
+{
+        uint64_t        num_entries = 0;
+        int             ret = 0;
+        char            key[256] = {0};
+        char           *hostname = NULL;
+        char           *path = NULL;
+        char           *status = NULL;
+        char            *shd_status = NULL;
+
+        snprintf (key, sizeof key, "%d-hostname", brick);
+        ret = dict_get_str (dict, key, &hostname);
+        if (ret)
+                goto out;
+        snprintf (key, sizeof key, "%d-path", brick);
+        ret = dict_get_str (dict, key, &path);
+        if (ret)
+                goto out;
+        cli_out ("\nBrick %s:%s", hostname, path);
+
+        snprintf (key, sizeof key, "%d-status", brick);
+        ret = dict_get_str (dict, key, &status);
+        if (status && strlen (status))
+                cli_out ("Status: %s", status);
+
+        snprintf (key, sizeof key, "%d-shd-status",brick);
+        ret = dict_get_str (dict, key, &shd_status);
+
+        if(!shd_status)
+        {
+                snprintf (key, sizeof key, "%d-hardlinks", brick);
+                ret = dict_get_uint64 (dict, key, &num_entries);
                 if (ret)
-                        continue;
-                cli_out ("%s", path);
+                        cli_out ("No gathered input for this brick");
+                else
+                        cli_out ("Number of entries: %"PRIu64, num_entries);
+
+
         }
+
 out:
         return;
 }
 
+
 int
-gf_cli3_1_heal_volume_cbk (struct rpc_req *req, struct iovec *iov,
+gf_cli_heal_volume_cbk (struct rpc_req *req, struct iovec *iov,
                              int count, void *myframe)
 {
         gf_cli_rsp              rsp   = {0,};
@@ -5569,30 +7081,32 @@ gf_cli3_1_heal_volume_cbk (struct rpc_req *req, struct iovec *iov,
         int                     brick_count = 0;
         int                     i = 0;
         gf_xl_afr_op_t          heal_op = GF_AFR_OP_INVALID;
+        char                    *operation = NULL;
+        char                    *substr = NULL;
+        char                    *heal_op_str = NULL;
 
         if (-1 == req->rpc_status) {
                 goto out;
         }
 
+        frame = myframe;
+
         ret = xdr_to_generic (*iov, &rsp, (xdrproc_t)xdr_gf_cli_rsp);
         if (ret < 0) {
-                gf_log ("", GF_LOG_ERROR, "error");
+                gf_log (frame->this->name, GF_LOG_ERROR,
+                        "Failed to decode xdr response");
                 goto out;
         }
 
-        frame = myframe;
-
-        if (frame) {
+        if (frame)
                 local = frame->local;
-                frame->local = NULL;
-        }
 
         if (local) {
                 input_dict = local->dict;
                 ret = dict_get_int32 (input_dict, "heal-op",
                                       (int32_t*)&heal_op);
         }
-
+//TODO: Proper XML output
 //#if (HAVE_LIB_XML)
 //        if (global_state->mode & GLUSTER_MODE_XML) {
 //                ret = cli_xml_output_dict ("volHeal", dict, rsp.op_ret,
@@ -5606,30 +7120,75 @@ gf_cli3_1_heal_volume_cbk (struct rpc_req *req, struct iovec *iov,
 
         ret = dict_get_str (input_dict, "volname", &volname);
         if (ret) {
-                gf_log (THIS->name, GF_LOG_ERROR, "failed to get volname");
+                gf_log (frame->this->name, GF_LOG_ERROR, "failed to get volname");
                 goto out;
         }
 
         gf_log ("cli", GF_LOG_INFO, "Received resp to heal volume");
 
-        if (rsp.op_ret && strcmp (rsp.op_errstr, ""))
-                cli_out ("%s", rsp.op_errstr);
-        else
-                cli_out ("Starting heal on volume %s has been %s", volname,
-                        (rsp.op_ret) ? "unsuccessful": "successful");
+        switch (heal_op) {
+                case    GF_AFR_OP_HEAL_INDEX:
+                        heal_op_str = "to perform index self heal";
+                        break;
+                case    GF_AFR_OP_HEAL_FULL:
+                        heal_op_str = "to perform full self heal";
+                        break;
+                case    GF_AFR_OP_INDEX_SUMMARY:
+                        heal_op_str = "list of entries to be healed";
+                        break;
+                case    GF_AFR_OP_HEALED_FILES:
+                        heal_op_str = "list of healed entries";
+                        break;
+                case    GF_AFR_OP_HEAL_FAILED_FILES:
+                        heal_op_str = "list of heal failed entries";
+                        break;
+                case    GF_AFR_OP_SPLIT_BRAIN_FILES:
+                        heal_op_str = "list of split brain entries";
+                        break;
+                case    GF_AFR_OP_STATISTICS:
+                        heal_op_str =  "crawl statistics";
+                        break;
+                case    GF_AFR_OP_STATISTICS_HEAL_COUNT:
+                        heal_op_str = "count of entries to be healed";
+                        break;
+                case    GF_AFR_OP_STATISTICS_HEAL_COUNT_PER_REPLICA:
+                        heal_op_str = "count of entries to be healed per replica";
+                        break;
+                case    GF_AFR_OP_INVALID:
+                        heal_op_str = "invalid heal op";
+                        break;
+        }
+
+        if ((heal_op == GF_AFR_OP_HEAL_FULL) ||
+            (heal_op == GF_AFR_OP_HEAL_INDEX)) {
+                operation = "Launching heal operation";
+                substr = "\nUse heal info commands to check status";
+        } else {
+                operation = "Gathering";
+                substr = "";
+        }
 
         if (rsp.op_ret) {
+                if (strcmp (rsp.op_errstr, "")) {
+                        cli_err ("%s", rsp.op_errstr);
+                } else {
+                        cli_err ("%s %s on volume %s has been unsuccessful",
+                                 operation, heal_op_str, volname);
+                }
+
                 ret = rsp.op_ret;
                 goto out;
+        } else {
+                cli_out ("%s %s on volume %s has been successful %s", operation,
+                         heal_op_str, volname, substr);
         }
 
+        ret = rsp.op_ret;
         if ((heal_op == GF_AFR_OP_HEAL_FULL) ||
-            (heal_op == GF_AFR_OP_HEAL_INDEX)) {
-                ret = 0;
+            (heal_op == GF_AFR_OP_HEAL_INDEX))
                 goto out;
-        }
-        dict = dict_new ();
 
+        dict = dict_new ();
         if (!dict) {
                 ret = -1;
                 goto out;
@@ -5651,32 +7210,48 @@ gf_cli3_1_heal_volume_cbk (struct rpc_req *req, struct iovec *iov,
                 goto out;
 
         if (!brick_count) {
-                cli_out ("All bricks of volume %s are down.", volname);
+                cli_err ("All bricks of volume %s are down.", volname);
                 goto out;
         }
 
-        for (i = 0; i < brick_count; i++)
-                cmd_heal_volume_brick_out (dict, i);
+        switch (heal_op) {
+                case GF_AFR_OP_STATISTICS:
+                        for (i = 0; i < brick_count; i++)
+                                cmd_heal_volume_statistics_out (dict, i);
+                        break;
+                case GF_AFR_OP_STATISTICS_HEAL_COUNT:
+                case GF_AFR_OP_STATISTICS_HEAL_COUNT_PER_REPLICA:
+                        for (i = 0; i < brick_count; i++)
+                                cmd_heal_volume_statistics_heal_count_out (dict,
+                                                                           i);
+                        break;
+                case GF_AFR_OP_INDEX_SUMMARY:
+                case GF_AFR_OP_HEALED_FILES:
+                case GF_AFR_OP_HEAL_FAILED_FILES:
+                case GF_AFR_OP_SPLIT_BRAIN_FILES:
+                        for (i = 0; i < brick_count; i++)
+                                cmd_heal_volume_brick_out (dict, i);
+                        break;
+                default:
+                        break;
+        }
+
         ret = rsp.op_ret;
 
 out:
         cli_cmd_broadcast_response (ret);
-        if (local)
-                cli_local_wipe (local);
-        if (rsp.op_errstr)
-                free (rsp.op_errstr);
+        free (rsp.op_errstr);
         if (dict)
                 dict_unref (dict);
         return ret;
 }
 
 int32_t
-gf_cli3_1_heal_volume (call_frame_t *frame, xlator_t *this,
+gf_cli_heal_volume (call_frame_t *frame, xlator_t *this,
                          void *data)
 {
         gf_cli_req              req = {{0,}};
         int                     ret = 0;
-        cli_local_t             *local = NULL;
         dict_t                  *dict = NULL;
 
         if (!frame || !this ||  !data) {
@@ -5685,39 +7260,22 @@ gf_cli3_1_heal_volume (call_frame_t *frame, xlator_t *this,
         }
 
         dict = data;
-        local = cli_local_get ();
-
-        if (local) {
-                local->dict = dict_ref (dict);
-                frame->local = local;
-        }
 
-        ret = dict_allocate_and_serialize (dict,
-                                           &req.dict.dict_val,
-                                           (size_t *)&req.dict.dict_len);
-        if (ret < 0) {
-                gf_log (this->name, GF_LOG_ERROR,
-                        "failed to serialize the data");
-
-                goto out;
-        }
-
-        ret = cli_cmd_submit (&req, frame, cli_rpc_prog,
-                              GLUSTER_CLI_HEAL_VOLUME, NULL,
-                              this, gf_cli3_1_heal_volume_cbk,
-                              (xdrproc_t) xdr_gf_cli_req);
+        ret = cli_to_glusterd (&req, frame, gf_cli_heal_volume_cbk,
+                               (xdrproc_t) xdr_gf_cli_req, dict,
+                               GLUSTER_CLI_HEAL_VOLUME, this, cli_rpc_prog,
+                               NULL);
 
 out:
         gf_log ("cli", GF_LOG_DEBUG, "Returning %d", ret);
 
-        if (req.dict.dict_val)
-                GF_FREE (req.dict.dict_val);
+        GF_FREE (req.dict.dict_val);
 
         return ret;
 }
 
 int32_t
-gf_cli3_1_statedump_volume_cbk (struct rpc_req *req, struct iovec *iov,
+gf_cli_statedump_volume_cbk (struct rpc_req *req, struct iovec *iov,
                                 int count, void *myframe)
 {
         gf_cli_rsp                      rsp = {0,};
@@ -5729,16 +7287,16 @@ gf_cli3_1_statedump_volume_cbk (struct rpc_req *req, struct iovec *iov,
         ret = xdr_to_generic (*iov, &rsp,
                               (xdrproc_t)xdr_gf_cli_rsp);
         if (ret < 0) {
-                gf_log (THIS->name, GF_LOG_ERROR, "XDR decoding failed");
+                gf_log (((call_frame_t *) myframe)->this->name, GF_LOG_ERROR,
+                        "Failed to decode xdr response");
                 goto out;
         }
-        gf_log ("cli", GF_LOG_DEBUG, "Recieved response to statedump");
+        gf_log ("cli", GF_LOG_DEBUG, "Received response to statedump");
         if (rsp.op_ret)
                 snprintf (msg, sizeof(msg), "%s", rsp.op_errstr);
         else
                 snprintf (msg, sizeof (msg), "Volume statedump successful");
 
-#if (HAVE_LIB_XML)
         if (global_state->mode & GLUSTER_MODE_XML) {
                 ret = cli_xml_output_str ("volStatedump", msg, rsp.op_ret,
                                           rsp.op_errno, rsp.op_errstr);
@@ -5747,9 +7305,11 @@ gf_cli3_1_statedump_volume_cbk (struct rpc_req *req, struct iovec *iov,
                                 "Error outputting to xml");
                 goto out;
         }
-#endif
 
-        cli_out ("%s", msg);
+        if (rsp.op_ret)
+                cli_err ("volume statedump: failed: %s", msg);
+        else
+                cli_out ("volume statedump: success");
         ret = rsp.op_ret;
 
 out:
@@ -5758,7 +7318,7 @@ out:
 }
 
 int32_t
-gf_cli3_1_statedump_volume (call_frame_t *frame, xlator_t *this,
+gf_cli_statedump_volume (call_frame_t *frame, xlator_t *this,
                             void *data)
 {
         gf_cli_req                      req = {{0,}};
@@ -5770,33 +7330,20 @@ gf_cli3_1_statedump_volume (call_frame_t *frame, xlator_t *this,
 
         options = data;
 
-        ret = dict_allocate_and_serialize (options,
-                                           &req.dict.dict_val,
-                                           (size_t *)&req.dict.dict_len);
-        if (ret < 0) {
-                gf_log (this->name, GF_LOG_ERROR,
-                        "failed to serialize the data");
-
-                goto out;
-        }
-
-        ret = cli_cmd_submit (&req, frame, cli_rpc_prog,
-                              GLUSTER_CLI_STATEDUMP_VOLUME, NULL,
-                              this, gf_cli3_1_statedump_volume_cbk,
-                              (xdrproc_t)xdr_gf_cli_req);
+        ret = cli_to_glusterd (&req, frame, gf_cli_statedump_volume_cbk,
+                               (xdrproc_t) xdr_gf_cli_req, options,
+                               GLUSTER_CLI_STATEDUMP_VOLUME, this, cli_rpc_prog,
+                               NULL);
 
 out:
-        if (options)
-                dict_destroy (options);
         gf_log ("cli", GF_LOG_DEBUG, "Returning %d", ret);
 
-        if (req.dict.dict_val)
-                GF_FREE (req.dict.dict_val);
+        GF_FREE (req.dict.dict_val);
         return ret;
 }
 
 int32_t
-gf_cli3_1_list_volume_cbk (struct rpc_req *req, struct iovec *iov,
+gf_cli_list_volume_cbk (struct rpc_req *req, struct iovec *iov,
                                 int count, void *myframe)
 {
         int             ret = -1;
@@ -5812,7 +7359,8 @@ gf_cli3_1_list_volume_cbk (struct rpc_req *req, struct iovec *iov,
         ret = xdr_to_generic (*iov, &rsp,
                               (xdrproc_t)xdr_gf_cli_rsp);
         if (ret < 0) {
-                gf_log (THIS->name, GF_LOG_ERROR, "XDR decoding failed");
+                gf_log (((call_frame_t *) myframe)->this->name, GF_LOG_ERROR,
+                        "Failed to decode xdr response");
                 goto out;
         }
 
@@ -5828,7 +7376,6 @@ gf_cli3_1_list_volume_cbk (struct rpc_req *req, struct iovec *iov,
                 goto out;
         }
 
-#if (HAVE_LIB_XML)
         if (global_state->mode & GLUSTER_MODE_XML) {
                 ret = cli_xml_output_vol_list (dict, rsp.op_ret, rsp.op_errno,
                                                rsp.op_errstr);
@@ -5837,28 +7384,25 @@ gf_cli3_1_list_volume_cbk (struct rpc_req *req, struct iovec *iov,
                                 "Error outputting to xml");
                 goto out;
         }
-#endif
+
         if (rsp.op_ret)
-                cli_out ("%s", rsp.op_errstr);
+                cli_err ("%s", rsp.op_errstr);
         else {
                 ret = dict_get_int32 (dict, "count", &vol_count);
                 if (ret)
                         goto out;
 
                 if (vol_count == 0) {
-                        cli_out ("No volumes present in cluster");
+                        cli_err ("No volumes present in cluster");
                         goto out;
                 }
-                cli_out ("%d %s present in cluster", vol_count,
-                         ((vol_count == 1) ? "volume" : "volumes"));
-
                 for (i = 0; i < vol_count; i++) {
                         memset (key, 0, sizeof (key));
                         snprintf (key, sizeof (key), "volume%d", i);
                         ret = dict_get_str (dict, key, &volname);
                         if (ret)
                                 goto out;
-                        cli_out ("\t%d. %s", i+1, volname);
+                        cli_out ("%s", volname);
                 }
         }
 
@@ -5870,7 +7414,7 @@ out:
 }
 
 int32_t
-gf_cli3_1_list_volume (call_frame_t *frame, xlator_t *this, void *data)
+gf_cli_list_volume (call_frame_t *frame, xlator_t *this, void *data)
 {
         int             ret = -1;
         gf_cli_req      req = {{0,}};
@@ -5880,7 +7424,7 @@ gf_cli3_1_list_volume (call_frame_t *frame, xlator_t *this, void *data)
 
         ret = cli_cmd_submit (&req, frame, cli_rpc_prog,
                               GLUSTER_CLI_LIST_VOLUME, NULL,
-                              this, gf_cli3_1_list_volume_cbk,
+                              this, gf_cli_list_volume_cbk,
                               (xdrproc_t)xdr_gf_cli_req);
 
 out:
@@ -5889,7 +7433,7 @@ out:
 }
 
 int32_t
-gf_cli3_1_clearlocks_volume_cbk (struct rpc_req *req, struct iovec *iov,
+gf_cli_clearlocks_volume_cbk (struct rpc_req *req, struct iovec *iov,
                                   int count, void *myframe)
 {
         gf_cli_rsp                      rsp = {0,};
@@ -5903,19 +7447,19 @@ gf_cli3_1_clearlocks_volume_cbk (struct rpc_req *req, struct iovec *iov,
         ret = xdr_to_generic (*iov, &rsp,
                               (xdrproc_t)xdr_gf_cli_rsp);
         if (ret < 0) {
-
-                gf_log ("cli", GF_LOG_ERROR, "XDR decoding failed");
+                gf_log (((call_frame_t *) myframe)->this->name, GF_LOG_ERROR,
+                        "Failed to decode xdr response");
                 goto out;
         }
         gf_log ("cli", GF_LOG_DEBUG, "Received response to clear-locks");
 
         if (rsp.op_ret) {
-                cli_out ("Volume clear-locks unsuccessful");
-                cli_out ("%s", rsp.op_errstr);
+                cli_err ("Volume clear-locks unsuccessful");
+                cli_err ("%s", rsp.op_errstr);
 
         } else {
                 if (!rsp.dict.dict_len) {
-                        cli_out ("Possibly no locks cleared");
+                        cli_err ("Possibly no locks cleared");
                         ret = 0;
                         goto out;
                 }
@@ -5958,12 +7502,14 @@ gf_cli3_1_clearlocks_volume_cbk (struct rpc_req *req, struct iovec *iov,
         ret = rsp.op_ret;
 
 out:
+        if (dict)
+                dict_unref (dict);
         cli_cmd_broadcast_response (ret);
         return ret;
 }
 
 int32_t
-gf_cli3_1_clearlocks_volume (call_frame_t *frame, xlator_t *this,
+gf_cli_clearlocks_volume (call_frame_t *frame, xlator_t *this,
                              void *data)
 {
         gf_cli_req                      req = {{0,}};
@@ -5975,67 +7521,1147 @@ gf_cli3_1_clearlocks_volume (call_frame_t *frame, xlator_t *this,
 
         options = data;
 
-        ret = dict_allocate_and_serialize (options,
-                                           &req.dict.dict_val,
-                                           (size_t *)&req.dict.dict_len);
+        ret = cli_to_glusterd (&req, frame, gf_cli_clearlocks_volume_cbk,
+                               (xdrproc_t) xdr_gf_cli_req, options,
+                               GLUSTER_CLI_CLRLOCKS_VOLUME, this, cli_rpc_prog,
+                               NULL);
+out:
+        gf_log ("cli", GF_LOG_DEBUG, "Returning %d", ret);
+
+        GF_FREE (req.dict.dict_val);
+        return ret;
+}
+
+int32_t
+cli_snapshot_remove_reply (gf_cli_rsp *rsp, dict_t *dict, call_frame_t *frame)
+{
+        int32_t       ret        = -1;
+        char         *snap_name  = NULL;
+
+        GF_ASSERT (rsp);
+        GF_ASSERT (dict);
+        GF_ASSERT (frame);
+
+        if (rsp->op_ret) {
+                cli_err("snapshot delete: failed: %s",
+                        rsp->op_errstr ? rsp->op_errstr :
+                        "Please check log file for details");
+                ret = rsp->op_ret;
+                goto out;
+        }
+
+        ret = dict_get_str (dict, "snapname", &snap_name);
+        if (ret) {
+                gf_log ("cli", GF_LOG_ERROR, "Failed to get snapname");
+                goto out;
+        }
+
+        cli_out ("snapshot delete: %s: snap removed successfully",
+                 snap_name);
+        ret = 0;
+
+out:
+        return ret;
+}
+
+int
+cli_snapshot_config_display (dict_t *dict, gf_cli_rsp *rsp)
+{
+        char                buf[PATH_MAX]        = "";
+        char               *volname              = NULL;
+        int                 ret                  = -1;
+        int                 config_command       = 0;
+        uint64_t            value                = 0;
+        uint64_t            hard_limit           = 0;
+        uint64_t            soft_limit           = 0;
+        uint64_t            i                    = 0;
+        uint64_t            voldisplaycount      = 0;
+
+        GF_ASSERT (dict);
+        GF_ASSERT (rsp);
+
+        if (rsp->op_ret) {
+                cli_err ("Snapshot Config : failed: %s",
+                         rsp->op_errstr ? rsp->op_errstr :
+                         "Please check log file for details");
+                ret = rsp->op_ret;
+                goto out;
+        }
+
+        ret = dict_get_int32 (dict, "config-command", &config_command);
+        if (ret) {
+                gf_log ("cli", GF_LOG_ERROR, "Could not fetch config type");
+                goto out;
+        }
+
+        ret = dict_get_str (dict, "volname", &volname);
+        /* Ignore the error, as volname is optional */
+
+        if (!volname) {
+                volname = "System";
+        }
+
+        ret = dict_get_uint64 (dict, "snap-max-hard-limit", &hard_limit);
+        /* Ignore the error, as the key specified is optional */
+        ret = dict_get_uint64 (dict, "snap-max-soft-limit", &soft_limit);
+
+        if (!hard_limit && !soft_limit
+                        && config_command != GF_SNAP_CONFIG_DISPLAY) {
+                ret = -1;
+                gf_log(THIS->name, GF_LOG_ERROR,
+                       "Could not fetch config-key");
+                goto out;
+        }
+
+        switch (config_command) {
+        case GF_SNAP_CONFIG_TYPE_SET:
+                if (hard_limit && soft_limit) {
+                        cli_out ("snapshot config: snap-max-hard-limit "
+                                "& snap-max-soft-limit for system set "
+                                 "successfully");
+                } else if (hard_limit){
+                        cli_out ("snapshot config: %s "
+                                 "for snap-max-hard-limit set successfully",
+                                 volname);
+                } else if (soft_limit) {
+                        cli_out ("snapshot config: %s "
+                                 "for snap-max-soft-limit set successfully",
+                                 volname);
+                }
+                break;
+
+        case GF_SNAP_CONFIG_DISPLAY :
+                cli_out ("\nSnapshot System Configuration:");
+                ret = dict_get_uint64 (dict, "snap-max-hard-limit",
+                                       &value);
+                if (ret) {
+                         gf_log ("cli", GF_LOG_ERROR, "Could not fetch "
+                                "snap_max_hard_limit for %s", volname);
+                         ret = -1;
+                         goto out;
+                }
+                cli_out ("snap-max-hard-limit : %"PRIu64, value);
+
+                ret = dict_get_uint64 (dict, "snap-max-soft-limit",
+                                       &soft_limit);
+                if (ret) {
+                        gf_log ("cli", GF_LOG_ERROR, "Could not fetch "
+                               "snap-max-soft-limit for %s", volname);
+                        ret = -1;
+                        goto out;
+                }
+                cli_out ("snap-max-soft-limit : %"PRIu64"%%\n",
+                         soft_limit);
+
+                cli_out ("Snapshot Volume Configuration:");
+
+                ret = dict_get_uint64 (dict, "voldisplaycount",
+                                       &voldisplaycount);
+                if (ret) {
+                        gf_log ("cli", GF_LOG_ERROR,
+                               "Could not fetch voldisplaycount");
+                        ret = -1;
+                        goto out;
+                }
+
+                for (i = 0; i < voldisplaycount; i++) {
+                        snprintf (buf, sizeof(buf), "volume%ld-volname", i);
+                        ret = dict_get_str (dict, buf, &volname);
+                        if (ret) {
+                                gf_log ("cli", GF_LOG_ERROR, "Could not fetch "
+                                       " %s", buf);
+                                ret = -1;
+                                goto out;
+                        }
+                        cli_out ("\nVolume : %s", volname);
+
+                        snprintf (buf, sizeof(buf),
+                                  "volume%ld-snap-max-hard-limit", i);
+                        ret = dict_get_uint64 (dict, buf, &value);
+                        if (ret) {
+                                gf_log ("cli", GF_LOG_ERROR, "Could not fetch "
+                                       " %s", buf);
+                                ret = -1;
+                                goto out;
+                        }
+                        cli_out ("snap-max-hard-limit : %"PRIu64, value);
+
+                        snprintf (buf, sizeof(buf),
+                                  "volume%ld-active-hard-limit", i);
+                        ret = dict_get_uint64 (dict, buf, &value);
+                        if (ret) {
+                                 gf_log ("cli", GF_LOG_ERROR, "Could not fetch"
+                                        " effective snap_max_hard_limit for "
+                                        "%s", volname);
+                                ret = -1;
+                                goto out;
+                        }
+                        cli_out ("Effective snap-max-hard-limit : %"PRIu64,
+                                 value);
+
+                        snprintf (buf, sizeof(buf),
+                                      "volume%ld-snap-max-soft-limit", i);
+                        ret = dict_get_uint64 (dict, buf, &value);
+                        if (ret) {
+                                gf_log ("cli", GF_LOG_ERROR, "Could not fetch "
+                                       " %s", buf);
+                                ret = -1;
+                                goto out;
+                        }
+                        cli_out ("Effective snap-max-soft-limit : %"PRIu64" "
+                                 "(%"PRIu64"%%)", value, soft_limit);
+                }
+                break;
+        default :
+                break;
+        }
+
+        ret = 0;
+out:
+        return ret;
+}
+
+/* This function is used to print the volume related information
+ * of a snap.
+ *
+ * arg - 0, dict       : Response Dictionary.
+ * arg - 1, prefix str : snaplist.snap{0..}.vol{0..}.*
+ */
+int
+cli_get_each_volinfo_in_snap (dict_t *dict, char *keyprefix,
+                              gf_boolean_t snap_driven) {
+        char             key[PATH_MAX]  =  "";
+        char            *get_buffer     =  NULL;
+        int              value          =  0;
+        int              ret            = -1;
+        char             indent[5]      = "\t";
+        char            *volname        =  NULL;
+
+        GF_ASSERT (dict);
+        GF_ASSERT (keyprefix);
+
+        if (snap_driven) {
+                ret = snprintf (key, sizeof (key), "%s.volname", keyprefix);
+                if (ret < 0) {
+                        goto out;
+                }
+
+                ret = dict_get_str (dict, key, &get_buffer);
+                if (ret) {
+                        gf_log ("cli", GF_LOG_ERROR, "Failed to get %s", key);
+                        goto out;
+                }
+                cli_out ("%s" INDENT_MAIN_HEAD "%s", indent,
+                        "Snap Volume Name", ":", get_buffer);
+
+                ret = snprintf (key, sizeof (key),
+                                "%s.origin-volname", keyprefix);
+                if (ret < 0) {
+                        goto out;
+                }
+
+                ret = dict_get_str (dict, key, &volname);
+                if (ret) {
+                        gf_log ("cli", GF_LOG_WARNING, "Failed to get %s", key);
+                        cli_out ("%-12s", "Origin:");
+                }
+                cli_out ("%s" INDENT_MAIN_HEAD "%s", indent,
+                        "Origin Volume name", ":", volname);
+
+
+                ret = snprintf (key, sizeof (key), "%s.snapcount",
+                                keyprefix);
+                if (ret < 0) {
+                        goto out;
+                }
+
+                ret = dict_get_int32 (dict, key, &value);
+                if (ret) {
+                        gf_log ("cli", GF_LOG_ERROR, "Failed to get %s", key);
+                        goto out;
+                }
+                cli_out ("%s%s %s      %s %d", indent, "Snaps taken for",
+                        volname, ":", value);
+
+                ret = snprintf (key, sizeof (key), "%s.snaps-available",
+                                keyprefix);
+                if (ret < 0) {
+                        goto out;
+                }
+
+                ret = dict_get_int32 (dict, key, &value);
+                if (ret) {
+                        gf_log ("cli", GF_LOG_ERROR, "Failed to get %s", key);
+                        goto out;
+                }
+                cli_out ("%s%s %s  %s %d", indent, "Snaps available for",
+                         volname, ":", value);
+        }
+
+
+        ret = snprintf (key, sizeof (key), "%s.vol-status", keyprefix);
         if (ret < 0) {
-                gf_log ("cli", GF_LOG_ERROR,
-                        "failed to serialize the data");
+                goto out;
+        }
 
+        ret = dict_get_str (dict, key, &get_buffer);
+        if (ret) {
+                gf_log ("cli", GF_LOG_ERROR, "Failed to get %s", key);
                 goto out;
         }
+        cli_out ("%s" INDENT_MAIN_HEAD "%s", indent, "Status",
+                 ":", get_buffer);
+out :
+        return ret;
+}
 
-        ret = cli_cmd_submit (&req, frame, cli_rpc_prog,
-                              GLUSTER_CLI_CLRLOCKS_VOLUME, NULL,
-                              this, gf_cli3_1_clearlocks_volume_cbk,
-                              (xdrproc_t)xdr_gf_cli_req);
+/* This function is used to print snap related information
+ * arg - 0, dict       : Response dictionary.
+ * arg - 1, prefix_str : snaplist.snap{0..}.*
+ */
+int
+cli_get_volinfo_in_snap (dict_t *dict, char *keyprefix) {
+
+        char            key[PATH_MAX]   = "";
+        int             i               = 0;
+        int             volcount        = 0;
+        int             ret             = -1;
+
+        GF_ASSERT (dict);
+        GF_ASSERT (keyprefix);
+
+        ret = snprintf (key, sizeof (key), "%s.vol-count", keyprefix);
+        if (ret < 0) {
+                goto out;
+        }
+
+        ret = dict_get_int32 (dict, key, &volcount);
+        for (i = 1 ; i <= volcount ; i++) {
+                ret = snprintf (key, sizeof (key),
+                                "%s.vol%d", keyprefix, i);
+                if (ret < 0) {
+                        goto out;
+                }
+                ret = cli_get_each_volinfo_in_snap (dict, key, _gf_true);
+                if (ret) {
+                        gf_log ("cli", GF_LOG_ERROR, "Could not list "
+                                "details of volume in a snap");
+                        goto out;
+                }
+                cli_out (" ");
+        }
+
+out :
+        return ret;
+}
+
+int
+cli_get_each_snap_info (dict_t *dict, char *prefix_str,
+                        gf_boolean_t snap_driven) {
+        char            key_buffer[PATH_MAX] = "";
+        char           *get_buffer           = NULL;
+        int             ret                  = -1;
+        char            indent[5]            = "";
+
+        GF_ASSERT (dict);
+        GF_ASSERT (prefix_str);
+
+        if (!snap_driven)
+                strcat (indent, "\t");
+
+        ret = snprintf (key_buffer, sizeof (key_buffer), "%s.snapname",
+                        prefix_str);
+        if (ret < 0 ) {
+                goto out;
+        }
+
+        ret = dict_get_str (dict, key_buffer, &get_buffer);
+        if (ret) {
+                gf_log ("cli", GF_LOG_ERROR, "Unable to fetch snapname %s ",
+                        key_buffer);
+                goto out;
+        }
+        cli_out ("%s" INDENT_MAIN_HEAD "%s", indent, "Snapshot",
+                ":", get_buffer);
+
+        ret = snprintf (key_buffer, sizeof (key_buffer), "%s.snap-id",
+                        prefix_str);
+        if (ret < 0 ) {
+                goto out;
+        }
+
+        ret = dict_get_str (dict, key_buffer, &get_buffer);
+        if (ret) {
+                gf_log ("cli", GF_LOG_ERROR, "Unable to fetch snap-id %s ",
+                        key_buffer);
+                goto out;
+        }
+        cli_out ("%s" INDENT_MAIN_HEAD "%s", indent, "Snap UUID",
+                ":", get_buffer);
+
+        ret = snprintf (key_buffer, sizeof (key_buffer), "%s.snap-desc",
+                        prefix_str);
+        if (ret < 0 ) {
+                goto out;
+        }
+
+        ret = dict_get_str (dict, key_buffer, &get_buffer);
+        if (!ret) {
+                /* Ignore error for description */
+                cli_out ("%s" INDENT_MAIN_HEAD "%s", indent,
+                         "Description", ":", get_buffer);
+        }
+
+        ret = snprintf (key_buffer, sizeof (key_buffer), "%s.snap-time",
+                        prefix_str);
+        if (ret < 0 ) {
+                goto out;
+        }
+
+        ret = dict_get_str (dict, key_buffer, &get_buffer);
+        if (ret) {
+                gf_log ("cli", GF_LOG_ERROR, "Unable to fetch snap-time %s ",
+                        prefix_str);
+                goto out;
+        }
+        cli_out ("%s" INDENT_MAIN_HEAD "%s", indent, "Created",
+                ":", get_buffer);
+
+        if (snap_driven) {
+                cli_out ("%-12s", "Snap Volumes:\n");
+                ret = cli_get_volinfo_in_snap (dict, prefix_str);
+                if (ret) {
+                        gf_log ("cli", GF_LOG_ERROR, "Unable to list details "
+                                "of the snaps");
+                        goto out;
+                }
+        }
+out :
+        return ret;
+}
+
+/* This is a generic function to print snap related information.
+ * arg - 0, dict : Response Dictionary
+ */
+int
+cli_call_snapshot_info (dict_t *dict, gf_boolean_t bool_snap_driven) {
+        int             snap_count       =  0;
+        char            key[PATH_MAX]    =  "";
+        int             ret              =  -1;
+        int             i                =   0;
+
+        GF_ASSERT (dict);
+
+        ret = dict_get_int32 (dict, "snap-count", &snap_count);
+        if (ret) {
+                gf_log ("cli", GF_LOG_ERROR, "Unable to get snap-count");
+                goto out;
+        }
+
+        if (snap_count == 0) {
+                cli_out ("No snapshots present");
+        }
+
+        for (i = 1 ; i <= snap_count ; i++) {
+                ret = snprintf (key, sizeof (key), "snap%d", i);
+                if (ret < 0) {
+                        goto out;
+                }
+                ret = cli_get_each_snap_info (dict, key, bool_snap_driven);
+                if (ret) {
+                        gf_log ("cli", GF_LOG_ERROR,
+                                "Unable to print snap details");
+                        goto out;
+                }
+        }
+out :
+        return ret;
+}
+
+int
+cli_get_snaps_in_volume (dict_t *dict) {
+        int      ret             =  -1;
+        int      i               =  0;
+        int      count           =  0;
+        int      avail           =  0;
+        char     key[PATH_MAX]   =  "";
+        char    *get_buffer      =  NULL;
+
+        GF_ASSERT (dict);
+
+        ret = dict_get_str (dict, "origin-volname", &get_buffer);
+        if (ret) {
+                gf_log ("cli", GF_LOG_ERROR, "Could not fetch origin-volname");
+                goto out;
+        }
+        cli_out (INDENT_MAIN_HEAD "%s", "Volume Name", ":", get_buffer);
+
+        ret = dict_get_int32 (dict, "snap-count", &avail);
+        if (ret) {
+                gf_log ("cli", GF_LOG_ERROR, "Could not fetch snap-count");
+                goto out;
+        }
+        cli_out (INDENT_MAIN_HEAD "%d", "Snaps Taken", ":", avail);
+
+        ret = dict_get_int32 (dict, "snaps-available", &count);
+        if (ret) {
+                gf_log ("cli", GF_LOG_ERROR, "Could not fetch snaps-available");
+                goto out;
+        }
+        cli_out (INDENT_MAIN_HEAD "%d", "Snaps Available", ":", count);
+
+        for (i = 1 ; i <= avail ; i++) {
+                snprintf (key, sizeof (key), "snap%d", i);
+                ret = cli_get_each_snap_info (dict, key, _gf_false);
+                if (ret) {
+                        gf_log ("cli", GF_LOG_ERROR,
+                                "Unable to print snap details");
+                        goto out;
+                }
+
+                ret = snprintf (key, sizeof (key), "snap%d.vol1", i);
+                if (ret < 0) {
+                        goto out;
+                }
+                ret = cli_get_each_volinfo_in_snap (dict, key, _gf_false);
+                if (ret) {
+                        gf_log ("cli", GF_LOG_ERROR, "Could not get volume "
+                                "related information");
+                        goto out;
+                }
+
+                cli_out (" ");
+        }
+out :
+        return ret;
+}
+
+int
+cli_snapshot_list (dict_t *dict) {
+        int     snapcount       =       0;
+        char    key[PATH_MAX]   =       "";
+        int     ret             =       -1;
+        int     i               =       0;
+        char    *get_buffer     =       NULL;
+
+        GF_ASSERT (dict);
+
+        ret = dict_get_int32 (dict, "snap-count", &snapcount);
+        if (ret) {
+                gf_log ("cli", GF_LOG_ERROR, "Could not fetch snap count");
+                goto out;
+        }
+
+        if (snapcount == 0) {
+                cli_out ("No snapshots present");
+        }
+
+        for (i = 1 ; i <= snapcount ; i++) {
+                ret = snprintf (key, sizeof (key), "snapname%d",i);
+                if (ret < 0) {
+                        goto out;
+                }
+
+                ret = dict_get_str (dict, key, &get_buffer);
+                if (ret) {
+                        gf_log ("cli", GF_LOG_ERROR, "Could not get %s ", key);
+                        goto out;
+                } else {
+                        cli_out ("%s", get_buffer);
+                }
+        }
+out :
+        return ret;
+}
+
+int
+cli_get_snap_volume_status (dict_t *dict, char *key_prefix)
+{
+        int             ret             =       -1;
+        char            key[PATH_MAX]   =       "";
+        char            *buffer         =       NULL;
+        int             brickcount      =       0;
+        int             i               =       0;
+        int             pid             =       0;
+
+        GF_ASSERT (dict);
+        GF_ASSERT (key_prefix);
+
+        ret = snprintf (key, sizeof (key), "%s.brickcount", key_prefix);
+        if (ret < 0) {
+                goto out;
+        }
+        ret = dict_get_int32 (dict, key, &brickcount);
+        if (ret) {
+                gf_log ("cli", GF_LOG_ERROR, "Failed to fetch brickcount");
+                goto out;
+        }
+
+        for ( i = 0 ; i < brickcount ; i++ ) {
+                ret = snprintf (key, sizeof (key), "%s.brick%d.path",
+                                key_prefix, i);
+                if (ret < 0) {
+                        goto out;
+                }
+
+                ret = dict_get_str (dict, key, &buffer);
+                if (ret) {
+                        gf_log ("cli", GF_LOG_INFO,
+                                "Unable to get Brick Path");
+                        continue;
+                }
+                cli_out ("\n\t%-17s %s   %s", "Brick Path", ":", buffer);
+
+                ret = snprintf (key, sizeof (key), "%s.brick%d.vgname",
+                                key_prefix, i);
+                if (ret < 0) {
+                        goto out;
+                }
+
+                ret = dict_get_str (dict, key, &buffer);
+                if (ret) {
+                        gf_log ("cli", GF_LOG_INFO,
+                                "Unable to get Volume Group");
+                        cli_out ("\t%-17s %s   %s", "Volume Group", ":", "N/A");
+                } else
+                        cli_out ("\t%-17s %s   %s", "Volume Group", ":", buffer);
+
+                ret = snprintf (key, sizeof (key), "%s.brick%d.status",
+                                key_prefix, i);
+                if (ret < 0) {
+                        goto out;
+                }
+
+                ret = dict_get_str (dict, key, &buffer);
+                if (ret) {
+                        gf_log ("cli", GF_LOG_INFO,
+                                "Unable to get Brick Running");
+                        cli_out ("\t%-17s %s   %s", "Brick Running", ":", "N/A");
+                } else
+                        cli_out ("\t%-17s %s   %s", "Brick Running", ":", buffer);
+
+                ret = snprintf (key, sizeof (key), "%s.brick%d.pid",
+                                key_prefix, i);
+                if (ret < 0) {
+                        goto out;
+                }
+
+                ret = dict_get_int32 (dict, key, &pid);
+                if (ret) {
+                        gf_log ("cli", GF_LOG_INFO,
+                                "Unable to get pid");
+                        cli_out ("\t%-17s %s   %s", "Brick PID", ":", "N/A");
+                } else
+                        cli_out ("\t%-17s %s   %d", "Brick PID", ":", pid);
+
+                ret = snprintf (key, sizeof (key), "%s.brick%d.data",
+                                key_prefix, i);
+                if (ret < 0) {
+                        goto out;
+                }
+
+                ret = dict_get_str (dict, key, &buffer);
+                if (ret) {
+                        gf_log ("cli", GF_LOG_INFO,
+                                "Unable to get Data Percent");
+                        cli_out ("\t%-17s %s   %s", "Data Percentage", ":", "N/A");
+                } else
+                        cli_out ("\t%-17s %s   %s", "Data Percentage", ":", buffer);
+
+                ret = snprintf (key, sizeof (key), "%s.brick%d.lvsize",
+                                key_prefix, i);
+                if (ret < 0) {
+                        goto out;
+                }
+                ret = dict_get_str (dict, key, &buffer);
+                if (ret) {
+                        gf_log ("cli", GF_LOG_INFO, "Unable to get LV Size");
+                        cli_out ("\t%-17s %s   %s", "LV Size", ":", "N/A");
+                } else
+                        cli_out ("\t%-17s %s   %s", "LV Size", ":", buffer);
+
+        }
+out :
+        return ret;
+}
+
+
+
+int
+cli_get_single_snap_status (dict_t *dict, char *keyprefix)
+{
+        int             ret             =       -1;
+        char            key[PATH_MAX]   =       "";
+        int             i               =       0;
+        int             volcount        =       0;
+        char            *get_buffer     =       NULL;
+
+        GF_ASSERT (dict);
+        GF_ASSERT (keyprefix);
+
+        ret = snprintf (key, sizeof (key), "%s.snapname", keyprefix);
+        if (ret < 0) {
+                goto out;
+        }
+
+        ret = dict_get_str (dict, key, &get_buffer);
+        if (ret) {
+                gf_log ("cli", GF_LOG_ERROR, "Unable to get snapname");
+                goto out;
+        }
+        cli_out ("\nSnap Name : %s", get_buffer);
+
+        ret = snprintf (key, sizeof (key), "%s.uuid", keyprefix);
+        if (ret < 0) {
+                goto out;
+        }
+
+        ret = dict_get_str (dict, key, &get_buffer);
+        if (ret) {
+                gf_log ("cli", GF_LOG_ERROR, "Unable to get snap UUID");
+                goto out;
+        }
+        cli_out ("Snap UUID : %s", get_buffer);
+
+        ret = snprintf (key, sizeof (key), "%s.volcount", keyprefix);
+        if (ret < 0) {
+                goto out;
+        }
+
+        ret = dict_get_int32 (dict, key, &volcount);
+        if (ret) {
+                gf_log ("cli", GF_LOG_ERROR, "Unable to get volume count");
+                goto out;
+        }
 
+        for (i = 0 ; i < volcount ; i++) {
+                ret = snprintf (key, sizeof (key), "%s.vol%d", keyprefix, i);
+                if (ret < 0) {
+                        goto out;
+                }
+
+                ret = cli_get_snap_volume_status (dict, key);
+                if (ret) {
+                        gf_log ("cli", GF_LOG_ERROR,
+                                "Could not get snap volume status");
+                        goto out;
+                }
+        }
+out :
+        return ret;
+}
+
+int
+cli_snap_status_all (dict_t *dict) {
+        int             ret             =       -1;
+        char            key[PATH_MAX]   =       "";
+        int             snapcount       =       0;
+        int             i               =       0;
+
+        GF_ASSERT (dict);
+
+        ret = dict_get_int32 (dict, "status.snapcount", &snapcount);
+        if (ret) {
+                gf_log ("cli", GF_LOG_ERROR, "Could not get snapcount");
+                goto out;
+        }
+
+        if (snapcount == 0) {
+                cli_out ("No snapshots present");
+        }
+
+        for (i = 0 ; i < snapcount; i++) {
+                ret = snprintf (key, sizeof (key), "status.snap%d",i);
+                if (ret < 0) {
+                        goto out;
+                }
+                ret = cli_get_single_snap_status (dict, key);
+        }
+out:
+        return ret;
+}
+
+
+int
+cli_snapshot_status_display (dict_t *dict, gf_cli_rsp *rsp)
+{
+        char            key[PATH_MAX]   =       "";
+        int             ret             =       -1;
+        int             status_cmd      =       -1;
+
+        GF_ASSERT (dict);
+        GF_ASSERT (rsp);
+
+        if (rsp->op_ret) {
+                cli_err ("Snapshot Status : failed: %s",
+                        rsp->op_errstr ? rsp->op_errstr :
+                        "Please check log file for details");
+                ret = rsp->op_ret;
+                goto out;
+        }
+
+        ret = dict_get_int32 (dict, "cmd", &status_cmd);
+        if (ret) {
+                gf_log ("cli", GF_LOG_ERROR, "Could not fetch status type");
+                goto out;
+        }
+        switch (status_cmd) {
+                case GF_SNAP_STATUS_TYPE_ALL :
+                {
+                        ret = cli_snap_status_all (dict);
+                        if (ret) {
+                                gf_log ("cli", GF_LOG_ERROR, "Could not fetch "
+                                        "status of all snap");
+                                goto out;
+                        }
+                break;
+                }
+
+                case GF_SNAP_STATUS_TYPE_SNAP :
+                {
+                        ret = snprintf (key, sizeof (key), "status.snap0");
+                        if (ret < 0) {
+                                goto out;
+                        }
+                        ret = cli_get_single_snap_status (dict, key);
+                        if (ret) {
+                                gf_log ("cli", GF_LOG_ERROR, "Could not fetch "
+                                        "status of snap");
+                                goto out;
+                        }
+                break;
+                }
+
+                case GF_SNAP_STATUS_TYPE_VOL :
+                {
+                        ret = cli_snap_status_all (dict);
+                        if (ret) {
+                                gf_log ("cli", GF_LOG_ERROR, "Could not fetch "
+                                        "status of snap in a volume");
+                                goto out;
+                        }
+                        break;
+                }
+                default :
+                        break;
+        }
+out :
+        return ret;
+}
+
+int
+gf_cli_snapshot_cbk (struct rpc_req *req, struct iovec *iov,
+                     int count, void *myframe)
+{
+        int                   ret                      = -1;
+        gf_cli_rsp            rsp                      = {0, };
+        dict_t               *dict                     = NULL;
+        char                 *snap_name                = NULL;
+        int32_t               type                     =  0;
+        call_frame_t         *frame                    = NULL;
+        gf_boolean_t         snap_driven               = _gf_false;
+
+        if (req->rpc_status == -1) {
+                ret = -1;
+                goto out;
+        }
+
+        frame = myframe;
+
+        ret = xdr_to_generic (*iov, &rsp, (xdrproc_t)xdr_gf_cli_rsp);
+        if (ret < 0) {
+                gf_log (frame->this->name, GF_LOG_ERROR,
+                        "Failed to decode xdr response");
+                goto out;
+        }
+
+        dict = dict_new ();
+
+        if (!dict) {
+                ret = -1;
+                goto out;
+        }
+
+        ret = dict_unserialize (rsp.dict.dict_val, rsp.dict.dict_len, &dict);
+
+        if (ret)
+                goto out;
+
+        ret = dict_get_int32 (dict, "type", &type);
+        if (ret) {
+                gf_log (frame->this->name, GF_LOG_ERROR, "failed to get type");
+                goto out;
+        }
+
+        switch (type) {
+        case GF_SNAP_OPTION_TYPE_CREATE:
+                if (rsp.op_ret) {
+                        cli_err("snapshot create: failed: %s",
+                                 rsp.op_errstr ? rsp.op_errstr :
+                                 "Please check log file for details");
+                                 ret = rsp.op_ret;
+                                 goto out;
+                }
+
+                ret = dict_get_str (dict, "snapname", &snap_name);
+                if (ret) {
+                        gf_log ("cli", GF_LOG_ERROR,
+                                "Failed to get snap name");
+                        goto out;
+                }
+                cli_out ("snapshot create: %s: snap created successfully",
+                         snap_name);
+                break;
+
+        case GF_SNAP_OPTION_TYPE_RESTORE:
+                /* TODO: Check if rsp.op_ret needs to be checked here. Or is
+                 * it ok to check this in the start of the function where we
+                 * get rsp.*/
+                if (rsp.op_ret) {
+                        cli_err("snapshot restore: failed: %s",
+                                 rsp.op_errstr ? rsp.op_errstr :
+                                 "Please check log file for details");
+                                 ret = rsp.op_ret;
+                                 goto out;
+                }
+
+                ret = dict_get_str (dict, "snapname", &snap_name);
+                if (ret) {
+                        gf_log ("cli", GF_LOG_ERROR,
+                                "Failed to get snap name");
+                        goto out;
+                }
+
+                cli_out ("Snapshot restore: %s: Snap restored "
+                                 "successfully", snap_name);
+
+                ret = 0;
+                break;
+
+        case GF_SNAP_OPTION_TYPE_INFO:
+                if (rsp.op_ret) {
+                        cli_err ("Snapshot info : failed: %s",
+                                  rsp.op_errstr ? rsp.op_errstr :
+                                  "Please check log file for details");
+                        ret = rsp.op_ret;
+                        goto out;
+                }
+
+                snap_driven = dict_get_str_boolean (dict, "snap-driven",
+                                                    _gf_false);
+                if (snap_driven == _gf_true) {
+                        ret = cli_call_snapshot_info  (dict, snap_driven);
+                        if (ret) {
+                                gf_log ("cli", GF_LOG_ERROR,
+                                        "Snapshot info failed");
+                                goto out;
+                        }
+                } else if (snap_driven == _gf_false) {
+                        ret = cli_get_snaps_in_volume (dict);
+                        if (ret) {
+                                gf_log ("cli", GF_LOG_ERROR,
+                                        "Snapshot info failed");
+                                goto out;
+                        }
+                }
+                break;
+
+        case GF_SNAP_OPTION_TYPE_CONFIG:
+                ret = cli_snapshot_config_display (dict, &rsp);
+                if (ret) {
+                        gf_log ("cli", GF_LOG_ERROR, "Failed to display "
+                                "snapshot config output.");
+                        goto out;
+                }
+                break;
+
+        case GF_SNAP_OPTION_TYPE_LIST:
+                if (rsp.op_ret) {
+                        cli_err ("Snapshot list : failed: %s",
+                                 rsp.op_errstr ? rsp.op_errstr :
+                                "Please check log file for details");
+                        ret = rsp.op_ret;
+                        goto out;
+                }
+
+                ret = cli_snapshot_list (dict);
+                if (ret) {
+                        gf_log ("cli", GF_LOG_ERROR, "Failed to display "
+                                "snapshot list");
+                        goto out;
+                }
+                break;
+
+        case GF_SNAP_OPTION_TYPE_DELETE:
+                ret = cli_snapshot_remove_reply (&rsp, dict, frame);
+                if (ret) {
+                        gf_log ("cli", GF_LOG_ERROR,
+                                "Failed to delete snap");
+                        goto out;
+                }
+                break;
+
+        case GF_SNAP_OPTION_TYPE_STATUS:
+                ret = cli_snapshot_status_display (dict, &rsp);
+                if (ret) {
+                        gf_log ("cli", GF_LOG_ERROR, "Failed to display "
+                                "snapshot status output.");
+                        goto out;
+                }
+                break;
+
+        default:
+                cli_err ("Unknown command executed");
+                ret = -1;
+                goto out;
+        }
+out:
+        if (dict)
+                dict_unref (dict);
+        cli_cmd_broadcast_response (ret);
+
+        free (rsp.dict.dict_val);
+        free (rsp.op_errstr);
+
+        return ret;
+}
+
+int32_t
+gf_cli_snapshot (call_frame_t *frame, xlator_t *this,
+                 void *data)
+{
+        gf_cli_req        req     = {{0,}};
+        dict_t           *options = NULL;
+        int               ret     = -1;
+
+        if (!frame || !this || !data)
+                goto out;
+
+        options = data;
+
+        ret = cli_to_glusterd (&req, frame, gf_cli_snapshot_cbk,
+                               (xdrproc_t) xdr_gf_cli_req, options,
+                               GLUSTER_CLI_SNAP, this, cli_rpc_prog,
+                               NULL);
 out:
-        if (options)
-                dict_destroy (options);
         gf_log ("cli", GF_LOG_DEBUG, "Returning %d", ret);
 
-        if (req.dict.dict_val)
-                GF_FREE (req.dict.dict_val);
+        GF_FREE (req.dict.dict_val);
+        return ret;
+}
+
+int
+cli_to_glusterd (gf_cli_req *req, call_frame_t *frame,
+                 fop_cbk_fn_t cbkfn, xdrproc_t xdrproc, dict_t *dict,
+                 int procnum, xlator_t *this, rpc_clnt_prog_t *prog,
+                 struct iobref *iobref)
+{
+        int                ret = 0;
+        size_t             len = 0;
+        char               *cmd = NULL;
+        int                i = 0;
+        const char         **words = NULL;
+        cli_local_t        *local = NULL;
+
+        if (!this || !frame || !dict) {
+                ret = -1;
+                goto out;
+        }
+
+        if (!frame->local) {
+                ret = -1;
+                goto out;
+        }
+
+        local = frame->local;
+
+        if (!local->words) {
+                ret = -1;
+                goto out;
+        }
+
+        words = local->words;
+
+        while (words[i])
+                len += strlen (words[i++]) + 1;
+
+        cmd = GF_CALLOC (1, len, gf_common_mt_char);
+
+        if (!cmd) {
+                ret = -1;
+                goto out;
+        }
+
+        for (i = 0; words[i]; i++) {
+                strncat (cmd, words[i], strlen (words[i]));
+                if (words[i+1] != NULL)
+                        strncat (cmd, " ", strlen (" "));
+        }
+
+        cmd [len - 1] = '\0';
+
+        ret = dict_set_dynstr (dict, "cmd-str", cmd);
+        if (ret)
+                goto out;
+
+        ret = dict_allocate_and_serialize (dict, &(req->dict).dict_val,
+                                           &(req->dict).dict_len);
+
+        if (ret < 0) {
+                gf_log (this->name, GF_LOG_DEBUG,
+                        "failed to get serialized length of dict");
+                goto out;
+        }
+
+        ret = cli_cmd_submit (req, frame, prog, procnum, iobref, this,
+                              cbkfn, (xdrproc_t) xdrproc);
+
+out:
         return ret;
+
 }
 
 struct rpc_clnt_procedure gluster_cli_actors[GLUSTER_CLI_MAXVALUE] = {
         [GLUSTER_CLI_NULL]             = {"NULL", NULL },
-        [GLUSTER_CLI_PROBE]            = {"PROBE_QUERY", gf_cli3_1_probe},
-        [GLUSTER_CLI_DEPROBE]          = {"DEPROBE_QUERY", gf_cli3_1_deprobe},
-        [GLUSTER_CLI_LIST_FRIENDS]     = {"LIST_FRIENDS", gf_cli3_1_list_friends},
-        [GLUSTER_CLI_CREATE_VOLUME]    = {"CREATE_VOLUME", gf_cli3_1_create_volume},
-        [GLUSTER_CLI_DELETE_VOLUME]    = {"DELETE_VOLUME", gf_cli3_1_delete_volume},
-        [GLUSTER_CLI_START_VOLUME]     = {"START_VOLUME", gf_cli3_1_start_volume},
-        [GLUSTER_CLI_STOP_VOLUME]      = {"STOP_VOLUME", gf_cli3_1_stop_volume},
-        [GLUSTER_CLI_RENAME_VOLUME]    = {"RENAME_VOLUME", gf_cli3_1_rename_volume},
-        [GLUSTER_CLI_DEFRAG_VOLUME]    = {"DEFRAG_VOLUME", gf_cli3_1_defrag_volume},
-        [GLUSTER_CLI_GET_VOLUME]       = {"GET_VOLUME", gf_cli3_1_get_volume},
-        [GLUSTER_CLI_GET_NEXT_VOLUME]  = {"GET_NEXT_VOLUME", gf_cli3_1_get_next_volume},
-        [GLUSTER_CLI_SET_VOLUME]       = {"SET_VOLUME", gf_cli3_1_set_volume},
-        [GLUSTER_CLI_ADD_BRICK]        = {"ADD_BRICK", gf_cli3_1_add_brick},
-        [GLUSTER_CLI_REMOVE_BRICK]     = {"REMOVE_BRICK", gf_cli3_1_remove_brick},
-        [GLUSTER_CLI_REPLACE_BRICK]    = {"REPLACE_BRICK", gf_cli3_1_replace_brick},
-        [GLUSTER_CLI_LOG_ROTATE]       = {"LOG ROTATE", gf_cli3_1_log_rotate},
-        [GLUSTER_CLI_GETSPEC]          = {"GETSPEC", gf_cli3_1_getspec},
-        [GLUSTER_CLI_PMAP_PORTBYBRICK] = {"PMAP PORTBYBRICK", gf_cli3_1_pmap_b2p},
-        [GLUSTER_CLI_SYNC_VOLUME]      = {"SYNC_VOLUME", gf_cli3_1_sync_volume},
-        [GLUSTER_CLI_RESET_VOLUME]     = {"RESET_VOLUME", gf_cli3_1_reset_volume},
-        [GLUSTER_CLI_FSM_LOG]          = {"FSM_LOG", gf_cli3_1_fsm_log},
-        [GLUSTER_CLI_GSYNC_SET]        = {"GSYNC_SET", gf_cli3_1_gsync_set},
-        [GLUSTER_CLI_PROFILE_VOLUME]   = {"PROFILE_VOLUME", gf_cli3_1_profile_volume},
-        [GLUSTER_CLI_QUOTA]            = {"QUOTA", gf_cli3_1_quota},
-        [GLUSTER_CLI_TOP_VOLUME]       = {"TOP_VOLUME", gf_cli3_1_top_volume},
-        [GLUSTER_CLI_GETWD]            = {"GETWD", gf_cli3_1_getwd},
-        [GLUSTER_CLI_STATUS_VOLUME]    = {"STATUS_VOLUME", gf_cli3_1_status_volume},
+        [GLUSTER_CLI_PROBE]            = {"PROBE_QUERY", gf_cli_probe},
+        [GLUSTER_CLI_DEPROBE]          = {"DEPROBE_QUERY", gf_cli_deprobe},
+        [GLUSTER_CLI_LIST_FRIENDS]     = {"LIST_FRIENDS", gf_cli_list_friends},
+        [GLUSTER_CLI_UUID_RESET]       = {"UUID_RESET", gf_cli3_1_uuid_reset},
+        [GLUSTER_CLI_UUID_GET]       = {"UUID_GET", gf_cli3_1_uuid_get},
+        [GLUSTER_CLI_CREATE_VOLUME]    = {"CREATE_VOLUME", gf_cli_create_volume},
+        [GLUSTER_CLI_DELETE_VOLUME]    = {"DELETE_VOLUME", gf_cli_delete_volume},
+        [GLUSTER_CLI_START_VOLUME]     = {"START_VOLUME", gf_cli_start_volume},
+        [GLUSTER_CLI_STOP_VOLUME]      = {"STOP_VOLUME", gf_cli_stop_volume},
+        [GLUSTER_CLI_RENAME_VOLUME]    = {"RENAME_VOLUME", gf_cli_rename_volume},
+        [GLUSTER_CLI_DEFRAG_VOLUME]    = {"DEFRAG_VOLUME", gf_cli_defrag_volume},
+        [GLUSTER_CLI_GET_VOLUME]       = {"GET_VOLUME", gf_cli_get_volume},
+        [GLUSTER_CLI_GET_NEXT_VOLUME]  = {"GET_NEXT_VOLUME", gf_cli_get_next_volume},
+        [GLUSTER_CLI_SET_VOLUME]       = {"SET_VOLUME", gf_cli_set_volume},
+        [GLUSTER_CLI_ADD_BRICK]        = {"ADD_BRICK", gf_cli_add_brick},
+        [GLUSTER_CLI_REMOVE_BRICK]     = {"REMOVE_BRICK", gf_cli_remove_brick},
+        [GLUSTER_CLI_REPLACE_BRICK]    = {"REPLACE_BRICK", gf_cli_replace_brick},
+        [GLUSTER_CLI_LOG_ROTATE]       = {"LOG ROTATE", gf_cli_log_rotate},
+        [GLUSTER_CLI_GETSPEC]          = {"GETSPEC", gf_cli_getspec},
+        [GLUSTER_CLI_PMAP_PORTBYBRICK] = {"PMAP PORTBYBRICK", gf_cli_pmap_b2p},
+        [GLUSTER_CLI_SYNC_VOLUME]      = {"SYNC_VOLUME", gf_cli_sync_volume},
+        [GLUSTER_CLI_RESET_VOLUME]     = {"RESET_VOLUME", gf_cli_reset_volume},
+        [GLUSTER_CLI_FSM_LOG]          = {"FSM_LOG", gf_cli_fsm_log},
+        [GLUSTER_CLI_GSYNC_SET]        = {"GSYNC_SET", gf_cli_gsync_set},
+        [GLUSTER_CLI_PROFILE_VOLUME]   = {"PROFILE_VOLUME", gf_cli_profile_volume},
+        [GLUSTER_CLI_QUOTA]            = {"QUOTA", gf_cli_quota},
+        [GLUSTER_CLI_TOP_VOLUME]       = {"TOP_VOLUME", gf_cli_top_volume},
+        [GLUSTER_CLI_GETWD]            = {"GETWD", gf_cli_getwd},
+        [GLUSTER_CLI_STATUS_VOLUME]    = {"STATUS_VOLUME", gf_cli_status_volume},
         [GLUSTER_CLI_STATUS_ALL]       = {"STATUS_ALL", gf_cli_status_volume_all},
-        [GLUSTER_CLI_MOUNT]            = {"MOUNT", gf_cli3_1_mount},
-        [GLUSTER_CLI_UMOUNT]           = {"UMOUNT", gf_cli3_1_umount},
-        [GLUSTER_CLI_HEAL_VOLUME]      = {"HEAL_VOLUME", gf_cli3_1_heal_volume},
-        [GLUSTER_CLI_STATEDUMP_VOLUME] = {"STATEDUMP_VOLUME", gf_cli3_1_statedump_volume},
-        [GLUSTER_CLI_LIST_VOLUME]      = {"LIST_VOLUME", gf_cli3_1_list_volume},
-        [GLUSTER_CLI_CLRLOCKS_VOLUME]  = {"CLEARLOCKS_VOLUME", gf_cli3_1_clearlocks_volume},
+        [GLUSTER_CLI_MOUNT]            = {"MOUNT", gf_cli_mount},
+        [GLUSTER_CLI_UMOUNT]           = {"UMOUNT", gf_cli_umount},
+        [GLUSTER_CLI_HEAL_VOLUME]      = {"HEAL_VOLUME", gf_cli_heal_volume},
+        [GLUSTER_CLI_STATEDUMP_VOLUME] = {"STATEDUMP_VOLUME", gf_cli_statedump_volume},
+        [GLUSTER_CLI_LIST_VOLUME]      = {"LIST_VOLUME", gf_cli_list_volume},
+        [GLUSTER_CLI_CLRLOCKS_VOLUME]  = {"CLEARLOCKS_VOLUME", gf_cli_clearlocks_volume},
+        [GLUSTER_CLI_COPY_FILE]        = {"COPY_FILE", gf_cli_copy_file},
+        [GLUSTER_CLI_SYS_EXEC]         = {"SYS_EXEC", gf_cli_sys_exec},
+        [GLUSTER_CLI_SNAP]             = {"SNAP", gf_cli_snapshot},
 };
 
 struct rpc_clnt_program cli_prog = {
diff --git a/cli/src/cli-xml-output.c b/cli/src/cli-xml-output.c
index 7f8f9cfc4..d8884d44b 100644
--- a/cli/src/cli-xml-output.c
+++ b/cli/src/cli-xml-output.c
@@ -1,22 +1,12 @@
 /*
-  Copyright (c) 2010-2011 Gluster, Inc. <http://www.gluster.com>
-  This file is part of GlusterFS.
-
-  GlusterFS is free software; you can redistribute it and/or modify
-  it under the terms of the GNU General Public License as published
-  by the Free Software Foundation; either version 3 of the License,
-  or (at your option) any later version.
-
-  GlusterFS is distributed in the hope that it will be useful, but
-  WITHOUT ANY WARRANTY; without even the implied warranty of
-  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-  General Public License for more details.
-
-  You should have received a copy of the GNU General Public License
-  along with this program.  If not, see
-  <http://www.gnu.org/licenses/>.
-*/
+   Copyright (c) 2010-2012 Red Hat, Inc. <http://www.redhat.com>
+   This file is part of GlusterFS.
 
+   This file is licensed to you under your choice of the GNU Lesser
+   General Public License, version 3 or any later version (LGPLv3 or
+   later), or the GNU General Public License, version 2 (GPLv2), in all
+   cases as published by the Free Software Foundation.
+*/
 #include <stdlib.h>
 #include "cli.h"
 #include "cli1-xdr.h"
@@ -25,6 +15,39 @@
 #include "syscall.h"
 
 
+enum gf_task_types {
+    GF_TASK_TYPE_REBALANCE,
+    GF_TASK_TYPE_REMOVE_BRICK
+};
+
+/*
+ * IMPORTANT NOTE:
+ * All exported functions in this file which use libxml need use a
+ * #if (HAVE_LIB_XML), #else, #endif
+ * For eg,
+ *      int exported_func () {
+ *              #if (HAVE_LIB_XML)
+ *                      <Stuff using libxml>
+ *              #else
+ *                      return 0;
+ *              #endif
+ *      }
+ *
+ *  All other functions, which are called internally within this file need to be
+ *  within #if (HAVE_LIB_XML), #endif statements
+ *  For eg,
+ *      #if (HAVE_LIB_XML)
+ *      int internal_func ()
+ *      {
+ *      }
+ *      #endif
+ *
+ *  Following the above formate ensures that all xml related code is compliled
+ *  only when libxml2 is present, and also keeps the rest of the codebase free
+ *  of #if (HAVE_LIB_XML)
+ */
+
+
 #if (HAVE_LIB_XML)
 
 #include <libxml/encoding.h>
@@ -40,18 +63,11 @@
         }while (0)                                              \
 
 int
-cli_begin_xml_output (xmlTextWriterPtr *writer, xmlBufferPtr *buf)
+cli_begin_xml_output (xmlTextWriterPtr *writer, xmlDocPtr *doc)
 {
         int             ret = -1;
 
-        *buf = xmlBufferCreateSize (8192);
-        if (buf == NULL) {
-                ret = -1;
-                goto out;
-        }
-        xmlBufferSetAllocationScheme (*buf, XML_BUFFER_ALLOC_DOUBLEIT);
-
-        *writer = xmlNewTextWriterMemory (*buf, 0);
+        *writer = xmlNewTextWriterDoc (doc, 0);
         if (writer == NULL) {
                 ret = -1;
                 goto out;
@@ -70,7 +86,7 @@ out:
 }
 
 int
-cli_end_xml_output (xmlTextWriterPtr writer, xmlBufferPtr buf)
+cli_end_xml_output (xmlTextWriterPtr writer, xmlDocPtr doc)
 {
         int             ret = -1;
 
@@ -81,10 +97,12 @@ cli_end_xml_output (xmlTextWriterPtr writer, xmlBufferPtr buf)
         ret = xmlTextWriterEndDocument (writer);
         XML_RET_CHECK_AND_GOTO (ret, out);
 
-        cli_out ("%s", (const char *)buf->content);
+
+        /* Dump xml document to stdout and pretty format it */
+        xmlSaveFormatFileEnc ("-", doc, "UTF-8", 1);
 
         xmlFreeTextWriter (writer);
-        xmlBufferFree (buf);
+        xmlFreeDoc (doc);
 
 out:
         gf_log ("cli", GF_LOG_DEBUG, "Returning %d", ret);
@@ -106,23 +124,25 @@ cli_xml_output_common (xmlTextWriterPtr writer, int op_ret, int op_errno,
         XML_RET_CHECK_AND_GOTO (ret, out);
 
         ret = xmlTextWriterWriteFormatElement (writer, (xmlChar *)"opErrstr",
-                                               "%s", op_errstr);
+                                                "%s", op_errstr);
         XML_RET_CHECK_AND_GOTO (ret, out);
 
 out:
         gf_log ("cli", GF_LOG_DEBUG, "Returning %d", ret);
         return ret;
 }
+#endif
 
 int
 cli_xml_output_str (char *op, char *str, int op_ret, int op_errno,
                     char *op_errstr)
 {
+#if (HAVE_LIB_XML)
         int                     ret = -1;
         xmlTextWriterPtr        writer = NULL;
-        xmlBufferPtr            buf = NULL;
+        xmlDocPtr               doc = NULL;
 
-        ret = cli_begin_xml_output (&writer, &buf);
+        ret = cli_begin_xml_output (&writer, &doc);
         if (ret)
                 goto out;
 
@@ -130,22 +150,32 @@ cli_xml_output_str (char *op, char *str, int op_ret, int op_errno,
         if (ret)
                 goto out;
 
-        ret = xmlTextWriterWriteFormatElement (writer, (xmlChar *)"cliOp",
-                                               "%s", op);
-        XML_RET_CHECK_AND_GOTO (ret, out);
+        if (op) {
+                ret = xmlTextWriterWriteFormatElement (writer,
+                                                       (xmlChar *)"cliOp",
+                                                       "%s", op);
+                XML_RET_CHECK_AND_GOTO (ret, out);
+        }
 
-        ret = xmlTextWriterWriteFormatElement (writer, (xmlChar *)"output",
-                                               "%s", str);
-        XML_RET_CHECK_AND_GOTO (ret, out);
+        if (str) {
+                ret = xmlTextWriterWriteFormatElement (writer,
+                                                       (xmlChar *)"output",
+                                                       "%s", str);
+                XML_RET_CHECK_AND_GOTO (ret, out);
+        }
 
-        ret = cli_end_xml_output (writer, buf);
+        ret = cli_end_xml_output (writer, doc);
 
 out:
         gf_log ("cli", GF_LOG_DEBUG, "Returning %d", ret);
         return ret;
+#else
+        return 0;
+#endif
 }
 
-void
+#if (HAVE_LIB_XML)
+int
 cli_xml_output_data_pair (dict_t *this, char *key, data_t *value,
                           void *data)
 {
@@ -157,39 +187,52 @@ cli_xml_output_data_pair (dict_t *this, char *key, data_t *value,
         ret = xmlTextWriterWriteFormatElement (*writer, (xmlChar *)key,
                                                "%s", value->data);
 
-        return;
+        return ret;
 }
+#endif
 
 int
 cli_xml_output_dict ( char *op, dict_t *dict, int op_ret, int op_errno,
                       char *op_errstr)
 {
+#if (HAVE_LIB_XML)
         int                     ret = -1;
         xmlTextWriterPtr        writer = NULL;
-        xmlBufferPtr            buf = NULL;
+        xmlDocPtr               doc = NULL;
 
-        ret = cli_begin_xml_output (&writer, &buf);
+        ret = cli_begin_xml_output (&writer, &doc);
         if (ret)
                 goto out;
+
+        ret = cli_xml_output_common (writer, op_ret, op_errno, op_errstr);
+        if (ret)
+                goto out;
+
         /* <"op"> */
         ret = xmlTextWriterStartElement (writer, (xmlChar *)op);
         XML_RET_CHECK_AND_GOTO (ret, out);
 
-        dict_foreach (dict, cli_xml_output_data_pair, &writer);
+        if (dict)
+                dict_foreach (dict, cli_xml_output_data_pair, &writer);
 
         /* </"op"> */
         ret = xmlTextWriterEndElement (writer);
         XML_RET_CHECK_AND_GOTO (ret, out);
 
-        ret = cli_end_xml_output (writer, buf);
+        ret = cli_end_xml_output (writer, doc);
 out:
         gf_log ("cli", GF_LOG_DEBUG, "Returning %d", ret);
         return ret;
+#else
+        return 0;
+#endif
 }
 
+#if (HAVE_LIB_XML)
 int
 cli_xml_output_vol_status_common (xmlTextWriterPtr writer, dict_t *dict,
-                                  int   brick_index, int *online)
+                                  int   brick_index, int *online,
+                                  gf_boolean_t *node_present)
 {
         int             ret = -1;
         char            *hostname = NULL;
@@ -201,8 +244,12 @@ cli_xml_output_vol_status_common (xmlTextWriterPtr writer, dict_t *dict,
 
         snprintf (key, sizeof (key), "brick%d.hostname", brick_index);
         ret = dict_get_str (dict, key, &hostname);
-        if (ret)
+        if (ret) {
+                *node_present = _gf_false;
                 goto out;
+        }
+        *node_present = _gf_true;
+
         ret = xmlTextWriterWriteFormatElement (writer, (xmlChar *)"hostname",
                                                "%s", hostname);
         XML_RET_CHECK_AND_GOTO (ret, out);
@@ -217,15 +264,6 @@ cli_xml_output_vol_status_common (xmlTextWriterPtr writer, dict_t *dict,
         XML_RET_CHECK_AND_GOTO (ret, out);
 
         memset (key, 0, sizeof (key));
-        snprintf (key, sizeof (key), "brick%d.port", brick_index);
-        ret = dict_get_int32 (dict, key, &port);
-        if (ret)
-                goto out;
-        ret = xmlTextWriterWriteFormatElement (writer, (xmlChar *)"port",
-                                               "%d", port);
-        XML_RET_CHECK_AND_GOTO (ret, out);
-
-        memset (key, 0, sizeof (key));
         snprintf (key, sizeof (key), "brick%d.status", brick_index);
         ret = dict_get_int32 (dict, key, &status);
         if (ret)
@@ -236,6 +274,27 @@ cli_xml_output_vol_status_common (xmlTextWriterPtr writer, dict_t *dict,
         *online = status;
 
         memset (key, 0, sizeof (key));
+        snprintf (key, sizeof (key), "brick%d.port", brick_index);
+        ret = dict_get_int32 (dict, key, &port);
+        if (ret)
+                goto out;
+
+        /* If the process is either offline or doesn't provide a port (shd)
+         * port = "N/A"
+         * else print the port number of the process.
+         */
+
+        if (*online == 1 && port != 0)
+                ret = xmlTextWriterWriteFormatElement (writer,
+                                                       (xmlChar *)"port",
+                                                       "%d", port);
+        else
+                ret = xmlTextWriterWriteFormatElement (writer,
+                                                       (xmlChar *)"port",
+                                                       "%s", "N/A");
+        XML_RET_CHECK_AND_GOTO (ret, out);
+
+        memset (key, 0, sizeof (key));
         snprintf (key, sizeof (key), "brick%d.pid", brick_index);
         ret = dict_get_int32 (dict, key, &pid);
         if (ret)
@@ -1272,44 +1331,305 @@ out:
         gf_log ("cli", GF_LOG_DEBUG, "Returning %d", ret);
         return ret;
 }
+#endif
 
 int
-cli_xml_output_vol_status (dict_t *dict, int op_ret, int op_errno,
-                           char *op_errstr)
+cli_xml_output_vol_status_begin (cli_local_t *local, int op_ret, int op_errno,
+                                 char *op_errstr)
 {
+#if (HAVE_LIB_XML)
         int                     ret = -1;
-        xmlTextWriterPtr        writer = NULL;
-        xmlBufferPtr            buf = NULL;
-        char                    *volname = NULL;
-        int                     brick_count = 0;
-        uint32_t                cmd = GF_CLI_STATUS_NONE;
-        int                     online = 0;
-        int                     i;
 
-        ret = cli_begin_xml_output (&writer, &buf);
+        ret = cli_begin_xml_output (&(local->writer), &(local->doc));
+        XML_RET_CHECK_AND_GOTO (ret, out);
+
+        ret = cli_xml_output_common (local->writer, op_ret, op_errno,
+                                     op_errstr);
+        XML_RET_CHECK_AND_GOTO (ret, out);
+
+        /* <volStatus> */
+        ret = xmlTextWriterStartElement (local->writer,
+                                         (xmlChar *) "volStatus");
+        XML_RET_CHECK_AND_GOTO (ret, out);
+
+        /* <volumes> */
+        ret = xmlTextWriterStartElement (local->writer, (xmlChar *)"volumes");
+        XML_RET_CHECK_AND_GOTO (ret, out);
+
+out:
+        gf_log ("cli", GF_LOG_DEBUG, "Returning %d", ret);
+        return ret;
+#else
+        return 0;
+#endif
+}
+
+int
+cli_xml_output_vol_status_end (cli_local_t *local)
+{
+#if (HAVE_LIB_XML)
+        int     ret = -1;
+
+        /* </volumes> */
+        ret = xmlTextWriterEndElement (local->writer);
+        XML_RET_CHECK_AND_GOTO (ret, out);
+
+        /* </volStatus> */
+        ret = xmlTextWriterEndElement (local->writer);
+        XML_RET_CHECK_AND_GOTO(ret, out);
+
+        ret = cli_end_xml_output (local->writer, local->doc);
+out:
+        gf_log ("cli", GF_LOG_DEBUG, "Returning %d", ret);
+        return ret;
+#else
+        return 0;
+#endif
+}
+
+#if (HAVE_LIB_XML)
+int
+cli_xml_output_remove_brick_task_params (xmlTextWriterPtr writer, dict_t *dict,
+                                         char *prefix)
+{
+        int             ret = -1;
+        char            key[1024] = {0,};
+        int             count = 0;
+        int             i = 0;
+        char            *brick = NULL;
+
+        /* <params> */
+        ret = xmlTextWriterStartElement (writer, (xmlChar *)"params");
+        XML_RET_CHECK_AND_GOTO (ret, out);
+
+        snprintf (key, sizeof (key), "%s.count", prefix);
+        ret = dict_get_int32 (dict, key, &count);
         if (ret)
                 goto out;
 
-        ret = cli_xml_output_common (writer, op_ret, op_errno, op_errstr);
+        for (i = 1; i <= count; i++) {
+                memset (key, 0, sizeof (key));
+                snprintf (key, sizeof (key), "%s.brick%d", prefix, i);
+                ret = dict_get_str (dict, key, &brick);
+                if (ret)
+                        goto out;
+                ret = xmlTextWriterWriteFormatElement (writer,
+                                                       (xmlChar *)"brick",
+                                                       "%s", brick);
+                XML_RET_CHECK_AND_GOTO (ret, out);
+                brick = NULL;
+        }
+
+        /* </param> */
+        ret = xmlTextWriterEndElement (writer);
+
+out:
+        gf_log ("cli", GF_LOG_DEBUG, "Returning %d", ret);
+        return ret;
+}
+
+int
+cli_xml_output_replace_brick_task_params (xmlTextWriterPtr writer, dict_t *dict,
+                                          char *prefix)
+{
+
+        int             ret = -1;
+        char            key[1024] = {0,};
+        char            *brick = NULL;
+
+        /* <params> */
+        ret = xmlTextWriterStartElement (writer, (xmlChar *)"params");
+        XML_RET_CHECK_AND_GOTO (ret, out);
+
+        snprintf (key, sizeof (key), "%s.src-brick", prefix);
+        ret = dict_get_str (dict, key, &brick);
         if (ret)
                 goto out;
+        ret = xmlTextWriterWriteFormatElement (writer, (xmlChar *)"srcBrick",
+                                               "%s", brick);
+        XML_RET_CHECK_AND_GOTO (ret, out);
 
-        /* <volStatus> */
-        ret = xmlTextWriterStartElement (writer, (xmlChar *)"volStatus");
+
+        memset (key, 0, sizeof (key));
+        snprintf (key, sizeof (key), "%s.dst-brick", prefix);
+        ret = dict_get_str (dict, key, &brick);
+        if (ret)
+                goto out;
+        ret = xmlTextWriterWriteFormatElement (writer, (xmlChar *)"dstBrick",
+                                               "%s", brick);
+        XML_RET_CHECK_AND_GOTO (ret, out);
+
+
+        /* </param> */
+        ret = xmlTextWriterEndElement (writer);
+
+out:
+        gf_log ("cli", GF_LOG_DEBUG, "Returning %d", ret);
+        return ret;
+}
+
+int
+cli_xml_output_vol_status_tasks (cli_local_t *local, dict_t *dict) {
+        int                     ret = -1;
+        char                    *task_type = NULL;
+        char                    *task_id_str = NULL;
+        int                     status = 0;
+        int                     tasks = 0;
+        char                    key[1024] = {0,};
+        int                     i = 0;
+
+        /* <tasks> */
+        ret = xmlTextWriterStartElement (local->writer, (xmlChar *)"tasks");
+        XML_RET_CHECK_AND_GOTO (ret, out);
+
+        ret = dict_get_int32 (dict, "tasks", &tasks);
+        if (ret)
+                goto out;
+
+        for (i = 0; i < tasks; i++) {
+                /* <task> */
+                ret = xmlTextWriterStartElement (local->writer,
+                                                 (xmlChar *)"task");
+                XML_RET_CHECK_AND_GOTO (ret, out);
+
+                memset (key, 0, sizeof (key));
+                snprintf (key, sizeof (key), "task%d.type", i);
+                ret = dict_get_str (dict, key, &task_type);
+                if (ret)
+                        goto out;
+                ret = xmlTextWriterWriteFormatElement (local->writer,
+                                                       (xmlChar *)"type",
+                                                       "%s", task_type);
+                XML_RET_CHECK_AND_GOTO (ret, out);
+
+                memset (key, 0, sizeof (key));
+                snprintf (key, sizeof (key), "task%d.id", i);
+                ret = dict_get_str (dict, key, &task_id_str);
+                if (ret)
+                        goto out;
+                ret = xmlTextWriterWriteFormatElement (local->writer,
+                                                       (xmlChar *)"id",
+                                                       "%s", task_id_str);
+                XML_RET_CHECK_AND_GOTO (ret, out);
+
+                memset (key, 0, sizeof (key));
+                snprintf (key, sizeof (key), "task%d.status", i);
+                ret = dict_get_int32 (dict, key, &status);
+                if (ret)
+                        goto out;
+                ret = xmlTextWriterWriteFormatElement (local->writer,
+                                                       (xmlChar *)"status",
+                                                       "%d", status);
+                XML_RET_CHECK_AND_GOTO (ret, out);
+
+                if (!strcmp (task_type, "Replace brick")) {
+                    if (status) {
+                        status = GF_DEFRAG_STATUS_COMPLETE;
+                    } else {
+                        status = GF_DEFRAG_STATUS_STARTED;
+                    }
+                }
+
+                ret = xmlTextWriterWriteFormatElement (local->writer,
+                                                       (xmlChar *)"statusStr",
+                                                       "%s",
+                                             cli_vol_task_status_str[status]);
+
+                XML_RET_CHECK_AND_GOTO (ret, out);
+
+                memset (key, 0, sizeof (key));
+                snprintf (key, sizeof (key), "task%d", i);
+                if (!strcmp (task_type, "Replace brick")) {
+                        ret = cli_xml_output_replace_brick_task_params
+                                (local->writer, dict, key);
+                        if (ret)
+                                goto out;
+                } else if (!strcmp (task_type, "Remove brick")) {
+                        ret = cli_xml_output_remove_brick_task_params
+                                (local->writer, dict, key);
+                        if (ret)
+                                goto out;
+                }
+
+
+                /* </task> */
+                ret = xmlTextWriterEndElement (local->writer);
+                XML_RET_CHECK_AND_GOTO (ret, out);
+        }
+
+        /* </tasks> */
+        ret = xmlTextWriterEndElement (local->writer);
+
+out:
+        gf_log ("cli", GF_LOG_DEBUG, "Returning %d", ret);
+        return ret;
+}
+
+int
+cli_xml_output_vol_status_tasks_detail (cli_local_t *local, dict_t *dict)
+{
+        int    ret     = -1;
+        char  *volname = NULL;
+
+        /*<volume>*/
+        ret = xmlTextWriterStartElement (local->writer, (xmlChar *)"volume");
         XML_RET_CHECK_AND_GOTO (ret, out);
 
         ret = dict_get_str (dict, "volname", &volname);
         if (ret)
                 goto out;
-        ret = xmlTextWriterWriteFormatElement (writer, (xmlChar *)"volName",
-                                               "%s", volname);
+        ret = xmlTextWriterWriteFormatElement (local->writer,
+                                               (xmlChar *)"volName", "%s",
+                                               volname);
+        XML_RET_CHECK_AND_GOTO (ret, out);
+
+        ret = cli_xml_output_vol_status_tasks (local, dict);
+        if (ret)
+                goto out;
+
+        /* </volume> */
+        ret = xmlTextWriterEndElement (local->writer);
+        XML_RET_CHECK_AND_GOTO (ret, out);
+
+out:
+        return ret;
+}
+#endif
+
+int
+cli_xml_output_vol_status (cli_local_t *local, dict_t *dict)
+{
+#if (HAVE_LIB_XML)
+        int                     ret = -1;
+        char                    *volname = NULL;
+        int                     brick_count = 0;
+        int                     brick_index_max = -1;
+        int                     other_count = 0;
+        int                     index_max = 0;
+        uint32_t                cmd = GF_CLI_STATUS_NONE;
+        int                     online = 0;
+        gf_boolean_t            node_present = _gf_true;
+        int                     i;
+
+
+        /* <volume> */
+        ret = xmlTextWriterStartElement (local->writer, (xmlChar *)"volume");
+        XML_RET_CHECK_AND_GOTO (ret, out);
+
+        ret = dict_get_str (dict, "volname", &volname);
+        if (ret)
+                goto out;
+        ret = xmlTextWriterWriteFormatElement (local->writer,
+                                               (xmlChar *)"volName", "%s",
+                                               volname);
         XML_RET_CHECK_AND_GOTO (ret, out);
 
         ret = dict_get_int32 (dict, "count", &brick_count);
         if (ret)
                 goto out;
-        ret = xmlTextWriterWriteFormatElement (writer, (xmlChar *)"brickCount",
-                                               "%d", brick_count);
+        ret = xmlTextWriterWriteFormatElement (local->writer,
+                                               (xmlChar *)"nodeCount", "%d",
+                                               brick_count);
         if (ret)
                 goto out;
 
@@ -1317,19 +1637,33 @@ cli_xml_output_vol_status (dict_t *dict, int op_ret, int op_errno,
         if (ret)
                 goto out;
 
-        for (i = 0; i < brick_count; i++) {
-                /* <brick> */
-                ret = xmlTextWriterStartElement (writer, (xmlChar *)"brick");
+        ret = dict_get_int32 (dict, "brick-index-max", &brick_index_max);
+        if (ret)
+                goto out;
+        ret = dict_get_int32 (dict, "other-count", &other_count);
+        if (ret)
+                goto out;
+
+        index_max = brick_index_max + other_count;
+
+        for (i = 0; i <= index_max; i++) {
+                /* <node> */
+                ret = xmlTextWriterStartElement (local->writer,
+                                                 (xmlChar *)"node");
                 XML_RET_CHECK_AND_GOTO (ret, out);
 
-                ret = cli_xml_output_vol_status_common (writer, dict, i,
-                                                        &online);
-                if (ret)
-                        goto out;
+                ret = cli_xml_output_vol_status_common (local->writer, dict, i,
+                                                        &online, &node_present);
+                if (ret) {
+                        if (node_present)
+                                goto out;
+                        else
+                                continue;
+                }
 
                 switch (cmd & GF_CLI_STATUS_MASK) {
                 case GF_CLI_STATUS_DETAIL:
-                        ret = cli_xml_output_vol_status_detail (writer,
+                        ret = cli_xml_output_vol_status_detail (local->writer,
                                                                 dict, i);
                         if (ret)
                                 goto out;
@@ -1338,7 +1672,7 @@ cli_xml_output_vol_status (dict_t *dict, int op_ret, int op_errno,
                 case GF_CLI_STATUS_MEM:
                         if (online) {
                                 ret = cli_xml_output_vol_status_mem
-                                        (writer, dict, i);
+                                        (local->writer, dict, i);
                                 if (ret)
                                         goto out;
                         }
@@ -1347,7 +1681,7 @@ cli_xml_output_vol_status (dict_t *dict, int op_ret, int op_errno,
                 case GF_CLI_STATUS_CLIENTS:
                         if (online) {
                                 ret = cli_xml_output_vol_status_clients
-                                        (writer, dict, i);
+                                        (local->writer, dict, i);
                                 if (ret)
                                         goto out;
                         }
@@ -1356,7 +1690,7 @@ cli_xml_output_vol_status (dict_t *dict, int op_ret, int op_errno,
                 case GF_CLI_STATUS_INODE:
                         if (online) {
                                 ret = cli_xml_output_vol_status_inode
-                                        (writer, dict, i);
+                                        (local->writer, dict, i);
                                 if (ret)
                                         goto out;
                         }
@@ -1365,7 +1699,7 @@ cli_xml_output_vol_status (dict_t *dict, int op_ret, int op_errno,
                 case GF_CLI_STATUS_FD:
                         if (online) {
                                 ret = cli_xml_output_vol_status_fd
-                                        (writer, dict, i);
+                                        (local->writer, dict, i);
                                 if (ret)
                                         goto out;
                         }
@@ -1374,47 +1708,54 @@ cli_xml_output_vol_status (dict_t *dict, int op_ret, int op_errno,
                 case GF_CLI_STATUS_CALLPOOL:
                         if (online) {
                                 ret = cli_xml_output_vol_status_callpool
-                                        (writer, dict, i);
+                                        (local->writer, dict, i);
                                 if (ret)
                                         goto out;
                         }
                         break;
-
                 default:
                         break;
 
                 }
-                /* </brick> */
-                ret = xmlTextWriterEndElement (writer);
+                /* </node> */
+                ret = xmlTextWriterEndElement (local->writer);
                 XML_RET_CHECK_AND_GOTO (ret, out);
         }
 
-        /* </volStatus> */
-        ret = xmlTextWriterEndElement (writer);
-        XML_RET_CHECK_AND_GOTO (ret, out);
+        /* Tasks are only present when a normal volume status call is done on a
+         * single volume or on all volumes
+         */
+        if (((cmd & GF_CLI_STATUS_MASK) == GF_CLI_STATUS_NONE) &&
+            (cmd & (GF_CLI_STATUS_VOL|GF_CLI_STATUS_ALL))) {
+                ret = cli_xml_output_vol_status_tasks (local, dict);
+                if (ret)
+                        goto out;
+        }
 
-        gf_log ("cli", GF_LOG_DEBUG, "Returning %d", ret);
-        ret = cli_end_xml_output (writer, buf);
-        if (ret)
-                goto out;
+        /* </volume> */
+        ret = xmlTextWriterEndElement (local->writer);
+        XML_RET_CHECK_AND_GOTO (ret, out);
 
 out:
         gf_log ("cli", GF_LOG_DEBUG, "Returning %d", ret);
         return ret;
+#else
+        return 0;
+#endif
 }
 
+#if (HAVE_LIB_XML)
 int
 cli_xml_output_vol_top_rw_perf (xmlTextWriterPtr writer, dict_t *dict,
                                 int brick_index, int member_index)
 {
-        int             ret = -1;
-        char            *filename = NULL;
-        uint64_t        throughput = 0;
-        long int        time_sec = 0;
-        long int        time_usec = 0;
-        struct tm       *tm = NULL;
-        char            timestr[256] = {0,};
-        char            key[1024] = {0,};
+        int        ret = -1;
+        char      *filename = NULL;
+        uint64_t   throughput = 0;
+        long int   time_sec = 0;
+        long int   time_usec = 0;
+        char       timestr[256] = {0,};
+        char       key[1024] = {0,};
 
         /* <file> */
         ret = xmlTextWriterStartElement (writer, (xmlChar *)"file");
@@ -1452,14 +1793,9 @@ cli_xml_output_vol_top_rw_perf (xmlTextWriterPtr writer, dict_t *dict,
         if (ret)
                 goto out;
 
-        tm = localtime (&time_sec);
-        if (!tm) {
-                ret = -1;
-                goto out;
-        }
-        strftime (timestr, sizeof (timestr), "%Y-%m-%d %H:%M:%S", tm);
+        gf_time_fmt (timestr, sizeof timestr, time_sec, gf_timefmt_FT);
         snprintf (timestr + strlen (timestr),
-                  sizeof (timestr) - strlen (timestr),
+                  sizeof timestr - strlen (timestr),
                   ".%"GF_PRI_SUSECONDS, time_usec);
         ret = xmlTextWriterWriteFormatElement (writer, (xmlChar *)"time",
                                                "%s", timestr);
@@ -1513,14 +1849,16 @@ out:
         gf_log ("cli", GF_LOG_DEBUG, "Returning %d", ret);
         return ret;
 }
+#endif
 
 int
 cli_xml_output_vol_top (dict_t *dict, int op_ret, int op_errno,
                         char *op_errstr)
 {
+#if (HAVE_LIB_XML)
         int                     ret = -1;
         xmlTextWriterPtr        writer = NULL;
-        xmlBufferPtr            buf = NULL;
+        xmlDocPtr               doc = NULL;
         int                     brick_count = 0;
         int                     top_op = GF_CLI_TOP_NONE;
         char                    *brick_name = NULL;
@@ -1534,7 +1872,7 @@ cli_xml_output_vol_top (dict_t *dict, int op_ret, int op_errno,
         int                     i = 0;
         int                     j = 0;
 
-        ret = cli_begin_xml_output (&writer, &buf);
+        ret = cli_begin_xml_output (&writer, &doc);
         if (ret)
                 goto out;
 
@@ -1623,8 +1961,6 @@ cli_xml_output_vol_top (dict_t *dict, int op_ret, int op_errno,
                 case GF_CLI_TOP_WRITE:
                 case GF_CLI_TOP_OPENDIR:
                 case GF_CLI_TOP_READDIR:
-                        if (!members)
-                                continue;
 
                         break;
 
@@ -1650,9 +1986,6 @@ cli_xml_output_vol_top (dict_t *dict, int op_ret, int op_errno,
                                          "%f", time_taken);
                         }
 
-                        if (!members)
-                                continue;
-
                         break;
 
                 default:
@@ -1682,13 +2015,17 @@ cli_xml_output_vol_top (dict_t *dict, int op_ret, int op_errno,
         /* </volTop> */
         ret = xmlTextWriterEndElement (writer);
         XML_RET_CHECK_AND_GOTO (ret, out);
-        ret = cli_end_xml_output (writer, buf);
+        ret = cli_end_xml_output (writer, doc);
 
 out:
         gf_log ("cli", GF_LOG_DEBUG, "Returning %d", ret);
         return ret;
+#else
+        return 0;
+#endif
 }
 
+#if (HAVE_LIB_XML)
 int
 cli_xml_output_vol_profile_stats (xmlTextWriterPtr writer, dict_t *dict,
                                   int brick_index, int interval)
@@ -1864,14 +2201,16 @@ out:
         gf_log ("cli", GF_LOG_DEBUG, "Returning %d", ret);
         return ret;
 }
+#endif
 
 int
 cli_xml_output_vol_profile (dict_t *dict, int op_ret, int op_errno,
                             char *op_errstr)
 {
+#if (HAVE_LIB_XML)
         int                     ret = -1;
         xmlTextWriterPtr        writer = NULL;
-        xmlBufferPtr            buf = NULL;
+        xmlDocPtr               doc = NULL;
         char                    *volname = NULL;
         int                     op = GF_CLI_STATS_NONE;
         int                     brick_count = 0;
@@ -1880,7 +2219,7 @@ cli_xml_output_vol_profile (dict_t *dict, int op_ret, int op_errno,
         char                    key[1024] = {0,};
         int                     i = 0;
 
-        ret = cli_begin_xml_output (&writer, &buf);
+        ret = cli_begin_xml_output (&writer, &doc);
         if (ret)
                 goto out;
 
@@ -1960,26 +2299,29 @@ cont:
         ret = xmlTextWriterEndElement (writer);
         XML_RET_CHECK_AND_GOTO (ret, out);
 
-        ret = cli_end_xml_output (writer, buf);
+        ret = cli_end_xml_output (writer, doc);
 out:
         gf_log ("cli", GF_LOG_DEBUG, "Returning %d", ret);
         return ret;
-
+#else
+        return 0;
+#endif
 }
 
 int
 cli_xml_output_vol_list (dict_t *dict, int op_ret, int op_errno,
                          char *op_errstr)
 {
+#if (HAVE_LIB_XML)
         int                     ret = -1;
         xmlTextWriterPtr        writer = NULL;
-        xmlBufferPtr            buf = NULL;
+        xmlDocPtr               doc = NULL;
         int                     count = 0;
         char                    *volname = NULL;
         char                    key[1024] = {0,};
         int                     i = 0;
 
-        ret = cli_begin_xml_output (&writer, &buf);
+        ret = cli_begin_xml_output (&writer, &doc);
         if (ret)
                 goto out;
 
@@ -2014,12 +2356,16 @@ cli_xml_output_vol_list (dict_t *dict, int op_ret, int op_errno,
         ret = xmlTextWriterEndElement (writer);
         XML_RET_CHECK_AND_GOTO (ret, out);
 
-        ret = cli_end_xml_output (writer, buf);
+        ret = cli_end_xml_output (writer, doc);
 out:
         gf_log ("cli", GF_LOG_DEBUG, "Returning %d", ret);
         return ret;
+#else
+        return 0;
+#endif
 }
 
+#if (HAVE_LIB_XML)
 int
 cli_xml_output_vol_info_option (xmlTextWriterPtr writer, char *substr,
                                 char *optstr, char *valstr)
@@ -2064,23 +2410,44 @@ out:
         return ret;
 }
 
+struct tmp_xml_option_logger {
+        char *key;
+        xmlTextWriterPtr writer;
+};
+
+static int
+_output_vol_info_option (dict_t *d, char *k, data_t *v,
+                         void *data)
+{
+        int                           ret   = 0;
+        char                         *ptr   = NULL;
+        struct tmp_xml_option_logger *tmp   = NULL;
+
+        tmp = data;
+
+        ptr = strstr (k, "option.");
+        if (!ptr)
+                goto out;
+
+        if (!v) {
+                ret = -1;
+                goto out;
+        }
+        ret = cli_xml_output_vol_info_option (tmp->writer, tmp->key, k,
+                                              v->data);
+
+out:
+        return ret;
+}
+
 int
 cli_xml_output_vol_info_options (xmlTextWriterPtr writer, dict_t *dict,
                                  char *prefix)
 {
         int             ret = -1;
         int             opt_count = 0;
-        data_pair_t     *pairs = 0;
-        data_t          *value = 0;
-        char            *ptr = NULL;
         char            key[1024] = {0,};
-        int             i = 0;
-
-        pairs = dict->members_list;
-        if (!pairs) {
-                ret = -1;
-                goto out;
-        }
+        struct tmp_xml_option_logger tmp = {0,};
 
         snprintf (key, sizeof (key), "%s.opt_count", prefix);
         ret = dict_get_int32 (dict, key, &opt_count);
@@ -2093,25 +2460,14 @@ cli_xml_output_vol_info_options (xmlTextWriterPtr writer, dict_t *dict,
         /* <options> */
         ret = xmlTextWriterStartElement (writer, (xmlChar *)"options");
         XML_RET_CHECK_AND_GOTO (ret, out);
-        while (i < opt_count) {
-                snprintf (key, sizeof (key), "%s.option.", prefix);
-                while (pairs) {
-                        ptr = strstr (pairs->key, "option.");
-                        if (ptr) {
-                                value = pairs->value;
-                                if (!value) {
-                                        ret = -1;
-                                        goto out;
-                                }
-                                ret = cli_xml_output_vol_info_option
-                                        (writer, key, pairs->key, value->data);
-                                if (ret)
-                                        goto out;
-                        }
-                        pairs = pairs->next;
-                }
-                i++;
-        }
+        snprintf (key, sizeof (key), "%s.option.", prefix);
+
+        tmp.key = key;
+        tmp.writer = writer;
+        ret = dict_foreach (dict, _output_vol_info_option, &tmp);
+        if (ret)
+                goto out;
+
         /* </options> */
         ret = xmlTextWriterEndElement (writer);
         XML_RET_CHECK_AND_GOTO (ret, out);
@@ -2119,14 +2475,17 @@ out:
         gf_log ("cli", GF_LOG_DEBUG, "Returning %d", ret);
         return ret;
 }
+#endif
 
 int
 cli_xml_output_vol_info (cli_local_t *local, dict_t *dict)
 {
+#if (HAVE_LIB_XML)
         int                     ret = 0;
         int                     count = 0;
         char                    *volname = NULL;
         char                    *volume_id = NULL;
+        char                    *uuid = NULL;
         int                     type = 0;
         int                     status = 0;
         int                     brick_count = 0;
@@ -2138,7 +2497,9 @@ cli_xml_output_vol_info (cli_local_t *local, dict_t *dict)
         char                    key[1024] = {0,};
         int                     i = 0;
         int                     j = 1;
-
+        char                    *caps = NULL;
+        int                     k __attribute__((unused)) = 0;
+        char                    *snap_volume = NULL;
 
         ret = dict_get_int32 (dict, "count", &count);
         if (ret)
@@ -2171,23 +2532,30 @@ cli_xml_output_vol_info (cli_local_t *local, dict_t *dict)
                 XML_RET_CHECK_AND_GOTO (ret, out);
 
                 memset (key, 0, sizeof (key));
-                snprintf (key, sizeof (key), "volume%d.type", i);
-                ret = dict_get_int32 (dict, key, &type);
+                snprintf (key, sizeof (key), "volume%d.status", i);
+                ret = dict_get_int32 (dict, key, &status);
                 if (ret)
                         goto out;
                 ret = xmlTextWriterWriteFormatElement (local->writer,
-                                                       (xmlChar *)"type",
-                                                       "%d", type);
+                                                       (xmlChar *)"status",
+                                                       "%d", status);
                 XML_RET_CHECK_AND_GOTO (ret, out);
 
                 memset (key, 0, sizeof (key));
-                snprintf (key, sizeof (key), "volume%d.status", i);
-                ret = dict_get_int32 (dict, key, &status);
+                snprintf (key, sizeof (key), "volume%d.snap_volume", i);
+                ret = dict_get_str (dict, key, &snap_volume);
                 if (ret)
                         goto out;
-                ret = xmlTextWriterWriteFormatElement (local->writer,
-                                                       (xmlChar *)"status",
-                                                       "%d", status);
+                if (snap_volume) {
+                        ret = xmlTextWriterWriteFormatElement (local->writer,
+                                                         (xmlChar *)"snapVol",
+                                                          "%s", snap_volume);
+                        XML_RET_CHECK_AND_GOTO (ret, out);
+                }
+
+                ret =xmlTextWriterWriteFormatElement
+                        (local->writer, (xmlChar *)"statusStr", "%s",
+                         cli_vol_status_str[status]);
                 XML_RET_CHECK_AND_GOTO (ret, out);
 
                 memset (key, 0, sizeof (key));
@@ -2231,6 +2599,25 @@ cli_xml_output_vol_info (cli_local_t *local, dict_t *dict)
                 XML_RET_CHECK_AND_GOTO (ret, out);
 
                 memset (key, 0, sizeof (key));
+                snprintf (key, sizeof (key), "volume%d.type", i);
+                ret = dict_get_int32 (dict, key, &type);
+                if (ret)
+                        goto out;
+                /* For Distributed-(stripe,replicate,stipe-replicate) types */
+                if ((type > 0) && (dist_count < brick_count))
+                        type += 3;
+                ret = xmlTextWriterWriteFormatElement (local->writer,
+                                                       (xmlChar *)"type",
+                                                       "%d", type);
+                XML_RET_CHECK_AND_GOTO (ret, out);
+
+                ret = xmlTextWriterWriteFormatElement (local->writer,
+                                                       (xmlChar *)"typeStr",
+                                                       "%s",
+                                                       cli_vol_type_str[type]);
+                XML_RET_CHECK_AND_GOTO (ret, out);
+
+                memset (key, 0, sizeof (key));
                 snprintf (key, sizeof (key), "volume%d.transport", i);
                 ret = dict_get_int32 (dict, key, &transport);
                 if (ret)
@@ -2240,20 +2627,95 @@ cli_xml_output_vol_info (cli_local_t *local, dict_t *dict)
                                                        "%d", transport);
                 XML_RET_CHECK_AND_GOTO (ret, out);
 
+#ifdef HAVE_BD_XLATOR
+                /* <xlators> */
+                ret = xmlTextWriterStartElement (local->writer,
+                                                 (xmlChar *)"xlators");
+                XML_RET_CHECK_AND_GOTO (ret, out);
+
+                for (k = 0; ; k++) {
+                        memset (key, 0, sizeof (key));
+                        snprintf (key, sizeof (key),"volume%d.xlator%d", i, k);
+                        ret = dict_get_str (dict, key, &caps);
+                        if (ret)
+                                break;
+
+                        /* <xlator> */
+                        ret = xmlTextWriterStartElement (local->writer,
+                                                         (xmlChar *)"xlator");
+                        XML_RET_CHECK_AND_GOTO (ret, out);
+
+                        ret = xmlTextWriterWriteFormatElement
+                                (local->writer, (xmlChar *)"name", "%s", caps);
+                        XML_RET_CHECK_AND_GOTO (ret, out);
+
+                        /* <capabilities> */
+                        ret = xmlTextWriterStartElement (local->writer,
+                                                         (xmlChar *)
+                                                         "capabilities");
+                        XML_RET_CHECK_AND_GOTO (ret, out);
+
+                        j = 0;
+                        for (j = 0; ;j++) {
+                                memset (key, 0, sizeof (key));
+                                snprintf (key, sizeof (key),
+                                          "volume%d.xlator%d.caps%d", i, k, j);
+                                ret = dict_get_str (dict, key, &caps);
+                                if (ret)
+                                        break;
+                                ret = xmlTextWriterWriteFormatElement
+                                        (local->writer, (xmlChar *)"capability",
+                                         "%s", caps);
+                                XML_RET_CHECK_AND_GOTO (ret, out);
+                        }
+                        /* </capabilities> */
+                        ret = xmlTextWriterEndElement (local->writer);
+                        XML_RET_CHECK_AND_GOTO (ret, out);
+                        /* </xlator> */
+                        ret = xmlTextWriterEndElement (local->writer);
+                        XML_RET_CHECK_AND_GOTO (ret, out);
+                }
+                ret = xmlTextWriterFullEndElement (local->writer);
+                XML_RET_CHECK_AND_GOTO (ret, out);
+                /* </xlators> */
+#else
+                caps = 0; /* Avoid compiler warnings when BD not enabled */
+#endif
+                j = 1;
+
                 /* <bricks> */
                 ret = xmlTextWriterStartElement (local->writer,
                                                  (xmlChar *)"bricks");
                 XML_RET_CHECK_AND_GOTO (ret, out);
                 while (j <= brick_count) {
+                        ret = xmlTextWriterStartElement
+                                (local->writer, (xmlChar *)"brick");
+                        XML_RET_CHECK_AND_GOTO (ret, out);
+
+                        memset (key, 0, sizeof (key));
+                        snprintf (key, sizeof (key), "volume%d.brick%d.uuid",
+                                  i, j);
+                        ret = dict_get_str (dict, key, &uuid);
+                        if (ret)
+                                goto out;
+                        ret = xmlTextWriterWriteFormatAttribute
+                                (local->writer, (xmlChar *)"uuid", "%s",
+                                 uuid);
+                        XML_RET_CHECK_AND_GOTO (ret, out);
+
                         memset (key, 0, sizeof (key));
                         snprintf (key, sizeof (key), "volume%d.brick%d", i, j);
                         ret = dict_get_str (dict, key, &brick);
                         if (ret)
                                 goto out;
-                        ret = xmlTextWriterWriteFormatElement
-                                (local->writer, (xmlChar *)"brick", "%s",
-                                 brick);
+                        ret = xmlTextWriterWriteFormatString
+                                (local->writer, "%s", brick);
+                        XML_RET_CHECK_AND_GOTO (ret, out);
+
+                        /* </brick> */
+                        ret = xmlTextWriterEndElement (local->writer);
                         XML_RET_CHECK_AND_GOTO (ret, out);
+
                         j++;
                 }
                 /* </bricks> */
@@ -2271,24 +2733,30 @@ cli_xml_output_vol_info (cli_local_t *local, dict_t *dict)
                 ret = xmlTextWriterEndElement (local->writer);
                 XML_RET_CHECK_AND_GOTO (ret, out);
         }
-        GF_FREE (local->get_vol.volname);
-        local->get_vol.volname = gf_strdup (volname);
-        local->vol_count += count;
 
+        if (volname) {
+                GF_FREE (local->get_vol.volname);
+                local->get_vol.volname = gf_strdup (volname);
+                local->vol_count += count;
+        }
 out:
         gf_log ("cli", GF_LOG_DEBUG, "Returning %d", ret);
         return ret;
+#else
+        return 0;
+#endif
 }
 
 int
 cli_xml_output_vol_info_begin (cli_local_t *local, int op_ret, int op_errno,
                                char *op_errstr)
 {
+#if (HAVE_LIB_XML)
         int                     ret = -1;
 
         GF_ASSERT (local);
 
-        ret = cli_begin_xml_output (&(local->writer), &(local->buf));
+        ret = cli_begin_xml_output (&(local->writer), &(local->doc));
         if (ret)
                 goto out;
 
@@ -2311,11 +2779,15 @@ cli_xml_output_vol_info_begin (cli_local_t *local, int op_ret, int op_errno,
 out:
         gf_log ("cli", GF_LOG_DEBUG, "Returning %d", ret);
         return ret;
+#else
+        return 0;
+#endif
 }
 
 int
 cli_xml_output_vol_info_end (cli_local_t *local)
 {
+#if (HAVE_LIB_XML)
         int             ret = -1;
 
         GF_ASSERT (local);
@@ -2332,11 +2804,14 @@ cli_xml_output_vol_info_end (cli_local_t *local)
         ret = xmlTextWriterEndElement (local->writer);
         XML_RET_CHECK_AND_GOTO (ret, out);
 
-        ret = cli_end_xml_output (local->writer, local->buf);
+        ret = cli_end_xml_output (local->writer, local->doc);
 
 out:
         gf_log ("cli", GF_LOG_ERROR, "Returning %d", ret);
         return ret;
+#else
+        return 0;
+#endif
 }
 
 int
@@ -2344,9 +2819,10 @@ cli_xml_output_vol_quota_limit_list (char *volname, char *limit_list,
                                      int op_ret, int op_errno,
                                      char *op_errstr)
 {
+#if (HAVE_LIB_XML)
         int                     ret = -1;
         xmlTextWriterPtr        writer = NULL;
-        xmlBufferPtr            buf = NULL;
+        xmlDocPtr               doc = NULL;
         int64_t                 size = 0;
         int64_t                 limit_value = 0;
         int                     i = 0;
@@ -2364,7 +2840,7 @@ cli_xml_output_vol_quota_limit_list (char *volname, char *limit_list,
         GF_ASSERT (volname);
         GF_ASSERT (limit_list);
 
-        ret = cli_begin_xml_output (&writer, &buf);
+        ret = cli_begin_xml_output (&writer, &doc);
         if (ret)
                 goto out;
 
@@ -2474,13 +2950,823 @@ cont:
         ret = xmlTextWriterEndElement (writer);
         XML_RET_CHECK_AND_GOTO (ret, out);
 
-        ret = cli_end_xml_output (writer, buf);
+        ret = cli_end_xml_output (writer, doc);
 
 out:
-        if (size_str)
-                GF_FREE (size_str);
+        GF_FREE (size_str);
         gf_log ("cli", GF_LOG_DEBUG, "Returning %d", ret);
         return ret;
+#else
+        return 0;
+#endif
 }
 
+int
+cli_xml_output_peer_status (dict_t *dict, int op_ret, int op_errno,
+                            char *op_errstr)
+{
+#if (HAVE_LIB_XML)
+        int                     ret = -1;
+        xmlTextWriterPtr        writer = NULL;
+        xmlDocPtr               doc = NULL;
+        int                     count = 0;
+        char                    *uuid = NULL;
+        char                    *hostname = NULL;
+        int                     connected = 0;
+        int                     state_id = 0;
+        char                    *state_str = NULL;
+        int                     i = 1;
+        char                    key[1024] = {0,};
+
+        ret = cli_begin_xml_output (&writer, &doc);
+        if (ret)
+                goto out;
+
+        ret = cli_xml_output_common (writer, op_ret, op_errno, op_errstr);
+        if (ret)
+                goto out;
+
+        /* <peerStatus> */
+        ret = xmlTextWriterStartElement (writer, (xmlChar *)"peerStatus");
+        XML_RET_CHECK_AND_GOTO (ret, out);
+
+        if (!dict)
+                goto cont;
+
+        ret = dict_get_int32 (dict, "count", &count);
+        if (ret)
+                goto out;
+
+        while (i <= count) {
+                /* <peer> */
+                ret = xmlTextWriterStartElement (writer, (xmlChar *)"peer");
+                XML_RET_CHECK_AND_GOTO (ret, out);
+
+                memset (key, 0, sizeof (key));
+                snprintf (key, sizeof (key), "friend%d.uuid", i);
+                ret = dict_get_str (dict, key, &uuid);
+                if (ret)
+                        goto out;
+
+                ret = xmlTextWriterWriteFormatElement (writer,
+                                                       (xmlChar *)"uuid",
+                                                       "%s", uuid);
+                XML_RET_CHECK_AND_GOTO (ret, out);
+
+                memset (key, 0, sizeof (key));
+                snprintf (key, sizeof (key), "friend%d.hostname", i);
+                ret = dict_get_str (dict, key, &hostname);
+                if (ret)
+                        goto out;
+
+                ret = xmlTextWriterWriteFormatElement (writer,
+                                                       (xmlChar *)"hostname",
+                                                       "%s", hostname);
+                XML_RET_CHECK_AND_GOTO (ret, out);
+
+                memset (key, 0, sizeof (key));
+                snprintf (key, sizeof (key), "friend%d.connected", i);
+                ret = dict_get_int32 (dict, key, &connected);
+                if (ret)
+                        goto out;
+
+                ret = xmlTextWriterWriteFormatElement (writer,
+                                                       (xmlChar *)"connected",
+                                                       "%d", connected);
+                XML_RET_CHECK_AND_GOTO (ret, out);
+
+                memset (key, 0, sizeof (key));
+                snprintf (key, sizeof (key), "friend%d.stateId", i);
+                ret = dict_get_int32 (dict, key, &state_id);
+                if (!ret) {
+                        /* ignore */
+
+                        ret = xmlTextWriterWriteFormatElement (writer,
+                                           (xmlChar *)"state", "%d", state_id);
+                        XML_RET_CHECK_AND_GOTO (ret, out);
+                }
+
+                memset (key, 0, sizeof (key));
+                snprintf (key, sizeof (key), "friend%d.state", i);
+                ret = dict_get_str (dict, key, &state_str);
+                if (!ret) {
+                        /* ignore */
+
+                        ret = xmlTextWriterWriteFormatElement (writer,
+                                       (xmlChar *)"stateStr", "%s", state_str);
+                        XML_RET_CHECK_AND_GOTO (ret, out);
+                }
+
+                /* </peer> */
+                ret = xmlTextWriterEndElement (writer);
+                XML_RET_CHECK_AND_GOTO (ret, out);
+
+                i++;
+        }
+
+cont:
+        /* </peerStatus> */
+        ret = xmlTextWriterEndElement (writer);
+        XML_RET_CHECK_AND_GOTO (ret, out);
+
+        ret = cli_end_xml_output (writer, doc);
+
+out:
+        gf_log ("cli", GF_LOG_DEBUG, "Returning %d", ret);
+        return ret;
+#else
+        return 0;
 #endif
+}
+
+#if (HAVE_LIB_XML)
+/* Used for rebalance stop/status, remove-brick status */
+int
+cli_xml_output_vol_rebalance_status (xmlTextWriterPtr writer, dict_t *dict,
+                                     enum gf_task_types task_type)
+{
+        int                     ret = -1;
+        int                     count = 0;
+        char                    *node_name = NULL;
+        char                    *node_uuid = NULL;
+        uint64_t                files = 0;
+        uint64_t                size = 0;
+        uint64_t                lookups = 0;
+        int                     status_rcd = 0;
+        uint64_t                failures = 0;
+        uint64_t                skipped = 0;
+        uint64_t                total_files = 0;
+        uint64_t                total_size = 0;
+        uint64_t                total_lookups = 0;
+        uint64_t                total_failures = 0;
+        uint64_t                total_skipped = 0;
+        char                    key[1024] = {0,};
+        int                     i = 0;
+        int                     overall_status = -1;
+        double                  elapsed = 0;
+        double                  overall_elapsed = 0;
+
+        if (!dict) {
+                ret = 0;
+                goto out;
+        }
+
+        ret = dict_get_int32 (dict, "count", &count);
+        if (ret)
+                goto out;
+        ret = xmlTextWriterWriteFormatElement (writer, (xmlChar *)"nodeCount",
+                                               "%d", count);
+        XML_RET_CHECK_AND_GOTO (ret, out);
+
+        while (i < count) {
+                i++;
+
+                /* <node> */
+                ret = xmlTextWriterStartElement (writer, (xmlChar *)"node");
+                XML_RET_CHECK_AND_GOTO (ret, out);
+
+                memset (key, 0, sizeof (key));
+                snprintf (key, sizeof (key), "node-name-%d", i);
+                ret = dict_get_str (dict, key, &node_name);
+                if (ret)
+                        goto out;
+                ret = xmlTextWriterWriteFormatElement (writer,
+                                                       (xmlChar *)"nodeName",
+                                                       "%s", node_name);
+
+                memset (key, 0, sizeof (key));
+                snprintf (key, sizeof (key), "node-uuid-%d", i);
+                ret = dict_get_str (dict, key, &node_uuid);
+                if (ret)
+                        goto out;
+                ret = xmlTextWriterWriteFormatElement (writer,
+                                                       (xmlChar *)"id",
+                                                       "%s", node_uuid);
+                XML_RET_CHECK_AND_GOTO (ret, out);
+
+                memset (key, 0, sizeof (key));
+                snprintf (key, sizeof (key), "files-%d", i);
+                ret = dict_get_uint64 (dict, key, &files);
+                if (ret)
+                        goto out;
+                total_files += files;
+                ret = xmlTextWriterWriteFormatElement (writer,
+                                                       (xmlChar *)"files",
+                                                       "%"PRIu64, files);
+                XML_RET_CHECK_AND_GOTO (ret, out);
+
+                memset (key, 0, sizeof (key));
+                snprintf (key, sizeof (key), "size-%d", i);
+                ret = dict_get_uint64 (dict, key, &size);
+                if (ret)
+                        goto out;
+                total_size += size;
+                ret = xmlTextWriterWriteFormatElement (writer,
+                                                       (xmlChar *)"size",
+                                                       "%"PRIu64,size);
+                XML_RET_CHECK_AND_GOTO (ret, out);
+
+                memset (key, 0, sizeof (key));
+                snprintf (key, sizeof (key), "lookups-%d", i);
+                ret = dict_get_uint64 (dict, key, &lookups);
+                if (ret)
+                        goto out;
+                total_lookups += lookups;
+                ret = xmlTextWriterWriteFormatElement (writer,
+                                                       (xmlChar *)"lookups",
+                                                       "%"PRIu64, lookups);
+                XML_RET_CHECK_AND_GOTO (ret, out);
+
+                memset (key, 0, sizeof (key));
+                snprintf (key, sizeof (key), "failures-%d", i);
+                ret = dict_get_uint64 (dict, key, &failures);
+                if (ret)
+                        goto out;
+                total_failures += failures;
+                ret = xmlTextWriterWriteFormatElement (writer,
+                                                       (xmlChar *)"failures",
+                                                       "%"PRIu64, failures);
+                XML_RET_CHECK_AND_GOTO (ret, out);
+
+                /* skipped-%d is not available for remove brick in dict,
+                   so using failures as skipped count in case of remove-brick
+                   similar to logic used in CLI(non xml output) */
+                if (task_type == GF_TASK_TYPE_REBALANCE) {
+                    memset (key, 0, sizeof (key));
+                    snprintf (key, sizeof (key), "skipped-%d", i);
+                }
+                else {
+                    memset (key, 0, sizeof (key));
+                    snprintf (key, sizeof (key), "failures-%d", i);
+                }
+
+                ret = dict_get_uint64 (dict, key, &skipped);
+                if (ret)
+                        goto out;
+                total_skipped += skipped;
+                ret = xmlTextWriterWriteFormatElement (writer,
+                                                       (xmlChar *)"skipped",
+                                                       "%"PRIu64, skipped);
+                XML_RET_CHECK_AND_GOTO (ret, out);
+
+                memset (key, 0, sizeof (key));
+                snprintf (key, sizeof (key), "status-%d", i);
+                ret = dict_get_int32 (dict, key, &status_rcd);
+                if (ret)
+                        goto out;
+                ret = xmlTextWriterWriteFormatElement (writer,
+                                                       (xmlChar *)"status",
+                                                       "%d", status_rcd);
+                XML_RET_CHECK_AND_GOTO (ret, out);
+
+                ret = xmlTextWriterWriteFormatElement (writer,
+                                                       (xmlChar *)"statusStr",
+                                                       "%s",
+                                         cli_vol_task_status_str[status_rcd]);
+
+                memset (key, 0, 256);
+                snprintf (key, 256, "run-time-%d", i);
+                ret = dict_get_double (dict, key, &elapsed);
+                if (ret)
+                        goto out;
+                ret = xmlTextWriterWriteFormatElement (writer,
+                                                       (xmlChar *)"runtime",
+                                                       "%.2f", elapsed);
+                XML_RET_CHECK_AND_GOTO (ret, out);
+
+                if (elapsed > overall_elapsed) {
+                    overall_elapsed = elapsed;
+                }
+
+                if (-1 == overall_status)
+                        overall_status = status_rcd;
+                else if ((GF_DEFRAG_STATUS_COMPLETE == overall_status ||
+                          status_rcd > overall_status) &&
+                         (status_rcd != GF_DEFRAG_STATUS_COMPLETE))
+                        overall_status = status_rcd;
+                XML_RET_CHECK_AND_GOTO (ret, out);
+
+                /* </node> */
+                ret = xmlTextWriterEndElement (writer);
+                XML_RET_CHECK_AND_GOTO (ret, out);
+        }
+
+        /* Aggregate status */
+        /* <aggregate> */
+        ret = xmlTextWriterStartElement (writer, (xmlChar *)"aggregate");
+        XML_RET_CHECK_AND_GOTO (ret, out);
+
+        ret = xmlTextWriterWriteFormatElement (writer,(xmlChar *)"files",
+                                               "%"PRIu64, total_files);
+        XML_RET_CHECK_AND_GOTO (ret, out);
+
+        ret = xmlTextWriterWriteFormatElement (writer,(xmlChar *)"size",
+                                               "%"PRIu64, total_size);
+        XML_RET_CHECK_AND_GOTO (ret, out);
+
+        ret = xmlTextWriterWriteFormatElement (writer,(xmlChar *)"lookups",
+                                               "%"PRIu64, total_lookups);
+        XML_RET_CHECK_AND_GOTO (ret, out);
+
+        ret = xmlTextWriterWriteFormatElement (writer,(xmlChar *)"failures",
+                                               "%"PRIu64, total_failures);
+        XML_RET_CHECK_AND_GOTO (ret, out);
+
+        ret = xmlTextWriterWriteFormatElement (writer,(xmlChar *)"skipped",
+                                               "%"PRIu64, total_skipped);
+        XML_RET_CHECK_AND_GOTO (ret, out);
+
+        ret = xmlTextWriterWriteFormatElement (writer,(xmlChar *)"status",
+                                               "%d", overall_status);
+        XML_RET_CHECK_AND_GOTO (ret, out);
+
+        ret = xmlTextWriterWriteFormatElement (writer,(xmlChar *)"statusStr",
+                                               "%s",
+                                      cli_vol_task_status_str[overall_status]);
+        XML_RET_CHECK_AND_GOTO (ret, out);
+
+        ret = xmlTextWriterWriteFormatElement (writer,(xmlChar *)"runtime",
+                                               "%.2f", overall_elapsed);
+        XML_RET_CHECK_AND_GOTO (ret, out);
+
+        /* </aggregate> */
+        ret = xmlTextWriterEndElement (writer);
+        XML_RET_CHECK_AND_GOTO (ret, out);
+
+out:
+        gf_log ("cli", GF_LOG_DEBUG, "Returning %d", ret);
+        return ret;
+}
+#endif
+
+int
+cli_xml_output_vol_rebalance (gf_cli_defrag_type op, dict_t *dict, int op_ret,
+                              int op_errno, char *op_errstr)
+{
+#if (HAVE_LIB_XML)
+        int                     ret = -1;
+        xmlTextWriterPtr        writer = NULL;
+        xmlDocPtr               doc = NULL;
+        char                    *task_id_str = NULL;
+
+        ret = cli_begin_xml_output (&writer, &doc);
+        if (ret)
+                goto out;
+
+        ret = cli_xml_output_common (writer, op_ret, op_errno, op_errstr);
+        if (ret)
+                goto out;
+
+        /* <volRebalance> */
+        ret = xmlTextWriterStartElement (writer, (xmlChar *)"volRebalance");
+        XML_RET_CHECK_AND_GOTO (ret, out);
+
+        ret = dict_get_str (dict, GF_REBALANCE_TID_KEY, &task_id_str);
+        if (ret == 0) {
+                ret = xmlTextWriterWriteFormatElement (writer,
+                                                       (xmlChar *)"task-id",
+                                                       "%s", task_id_str);
+                XML_RET_CHECK_AND_GOTO (ret, out);
+        }
+
+        ret = xmlTextWriterWriteFormatElement (writer, (xmlChar *)"op",
+                                               "%d", op);
+        XML_RET_CHECK_AND_GOTO (ret, out);
+
+        if ((GF_DEFRAG_CMD_STOP == op) || (GF_DEFRAG_CMD_STATUS == op)) {
+                ret = cli_xml_output_vol_rebalance_status (writer, dict,
+                                                      GF_TASK_TYPE_REBALANCE);
+                if (ret)
+                        goto out;
+        }
+
+        /* </volRebalance> */
+        ret = xmlTextWriterEndElement (writer);
+        XML_RET_CHECK_AND_GOTO (ret, out);
+
+
+        ret = cli_end_xml_output (writer, doc);
+
+out:
+        gf_log ("cli", GF_LOG_DEBUG, "Returning %d", ret);
+        return ret;
+#else
+        return 0;
+#endif
+}
+
+int
+cli_xml_output_vol_remove_brick (gf_boolean_t status_op, dict_t *dict,
+                                 int op_ret, int op_errno, char *op_errstr)
+{
+#if (HAVE_LIB_XML)
+        int                     ret = -1;
+        xmlTextWriterPtr        writer = NULL;
+        xmlDocPtr               doc = NULL;
+        char                    *task_id_str = NULL;
+
+        ret = cli_begin_xml_output (&writer, &doc);
+        if (ret)
+                goto out;
+
+        ret = cli_xml_output_common (writer, op_ret, op_errno, op_errstr);
+        if (ret)
+                goto out;
+
+        /* <volRemoveBrick> */
+        ret = xmlTextWriterStartElement (writer, (xmlChar *)"volRemoveBrick");
+        XML_RET_CHECK_AND_GOTO (ret, out);
+
+        ret = dict_get_str (dict, GF_REMOVE_BRICK_TID_KEY, &task_id_str);
+        if (ret == 0) {
+                ret = xmlTextWriterWriteFormatElement (writer,
+                                                       (xmlChar *)"task-id",
+                                                       "%s", task_id_str);
+                XML_RET_CHECK_AND_GOTO (ret, out);
+        }
+
+        if (status_op) {
+                ret = cli_xml_output_vol_rebalance_status (writer, dict,
+                                                   GF_TASK_TYPE_REMOVE_BRICK);
+                if (ret)
+                        goto out;
+        }
+
+        /* </volRemoveBrick> */
+        ret = xmlTextWriterEndElement (writer);
+        XML_RET_CHECK_AND_GOTO (ret, out);
+
+
+        ret = cli_end_xml_output (writer, doc);
+
+out:
+        gf_log ("cli", GF_LOG_DEBUG, "Returning %d", ret);
+        return ret;
+#else
+        return 0;
+#endif
+}
+
+int
+cli_xml_output_vol_replace_brick (gf1_cli_replace_op op, dict_t *dict,
+                                  int op_ret, int op_errno, char *op_errstr)
+{
+#if (HAVE_LIB_XML)
+        int                     ret = -1;
+        int                     status = 0;
+        uint64_t                files = 0;
+        char                    *current_file = 0;
+        char                    *task_id_str = NULL;
+        xmlTextWriterPtr        writer = NULL;
+        xmlDocPtr               doc = NULL;
+
+        ret = cli_begin_xml_output (&writer, &doc);
+        if (ret)
+                goto out;
+
+        ret = cli_xml_output_common (writer, op_ret, op_errno, op_errstr);
+        if (ret)
+                goto out;
+
+        /* <volReplaceBrick> */
+        ret = xmlTextWriterStartElement (writer, (xmlChar *)"volReplaceBrick");
+        XML_RET_CHECK_AND_GOTO (ret, out);
+
+        ret = dict_get_str (dict, GF_REPLACE_BRICK_TID_KEY, &task_id_str);
+        if (ret == 0) {
+                ret = xmlTextWriterWriteFormatElement (writer,
+                                                       (xmlChar *)"task-id",
+                                                       "%s", task_id_str);
+                XML_RET_CHECK_AND_GOTO (ret, out);
+        }
+
+        ret = xmlTextWriterWriteFormatElement (writer, (xmlChar *)"op",
+                                               "%d", op);
+        XML_RET_CHECK_AND_GOTO (ret, out);
+
+        if (GF_REPLACE_OP_STATUS == op) {
+                ret = dict_get_int32 (dict, "status", &status);
+                if (ret)
+                        goto out;
+                ret = xmlTextWriterWriteFormatElement (writer,
+                                                       (xmlChar *)"status",
+                                                       "%d", status);
+                XML_RET_CHECK_AND_GOTO (ret, out);
+
+                ret = dict_get_uint64 (dict, "files", &files);
+                if (ret)
+                        goto out;
+                ret = xmlTextWriterWriteFormatElement (writer,
+                                                       (xmlChar *)"files",
+                                                       "%"PRIu64, files);
+                XML_RET_CHECK_AND_GOTO (ret, out);
+
+                if (status)
+                        goto cont;
+
+                ret = dict_get_str (dict, "current_file", &current_file);
+                if (ret)
+                        goto out;
+                ret = xmlTextWriterWriteFormatElement (writer,
+                                                       (xmlChar *)"currentFile",
+                                                       "%s", current_file);
+                XML_RET_CHECK_AND_GOTO (ret, out);
+        }
+cont:
+        /* </volReplaceBrick> */
+        ret = xmlTextWriterEndElement (writer);
+        XML_RET_CHECK_AND_GOTO (ret, out);
+
+        ret = cli_end_xml_output (writer, doc);
+
+out:
+        gf_log ("cli", GF_LOG_DEBUG, "Returning %d", ret);
+        return ret;
+#else
+        return 0;
+#endif
+}
+
+int
+cli_xml_output_vol_create (dict_t *dict, int op_ret, int op_errno,
+                           char *op_errstr)
+{
+#if (HAVE_LIB_XML)
+        int                     ret = -1;
+        xmlTextWriterPtr        writer = NULL;
+        xmlDocPtr               doc = NULL;
+        char                    *volname = NULL;
+        char                    *volid = NULL;
+
+        ret = cli_begin_xml_output (&writer, &doc);
+        if (ret)
+                goto out;
+
+        ret = cli_xml_output_common (writer, op_ret, op_errno, op_errstr);
+        if (ret)
+                goto out;
+
+        if (dict) {
+                /* <volCreate> */
+                ret = xmlTextWriterStartElement (writer,
+                                                 (xmlChar *)"volCreate");
+                XML_RET_CHECK_AND_GOTO (ret, out);
+
+                /* <volume> */
+                ret = xmlTextWriterStartElement (writer, (xmlChar *)"volume");
+                XML_RET_CHECK_AND_GOTO (ret, out);
+
+                ret = dict_get_str (dict, "volname", &volname);
+                if (ret)
+                        goto out;
+                ret = xmlTextWriterWriteFormatElement (writer,
+                                                       (xmlChar *) "name",
+                                                       "%s", volname);
+                XML_RET_CHECK_AND_GOTO (ret, out);
+
+                ret = dict_get_str (dict, "volume-id", &volid);
+                if (ret)
+                        goto out;
+                ret = xmlTextWriterWriteFormatElement (writer, (xmlChar *)"id",
+                                                       "%s", volid);
+                XML_RET_CHECK_AND_GOTO (ret, out);
+
+                /* </volume> */
+                ret = xmlTextWriterEndElement (writer);
+                XML_RET_CHECK_AND_GOTO (ret, out);
+
+                /* </volCreate> */
+                ret = xmlTextWriterEndElement (writer);
+                XML_RET_CHECK_AND_GOTO (ret, out);
+        }
+
+        ret = cli_end_xml_output (writer, doc);
+
+out:
+        gf_log ("cli", GF_LOG_DEBUG, "Returning %d", ret);
+        return ret;
+#else
+        return 0;
+#endif
+}
+
+int
+cli_xml_output_generic_volume (char *op, dict_t *dict, int op_ret, int op_errno,
+                               char *op_errstr)
+{
+#if (HAVE_LIB_XML)
+        int                     ret = -1;
+        xmlTextWriterPtr        writer = NULL;
+        xmlDocPtr               doc = NULL;
+        char                    *volname = NULL;
+        char                    *volid = NULL;
+
+        GF_ASSERT (op);
+
+        ret = cli_begin_xml_output (&writer, &doc);
+        if (ret)
+                goto out;
+
+        ret = cli_xml_output_common (writer, op_ret, op_errno, op_errstr);
+        if (ret)
+                goto out;
+
+        if (dict) {
+                /* <"op"> */
+                ret = xmlTextWriterStartElement (writer, (xmlChar *)op);
+                XML_RET_CHECK_AND_GOTO (ret, out);
+
+                /* <volume> */
+                ret = xmlTextWriterStartElement (writer, (xmlChar *)"volume");
+                XML_RET_CHECK_AND_GOTO (ret, out);
+
+                ret = dict_get_str (dict, "volname", &volname);
+                if (ret)
+                        goto out;
+                ret = xmlTextWriterWriteFormatElement (writer,
+                                                       (xmlChar *) "name",
+                                                       "%s", volname);
+                XML_RET_CHECK_AND_GOTO (ret, out);
+
+                ret = dict_get_str (dict, "vol-id", &volid);
+                if (ret)
+                        goto out;
+                ret = xmlTextWriterWriteFormatElement (writer, (xmlChar *)"id",
+                                                       "%s", volid);
+                XML_RET_CHECK_AND_GOTO (ret, out);
+
+                /* </volume> */
+                ret = xmlTextWriterEndElement (writer);
+                XML_RET_CHECK_AND_GOTO (ret, out);
+
+                /* </"op"> */
+                ret = xmlTextWriterEndElement (writer);
+                XML_RET_CHECK_AND_GOTO (ret, out);
+        }
+
+        ret = cli_end_xml_output (writer, doc);
+
+out:
+        gf_log ("cli", GF_LOG_DEBUG, "Returning %d", ret);
+        return ret;
+#else
+        return 0;
+#endif
+}
+
+#if (HAVE_LIB_XML)
+int
+cli_xml_output_vol_gsync_status (dict_t *dict, xmlTextWriterPtr writer)
+{
+        char                    master_key[PATH_MAX] = "";
+        char                    slave_key[PATH_MAX] = "";
+        char                    status_key[PATH_MAX] = "";
+        char                    node_key[PATH_MAX] = "";
+        char                    *master = NULL;
+        char                    *slave = NULL;
+        char                    *status = NULL;
+        char                    *node = NULL;
+        int                     ret = -1;
+        int                     gsync_count = 0;
+        int                     i = 1;
+
+        ret = dict_get_int32 (dict, "gsync-count", &gsync_count);
+        if (ret)
+                goto out;
+
+        for (i=1; i <= gsync_count; i++) {
+                snprintf (node_key, sizeof(node_key), "node%d", i);
+                snprintf (master_key, sizeof(master_key), "master%d", i);
+                snprintf (slave_key, sizeof(slave_key), "slave%d", i);
+                snprintf (status_key, sizeof(status_key), "status%d", i);
+
+                ret = dict_get_str (dict, node_key, &node);
+                if (ret)
+                        goto out;
+
+                ret = dict_get_str (dict, master_key, &master);
+                if (ret)
+                        goto out;
+
+                ret = dict_get_str (dict, slave_key, &slave);
+                if (ret)
+                        goto out;
+
+                ret = dict_get_str (dict, status_key, &status);
+                if (ret)
+                        goto out;
+
+                /* <pair> */
+                ret = xmlTextWriterStartElement (writer, (xmlChar *)"pair");
+                XML_RET_CHECK_AND_GOTO (ret, out);
+
+                ret = xmlTextWriterWriteFormatElement (writer,
+                                                       (xmlChar *)"node",
+                                                       "%s", node);
+                XML_RET_CHECK_AND_GOTO (ret, out);
+
+                ret = xmlTextWriterWriteFormatElement (writer,
+                                                       (xmlChar *)"master",
+                                                       "%s", master);
+                XML_RET_CHECK_AND_GOTO (ret, out);
+
+                ret = xmlTextWriterWriteFormatElement (writer,
+                                                       (xmlChar *)"slave",
+                                                       "%s", slave);
+                XML_RET_CHECK_AND_GOTO (ret, out);
+
+                ret = xmlTextWriterWriteFormatElement (writer,
+                                                       (xmlChar *)"status",
+                                                       "%s", status);
+                XML_RET_CHECK_AND_GOTO (ret, out);
+
+                /* </pair> */
+                ret = xmlTextWriterEndElement (writer);
+                XML_RET_CHECK_AND_GOTO (ret, out);
+
+        }
+
+out:
+        gf_log ("cli",GF_LOG_DEBUG, "Returning %d", ret);
+        return ret;
+}
+#endif
+
+int
+cli_xml_output_vol_gsync (dict_t *dict, int op_ret, int op_errno,
+                          char *op_errstr)
+{
+#if (HAVE_LIB_XML)
+        int                     ret = -1;
+        xmlTextWriterPtr        writer = NULL;
+        xmlDocPtr               doc = NULL;
+        char                    *master = NULL;
+        char                    *slave = NULL;
+        int                     type = 0;
+
+        GF_ASSERT (dict);
+
+        ret = cli_begin_xml_output (&writer, &doc);
+        if (ret)
+                goto out;
+
+        ret = cli_xml_output_common (writer, op_ret, op_errno, op_errstr);
+        if (ret)
+                goto out;
+
+        /* <geoRep> */
+        ret = xmlTextWriterStartElement (writer, (xmlChar *)"geoRep");
+        XML_RET_CHECK_AND_GOTO (ret, out);
+
+        ret = dict_get_int32 (dict, "type", &type);
+        if (ret) {
+                gf_log ("cli", GF_LOG_ERROR, "Failed to get type");
+                goto out;
+        }
+
+        ret = xmlTextWriterWriteFormatElement (writer, (xmlChar *)"type",
+                                               "%d", type);
+        XML_RET_CHECK_AND_GOTO (ret, out);
+
+        switch (type) {
+        case GF_GSYNC_OPTION_TYPE_START:
+        case GF_GSYNC_OPTION_TYPE_STOP:
+                if (dict_get_str (dict, "master", &master) != 0)
+                        master = "???";
+                if (dict_get_str (dict, "slave", &slave) != 0)
+                        slave = "???";
+
+                ret = xmlTextWriterWriteFormatElement (writer,
+                                                       (xmlChar *)"master",
+                                                       "%s", master);
+                XML_RET_CHECK_AND_GOTO (ret, out);
+
+                ret = xmlTextWriterWriteFormatElement (writer,
+                                                       (xmlChar *)"slave",
+                                                       "%s", slave);
+                XML_RET_CHECK_AND_GOTO (ret, out);
+
+                break;
+
+        case GF_GSYNC_OPTION_TYPE_CONFIG:
+                break;
+        case GF_GSYNC_OPTION_TYPE_STATUS:
+                ret = cli_xml_output_vol_gsync_status(dict, writer);
+                break;
+        default:
+                ret = 0;
+                break;
+        }
+
+        /* </geoRep> */
+        ret = xmlTextWriterEndElement (writer);
+        XML_RET_CHECK_AND_GOTO (ret, out);
+
+        ret = cli_end_xml_output (writer, doc);
+out:
+        gf_log ("cli",GF_LOG_DEBUG, "Returning %d", ret);
+        return ret;
+#else
+        return 0;
+#endif
+}
diff --git a/cli/src/cli.c b/cli/src/cli.c
index c638dc827..91b315ff1 100644
--- a/cli/src/cli.c
+++ b/cli/src/cli.c
@@ -1,22 +1,12 @@
 /*
-  Copyright (c) 2010-2011 Gluster, Inc. <http://www.gluster.com>
-  This file is part of GlusterFS.
-
-  GlusterFS is free software; you can redistribute it and/or modify
-  it under the terms of the GNU General Public License as published
-  by the Free Software Foundation; either version 3 of the License,
-  or (at your option) any later version.
-
-  GlusterFS is distributed in the hope that it will be useful, but
-  WITHOUT ANY WARRANTY; without even the implied warranty of
-  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-  General Public License for more details.
-
-  You should have received a copy of the GNU General Public License
-  along with this program.  If not, see
-  <http://www.gnu.org/licenses/>.
-*/
+   Copyright (c) 2010-2012 Red Hat, Inc. <http://www.redhat.com>
+   This file is part of GlusterFS.
 
+   This file is licensed to you under your choice of the GNU Lesser
+   General Public License, version 3 or any later version (LGPLv3 or
+   later), or the GNU General Public License, version 2 (GPLv2), in all
+   cases as published by the Free Software Foundation.
+*/
 #include <stdio.h>
 #include <string.h>
 #include <stdlib.h>
@@ -99,43 +89,6 @@ rpc_clnt_prog_t *cli_rpc_prog;
 
 extern struct rpc_clnt_program cli_prog;
 
-
-
-
-static char *
-generate_uuid ()
-{
-        char           tmp_str[1024] = {0,};
-        char           hostname[256] = {0,};
-        struct timeval tv = {0,};
-        struct tm      now = {0, };
-        char           now_str[32];
-
-        if (gettimeofday (&tv, NULL) == -1) {
-                gf_log ("glusterfsd", GF_LOG_ERROR,
-                        "gettimeofday: failed %s",
-                        strerror (errno));
-        }
-
-        if (gethostname (hostname, 256) == -1) {
-                gf_log ("glusterfsd", GF_LOG_ERROR,
-                        "gethostname: failed %s",
-                        strerror (errno));
-        }
-
-        localtime_r (&tv.tv_sec, &now);
-        strftime (now_str, 32, "%Y/%m/%d-%H:%M:%S", &now);
-        snprintf (tmp_str, 1024, "%s-%d-%s:%"
-#ifdef GF_DARWIN_HOST_OS
-                  PRId32,
-#else
-                  "ld",
-#endif
-                  hostname, getpid(), now_str, tv.tv_usec);
-
-        return gf_strdup (tmp_str);
-}
-
 static int
 glusterfs_ctx_defaults_init (glusterfs_ctx_t *ctx)
 {
@@ -145,7 +98,7 @@ glusterfs_ctx_defaults_init (glusterfs_ctx_t *ctx)
 
         xlator_mem_acct_init (THIS, cli_mt_end);
 
-        ctx->process_uuid = generate_uuid ();
+        ctx->process_uuid = generate_glusterfs_ctx_id ();
         if (!ctx->process_uuid)
                 return -1;
 
@@ -210,12 +163,13 @@ glusterfs_ctx_defaults_init (glusterfs_ctx_t *ctx)
 
 
 static int
-logging_init (struct cli_state *state)
+logging_init (glusterfs_ctx_t *ctx, struct cli_state *state)
 {
         char *log_file = state->log_file ? state->log_file :
                          DEFAULT_CLI_LOG_FILE_DIRECTORY "/cli.log";
 
-        if (gf_log_init (log_file) == -1) {
+        /* passing ident as NULL means to use default ident for syslog */
+        if (gf_log_init (ctx, log_file, NULL) == -1) {
                 fprintf (stderr, "ERROR: failed to open logfile %s\n",
                          log_file);
                 return -1;
@@ -330,18 +284,41 @@ cli_rpc_notify (struct rpc_clnt *rpc, void *mydata, rpc_clnt_event_t event,
         return ret;
 }
 
+
+/*
+ * ret: 0: option successfully processed
+ *      1: signalling end of option list
+ *     -1: unknown option or other issue
+ */
 int
 cli_opt_parse (char *opt, struct cli_state *state)
 {
         char *oarg;
 
+        if (strcmp (opt, "") == 0)
+                return 1;
+
         if (strcmp (opt, "version") == 0) {
-                puts (argp_program_version);
+                cli_out ("%s", argp_program_version);
+                exit (0);
+        }
+
+        if (strcmp (opt, "print-logdir") == 0) {
+                cli_out ("%s", DEFAULT_LOG_FILE_DIRECTORY);
+                exit (0);
+        }
+
+        if (strcmp (opt, "print-statedumpdir") == 0) {
+                cli_out ("%s", DEFAULT_VAR_RUN_DIRECTORY);
                 exit (0);
         }
 
         if (strcmp (opt, "xml") == 0) {
+#if (HAVE_LIB_XML)
                 state->mode |= GLUSTER_MODE_XML;
+#else
+                cli_err ("XML output not supported. Ignoring '--xml' option");
+#endif
                 return 0;
         }
 
@@ -376,6 +353,12 @@ cli_opt_parse (char *opt, struct cli_state *state)
                 return 0;
         }
 
+        oarg = strtail (opt, "glusterd-sock=");
+        if (oarg) {
+                state->glusterd_sock = oarg;
+                return 0;
+        }
+
         return -1;
 }
 
@@ -403,9 +386,16 @@ parse_cmdline (int argc, char *argv[], struct cli_state *state)
                         state->argc--;
                         /* argv shifted, next check should be at i again */
                         i--;
+                        if (ret == 1) {
+                                /* end of cli options */
+                                ret = 0;
+                                break;
+                        }
                 }
         }
 
+        state->argv[state->argc] = NULL;
+
         return ret;
 }
 
@@ -430,7 +420,6 @@ cli_state_init (struct cli_state *state)
         int                   ret = 0;
 
 
-        state->remote_host = "localhost";
         state->log_level = -1;
 
         tree = &state->tree;
@@ -450,11 +439,35 @@ cli_usage_out (const char *usage)
         if (!usage || usage[0] == '\0')
                 return -1;
 
-        cli_out ("Usage: %s", usage);
+        cli_err ("Usage: %s", usage);
         return 0;
 }
 
 int
+_cli_err (const char *fmt, ...)
+{
+        struct cli_state *state = NULL;
+        va_list           ap;
+        int               ret = 0;
+
+        state = global_state;
+
+        va_start (ap, fmt);
+
+#ifdef HAVE_READLINE
+        if (state->rl_enabled && !state->rl_processing)
+                return cli_rl_err(state, fmt, ap);
+#endif
+
+        ret = vfprintf (stderr, fmt, ap);
+        fprintf (stderr, "\n");
+        va_end (ap);
+
+        return ret;
+}
+
+
+int
 _cli_out (const char *fmt, ...)
 {
         struct cli_state *state = NULL;
@@ -493,23 +506,46 @@ cli_rpc_init (struct cli_state *state)
         if (!options)
                 goto out;
 
-        ret = dict_set_str (options, "remote-host", state->remote_host);
-        if (ret)
-                goto out;
+        /* Connect using to glusterd using the specified method, giving
+         * preference to unix socket connection. If nothing is specified connect
+         * to the default glusterd socket
+         */
+        if (state->glusterd_sock) {
+                gf_log ("cli", GF_LOG_INFO, "Connecting to glusterd using "
+                        "sockfile %s", state->glusterd_sock);
+                ret = rpc_transport_unix_options_build (&options,
+                                                        state->glusterd_sock,
+                                                        0);
+                if (ret)
+                        goto out;
+        } else if (state->remote_host) {
+                gf_log ("cli", GF_LOG_INFO, "Connecting to remote glusterd at "
+                        "%s", state->remote_host);
+                ret = dict_set_str (options, "remote-host", state->remote_host);
+                if (ret)
+                        goto out;
 
-        if (state->remote_port)
-                port = state->remote_port;
+                if (state->remote_port)
+                        port = state->remote_port;
 
-        ret = dict_set_int32 (options, "remote-port", port);
-        if (ret)
-                goto out;
+                ret = dict_set_int32 (options, "remote-port", port);
+                if (ret)
+                        goto out;
 
-        ret = dict_set_str (options, "transport.address-family", "inet/inet6");
-        if (ret)
-                goto out;
+                ret = dict_set_str (options, "transport.address-family",
+                                    "inet");
+                if (ret)
+                        goto out;
+        } else {
+                gf_log ("cli", GF_LOG_DEBUG, "Connecting to glusterd using "
+                        "default socket");
+                ret = rpc_transport_unix_options_build
+                        (&options, DEFAULT_GLUSTERD_SOCKFILE, 0);
+                if (ret)
+                        goto out;
+        }
 
         rpc = rpc_clnt_new (options, this->ctx, this->name, 16);
-
         if (!rpc)
                 goto out;
 
@@ -519,7 +555,7 @@ cli_rpc_init (struct cli_state *state)
                 goto out;
         }
 
-        rpc_clnt_start (rpc);
+        ret = rpc_clnt_start (rpc);
 out:
         if (ret) {
                 if (rpc)
@@ -543,76 +579,15 @@ void
 cli_local_wipe (cli_local_t *local)
 {
         if (local) {
+                GF_FREE (local->get_vol.volname);
+                if (local->dict)
+                        dict_unref (local->dict);
                 GF_FREE (local);
         }
 
         return;
 }
 
-/* If the path exists use realpath(3) to handle extra slashes and to resolve
- * symlinks else strip the extra slashes in the path and return */
-
-int
-cli_canonicalize_path (char *path)
-{
-        struct stat     sb = {0};
-        int             ret = -1;
-        char            *tmppath = NULL;
-        char            *dir = NULL;
-        char            *tmpstr = NULL;
-        int             path_len = 0;
-
-        if (!path)
-                return ret;
-
-        ret = stat (path, &sb);
-        if (ret == -1) {
-                /* Strip the extra slashes and return */
-                tmppath = gf_strdup (path);
-                if (tmppath == NULL) {
-                        ret = -1;
-                        gf_log ("cli", GF_LOG_ERROR, "Out of memory.");
-                        goto out;
-                }
-                bzero (path, strlen(path));
-                path[0] = '/';
-                dir = strtok_r(tmppath, "/", &tmpstr);
-                while (dir) {
-                        strncpy ((path + path_len + 1), dir, strlen(dir));
-                        path_len = strlen (path);
-                        dir = strtok_r(NULL, "/", &tmpstr);
-                        if (dir)
-                                strncpy((path + path_len), "/", 1);
-                }
-                if (path_len == 0)
-                        path[1] = '\0';
-                else
-                        path[path_len] = '\0';
-                ret = 0;
-                goto out;
-        } else {
-                tmppath = gf_strdup(path);
-                if (tmppath == NULL) {
-                        ret = -1;
-                        gf_log ("cli", GF_LOG_ERROR, "Out of memory.");
-                        goto out;
-                }
-                if (realpath (tmppath, path) == NULL) {
-                        cli_out ("Path manipulation failed: %s",
-                                 strerror(errno));
-                        gf_log ("cli", GF_LOG_ERROR, "Path manipulation "
-                                 "failed: %s", strerror(errno));
-                        ret = -1;
-                        goto out;
-                }
-                ret = 0;
-        }
-out:
-        if (tmppath)
-                GF_FREE(tmppath);
-        return ret;
-}
-
 struct cli_state *global_state;
 
 int
@@ -622,13 +597,19 @@ main (int argc, char *argv[])
         int                ret = -1;
         glusterfs_ctx_t   *ctx = NULL;
 
-        ret = glusterfs_globals_init ();
+        ctx = glusterfs_ctx_new ();
+        if (!ctx)
+                return ENOMEM;
+
+#ifdef DEBUG
+        gf_mem_acct_enable_set (ctx);
+#endif
+
+        ret = glusterfs_globals_init (ctx);
         if (ret)
                 return ret;
 
-        ctx = glusterfs_ctx_get ();
-        if (!ctx)
-                return ENOMEM;
+	THIS->ctx = ctx;
 
         ret = glusterfs_ctx_defaults_init (ctx);
         if (ret)
@@ -645,7 +626,7 @@ main (int argc, char *argv[])
         if (ret)
                 goto out;
 
-        ret = logging_init (&state);
+        ret = logging_init (ctx, &state);
         if (ret)
                 goto out;
 
diff --git a/cli/src/cli.h b/cli/src/cli.h
index 2a24378ff..8daa4b741 100644
--- a/cli/src/cli.h
+++ b/cli/src/cli.h
@@ -1,22 +1,12 @@
 /*
-   Copyright (c) 2006-2011 Gluster, Inc. <http://www.gluster.com>
+   Copyright (c) 2006-2012 Red Hat, Inc. <http://www.redhat.com>
    This file is part of GlusterFS.
 
-   GlusterFS is free software; you can redistribute it and/or modify
-   it under the terms of the GNU General Public License as published
-   by the Free Software Foundation; either version 3 of the License,
-   or (at your option) any later version.
-
-   GlusterFS is distributed in the hope that it will be useful, but
-   WITHOUT ANY WARRANTY; without even the implied warranty of
-   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-   General Public License for more details.
-
-   You should have received a copy of the GNU General Public License
-   along with this program.  If not, see
-   <http://www.gnu.org/licenses/>.
+   This file is licensed to you under your choice of the GNU Lesser
+   General Public License, version 3 or any later version (LGPLv3 or
+   later), or the GNU General Public License, version 2 (GPLv2), in all
+   cases as published by the Free Software Foundation.
 */
-
 #ifndef __CLI_H__
 #define __CLI_H__
 
@@ -28,6 +18,9 @@
 #include "rpc-clnt.h"
 #include "glusterfs.h"
 #include "protocol-common.h"
+#include "logging.h"
+
+#include "cli1-xdr.h"
 
 #if (HAVE_LIB_XML)
 #include <libxml/encoding.h>
@@ -38,9 +31,8 @@
 #define CLI_GLUSTERD_PORT                  24007
 #define CLI_DEFAULT_CONN_TIMEOUT             120
 #define CLI_DEFAULT_CMD_TIMEOUT              120
-#define CLI_TOP_CMD_TIMEOUT                  600 //Longer timeout for volume top
+#define CLI_TEN_MINUTES_TIMEOUT              600 //Longer timeout for volume top
 #define DEFAULT_CLI_LOG_FILE_DIRECTORY     DATADIR "/log/glusterfs"
-#define DEFAULT_LOG_FILE_DIRECTORY         DATADIR "/log/glusterfs"
 #define CLI_VOL_STATUS_BRICK_LEN              55
 #define CLI_TAB_LENGTH                         8
 #define CLI_BRICK_STATUS_LINE_LEN             78
@@ -58,6 +50,10 @@ struct cli_cmd_word;
 struct cli_cmd_tree;
 struct cli_cmd;
 
+extern char *cli_vol_type_str[];
+extern char *cli_vol_status_str[];
+extern char *cli_vol_task_status_str[];
+
 typedef int (cli_cmd_cbk_t)(struct cli_state *state,
                             struct cli_cmd_word *word,
                             const char **words,
@@ -119,6 +115,8 @@ struct cli_state {
 
         char                 *log_file;
         gf_loglevel_t         log_level;
+
+        char                 *glusterd_sock;
 };
 
 struct cli_local {
@@ -128,13 +126,28 @@ struct cli_local {
         } get_vol;
 
         dict_t          *dict;
+        const char      **words;
+        /* Marker for volume status all */
+        gf_boolean_t    all;
 #if (HAVE_LIB_XML)
         xmlTextWriterPtr        writer;
-        xmlBufferPtr            buf;
+        xmlDocPtr               doc;
         int                     vol_count;
 #endif
 };
 
+struct gf_cli_gsync_detailed_status_ {
+        char *node;
+        char *master;
+        char *slave;
+        char *health;
+        char *uptime;
+        char *files_syncd;
+        char *files_pending;
+        char *bytes_pending;
+        char *deletes_pending;
+};
+
 struct cli_volume_status {
         int            port;
         int            online;
@@ -153,6 +166,13 @@ struct cli_volume_status {
 #endif
 };
 
+struct snap_config_opt_vals_ {
+        char           *op_name;
+        char           *question;
+};
+
+typedef struct gf_cli_gsync_detailed_status_ gf_cli_gsync_status_t;
+
 typedef struct cli_volume_status cli_volume_status_t;
 
 typedef struct cli_local cli_local_t;
@@ -175,10 +195,12 @@ int cli_cmd_process_line (struct cli_state *state, const char *line);
 
 int cli_rl_enable (struct cli_state *state);
 int cli_rl_out (struct cli_state *state, const char *fmt, va_list ap);
+int cli_rl_err (struct cli_state *state, const char *fmt, va_list ap);
 
 int cli_usage_out (const char *usage);
 
 int _cli_out (const char *fmt, ...);
+int _cli_err (const char *fmt, ...);
 
 #define cli_out(fmt...) do {                       \
                 FMT_WARN (fmt);                    \
@@ -187,6 +209,13 @@ int _cli_out (const char *fmt, ...);
                                                    \
         } while (0)
 
+#define cli_err(fmt...) do {                       \
+                FMT_WARN (fmt);                    \
+                                                   \
+                _cli_err(fmt);                     \
+                                                   \
+        } while (0)
+
 int
 cli_submit_request (void *req, call_frame_t *frame,
                     rpc_clnt_prog_t *prog,
@@ -208,7 +237,7 @@ cli_cmd_quota_parse (const char **words, int wordcount, dict_t **opt);
 
 int32_t
 cli_cmd_volume_set_parse (const char **words, int wordcount,
-                          dict_t **options);
+                          dict_t **options, char **op_errstr);
 
 int32_t
 cli_cmd_volume_add_brick_parse (const char **words, int wordcount,
@@ -251,9 +280,6 @@ int
 cli_rpc_notify (struct rpc_clnt *rpc, void *mydata, rpc_clnt_event_t event,
                 void *data);
 
-int
-cli_canonicalize_path (char *path);
-
 int32_t
 cli_cmd_volume_profile_parse (const char **words, int wordcount,
                               dict_t **options);
@@ -274,6 +300,10 @@ cli_cmd_volume_heal_options_parse (const char **words, int wordcount,
                                    dict_t **options);
 
 int
+cli_cmd_volume_defrag_parse (const char **words, int wordcount,
+                             dict_t **options);
+
+int
 cli_print_brick_status (cli_volume_status_t *status);
 
 void
@@ -285,7 +315,6 @@ cli_get_detail_status (dict_t *dict, int i, cli_volume_status_t *status);
 void
 cli_print_line (int len);
 
-#if (HAVE_LIB_XML)
 int
 cli_xml_output_str (char *op, char *str, int op_ret, int op_errno,
                     char *op_errstr);
@@ -303,8 +332,14 @@ cli_xml_output_vol_profile (dict_t *dict, int op_ret, int op_errno,
                             char *op_errstr);
 
 int
-cli_xml_output_vol_status (dict_t *dict, int op_ret, int op_errno,
-                           char *op_errstr);
+cli_xml_output_vol_status_begin (cli_local_t *local, int op_ret, int op_errno,
+                                 char *op_errstr);
+
+int
+cli_xml_output_vol_status_end (cli_local_t *local);
+
+int
+cli_xml_output_vol_status (cli_local_t *local, dict_t *dict);
 
 int
 cli_xml_output_vol_list (dict_t *dict, int op_ret, int op_errno,
@@ -324,6 +359,42 @@ int
 cli_xml_output_vol_quota_limit_list (char *volname, char *limit_list,
                                       int op_ret, int op_errno,
                                       char *op_errstr);
-#endif
+
+int
+cli_xml_output_peer_status (dict_t *dict, int op_ret, int op_errno,
+                            char *op_errstr);
+
+int
+cli_xml_output_vol_rebalance (gf_cli_defrag_type op, dict_t *dict, int op_ret,
+                              int op_errno, char *op_errstr);
+
+int
+cli_xml_output_vol_remove_brick (gf_boolean_t status_op, dict_t *dict,
+                                 int op_ret, int op_errno, char *op_errstr);
+
+int
+cli_xml_output_vol_replace_brick (gf1_cli_replace_op op, dict_t *dict,
+                                  int op_ret, int op_errno, char *op_errstr);
+
+int
+cli_xml_output_vol_create (dict_t *dict, int op_ret, int op_errno,
+                           char *op_errstr);
+
+int
+cli_xml_output_generic_volume (char *op, dict_t *dict, int op_ret, int op_errno,
+                               char *op_errstr);
+
+int
+cli_xml_output_vol_gsync (dict_t *dict, int op_ret, int op_errno,
+                          char *op_errstr);
+int
+cli_xml_output_vol_status_tasks_detail (cli_local_t *local, dict_t *dict);
+
+char *
+is_server_debug_xlator (void *myframe);
+
+int32_t
+cli_cmd_snapshot_parse (const char **words, int wordcount, dict_t **options,
+                        struct cli_state *state);
 
 #endif /* __CLI_H__ */
diff --git a/cli/src/input.c b/cli/src/input.c
index a88d35874..a8ea46c6d 100644
--- a/cli/src/input.c
+++ b/cli/src/input.c
@@ -1,22 +1,12 @@
 /*
-  Copyright (c) 2010-2011 Gluster, Inc. <http://www.gluster.com>
-  This file is part of GlusterFS.
-
-  GlusterFS is free software; you can redistribute it and/or modify
-  it under the terms of the GNU General Public License as published
-  by the Free Software Foundation; either version 3 of the License,
-  or (at your option) any later version.
-
-  GlusterFS is distributed in the hope that it will be useful, but
-  WITHOUT ANY WARRANTY; without even the implied warranty of
-  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-  General Public License for more details.
-
-  You should have received a copy of the GNU General Public License
-  along with this program.  If not, see
-  <http://www.gnu.org/licenses/>.
-*/
+   Copyright (c) 2010-2012 Red Hat, Inc. <http://www.redhat.com>
+   This file is part of GlusterFS.
 
+   This file is licensed to you under your choice of the GNU Lesser
+   General Public License, version 3 or any later version (LGPLv3 or
+   later), or the GNU General Public License, version 2 (GPLv2), in all
+   cases as published by the Free Software Foundation.
+*/
 #include <stdio.h>
 #include <string.h>
 #include <stdlib.h>
@@ -44,7 +34,7 @@ cli_batch (void *d)
         ret = cli_cmd_process (state, state->argc, state->argv);
 
         gf_log ("", GF_LOG_INFO, "Exiting with: %d", ret);
-        exit (ret);
+        exit (-ret);
 
         return NULL;
 }
@@ -71,11 +61,11 @@ cli_input (void *d)
                 if (len > 0 && cmd[len - 1] == '\n') //strip trailing \n
                         cmd[len - 1] = '\0';
                 ret = cli_cmd_process_line (state, cmd);
-                if (ret == -1 && state->mode & GLUSTER_MODE_ERR_FATAL)
+                if (ret != 0 && state->mode & GLUSTER_MODE_ERR_FATAL)
                         break;
         }
 
-        exit (ret);
+        exit (-ret);
 
         return NULL;
 }
diff --git a/cli/src/registry.c b/cli/src/registry.c
index fc3b2decd..c4abe3be0 100644
--- a/cli/src/registry.c
+++ b/cli/src/registry.c
@@ -1,22 +1,12 @@
 /*
-  Copyright (c) 2010-2011 Gluster, Inc. <http://www.gluster.com>
-  This file is part of GlusterFS.
-
-  GlusterFS is free software; you can redistribute it and/or modify
-  it under the terms of the GNU General Public License as published
-  by the Free Software Foundation; either version 3 of the License,
-  or (at your option) any later version.
-
-  GlusterFS is distributed in the hope that it will be useful, but
-  WITHOUT ANY WARRANTY; without even the implied warranty of
-  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-  General Public License for more details.
-
-  You should have received a copy of the GNU General Public License
-  along with this program.  If not, see
-  <http://www.gnu.org/licenses/>.
-*/
+   Copyright (c) 2010-2012 Red Hat, Inc. <http://www.redhat.com>
+   This file is part of GlusterFS.
 
+   This file is licensed to you under your choice of the GNU Lesser
+   General Public License, version 3 or any later version (LGPLv3 or
+   later), or the GNU General Public License, version 2 (GPLv2), in all
+   cases as published by the Free Software Foundation.
+*/
 
 #ifndef _CONFIG_H
 #define _CONFIG_H
diff --git a/configure.ac b/configure.ac
index ca3d87750..b3d1ed184 100644
--- a/configure.ac
+++ b/configure.ac
@@ -1,19 +1,10 @@
-dnl Copyright (c) 2006-2011 Gluster, Inc. <http://www.gluster.com>
-
-dnl This file is part of GlusterFS.
-dnl
-dnl GlusterFS is free software; you can redistribute it and/or modify
-dnl it under the terms of the GNU General Public License as published by
-dnl the Free Software Foundation; either version 3 of the License, or
-dnl (at your option) any later version.
-dnl
-dnl GlusterFS is distributed in the hope that it will be useful,
-dnl but WITHOUT ANY WARRANTY; without even the implied warranty of
-dnl MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-dnl GNU General Public License for more details.
+dnl  Copyright (c) 2006-2012 Red Hat, Inc. <http://www.redhat.com>
+dnl  This file is part of GlusterFS.
 dnl
-dnl You should have received a copy of the GNU General Public License
-dnl along with this program.  If not, see <http://www.gnu.org/licenses/>.
+dnl  This file is licensed to you under your choice of the GNU Lesser
+dnl  General Public License, version 3 or any later version (LGPLv3 or
+dnl  later), or the GNU General Public License, version 2 (GPLv2), in all
+dnl  cases as published by the Free Software Foundation.
 
 AC_INIT([glusterfs],[3git],[gluster-users@gluster.org],,[https://github.com/gluster/glusterfs.git])
 
@@ -33,13 +24,15 @@ if libtool --help 2>&1 | grep -q quiet; then
    AM_LIBTOOLFLAGS="--quiet";
 fi
 
-AM_CONFIG_HEADER([config.h])
+AC_CONFIG_HEADERS([config.h])
 
 AC_CONFIG_FILES([Makefile
-		libglusterfs/Makefile
-		libglusterfs/src/Makefile
-		glusterfsd/Makefile
-		glusterfsd/src/Makefile
+                libglusterfs/Makefile
+                libglusterfs/src/Makefile
+                geo-replication/src/peer_gsec_create
+                geo-replication/src/peer_add_secret_pub
+                glusterfsd/Makefile
+                glusterfsd/src/Makefile
                 rpc/Makefile
                 rpc/rpc-lib/Makefile
                 rpc/rpc-lib/src/Makefile
@@ -50,114 +43,162 @@ AC_CONFIG_FILES([Makefile
                 rpc/rpc-transport/rdma/src/Makefile
                 rpc/xdr/Makefile
                 rpc/xdr/src/Makefile
-		xlators/Makefile
-		xlators/mount/Makefile
-		xlators/mount/fuse/Makefile
-		xlators/mount/fuse/src/Makefile
-		xlators/mount/fuse/utils/mount.glusterfs
-		xlators/mount/fuse/utils/mount_glusterfs
-		xlators/mount/fuse/utils/Makefile
-		xlators/storage/Makefile
-		xlators/storage/posix/Makefile
-		xlators/storage/posix/src/Makefile
-		xlators/cluster/Makefile
-		xlators/cluster/afr/Makefile
-		xlators/cluster/afr/src/Makefile
-		xlators/cluster/stripe/Makefile
-		xlators/cluster/stripe/src/Makefile
-		xlators/cluster/dht/Makefile
-		xlators/cluster/dht/src/Makefile
-		xlators/performance/Makefile
-		xlators/performance/write-behind/Makefile
-		xlators/performance/write-behind/src/Makefile
-		xlators/performance/read-ahead/Makefile
-		xlators/performance/read-ahead/src/Makefile
-		xlators/performance/io-threads/Makefile
-		xlators/performance/io-threads/src/Makefile
-		xlators/performance/io-cache/Makefile
-		xlators/performance/io-cache/src/Makefile
-		xlators/performance/symlink-cache/Makefile
-		xlators/performance/symlink-cache/src/Makefile
-		xlators/performance/quick-read/Makefile
-		xlators/performance/quick-read/src/Makefile
+                xlators/Makefile
+                xlators/mount/Makefile
+                xlators/mount/fuse/Makefile
+                xlators/mount/fuse/src/Makefile
+                xlators/mount/fuse/utils/mount.glusterfs
+                xlators/mount/fuse/utils/mount_glusterfs
+                xlators/mount/fuse/utils/Makefile
+                xlators/storage/Makefile
+                xlators/storage/posix/Makefile
+                xlators/storage/posix/src/Makefile
+                xlators/storage/bd/Makefile
+                xlators/storage/bd/src/Makefile
+                xlators/cluster/Makefile
+                xlators/cluster/afr/Makefile
+                xlators/cluster/afr/src/Makefile
+                xlators/cluster/stripe/Makefile
+                xlators/cluster/stripe/src/Makefile
+                xlators/cluster/dht/Makefile
+                xlators/cluster/dht/src/Makefile
+                xlators/performance/Makefile
+                xlators/performance/write-behind/Makefile
+                xlators/performance/write-behind/src/Makefile
+                xlators/performance/read-ahead/Makefile
+                xlators/performance/read-ahead/src/Makefile
+                xlators/performance/readdir-ahead/Makefile
+                xlators/performance/readdir-ahead/src/Makefile
+                xlators/performance/io-threads/Makefile
+                xlators/performance/io-threads/src/Makefile
+                xlators/performance/io-cache/Makefile
+                xlators/performance/io-cache/src/Makefile
+                xlators/performance/symlink-cache/Makefile
+                xlators/performance/symlink-cache/src/Makefile
+                xlators/performance/quick-read/Makefile
+                xlators/performance/quick-read/src/Makefile
+                xlators/performance/open-behind/Makefile
+                xlators/performance/open-behind/src/Makefile
                 xlators/performance/md-cache/Makefile
                 xlators/performance/md-cache/src/Makefile
-		xlators/debug/Makefile
-		xlators/debug/trace/Makefile
-		xlators/debug/trace/src/Makefile
-		xlators/debug/error-gen/Makefile
-		xlators/debug/error-gen/src/Makefile
-		xlators/debug/io-stats/Makefile
-		xlators/debug/io-stats/src/Makefile
-		xlators/protocol/Makefile
-		xlators/protocol/auth/Makefile
-		xlators/protocol/auth/addr/Makefile
-		xlators/protocol/auth/addr/src/Makefile
-		xlators/protocol/auth/login/Makefile
-		xlators/protocol/auth/login/src/Makefile
-		xlators/protocol/client/Makefile
-		xlators/protocol/client/src/Makefile
-		xlators/protocol/server/Makefile
-		xlators/protocol/server/src/Makefile
-		xlators/features/Makefile
-		xlators/features/locks/Makefile
-		xlators/features/locks/src/Makefile
-		xlators/features/trash/Makefile
-		xlators/features/trash/src/Makefile
-		xlators/features/quota/Makefile
-		xlators/features/quota/src/Makefile
+                xlators/debug/Makefile
+                xlators/debug/trace/Makefile
+                xlators/debug/trace/src/Makefile
+                xlators/debug/error-gen/Makefile
+                xlators/debug/error-gen/src/Makefile
+                xlators/debug/io-stats/Makefile
+                xlators/debug/io-stats/src/Makefile
+                xlators/protocol/Makefile
+                xlators/protocol/auth/Makefile
+                xlators/protocol/auth/addr/Makefile
+                xlators/protocol/auth/addr/src/Makefile
+                xlators/protocol/auth/login/Makefile
+                xlators/protocol/auth/login/src/Makefile
+                xlators/protocol/client/Makefile
+                xlators/protocol/client/src/Makefile
+                xlators/protocol/server/Makefile
+                xlators/protocol/server/src/Makefile
+                xlators/features/Makefile
+		xlators/features/changelog/Makefile
+		xlators/features/changelog/src/Makefile
+		xlators/features/changelog/lib/Makefile
+		xlators/features/changelog/lib/src/Makefile
+                xlators/features/glupy/Makefile
+                xlators/features/glupy/src/Makefile
+                xlators/features/locks/Makefile
+                xlators/features/locks/src/Makefile
+                xlators/features/quota/Makefile
+                xlators/features/quota/src/Makefile
                 xlators/features/marker/Makefile
                 xlators/features/marker/src/Makefile
-                xlators/features/marker/utils/Makefile
-                xlators/features/marker/utils/src/Makefile
-                xlators/features/marker/utils/syncdaemon/Makefile
-		xlators/features/read-only/Makefile
-		xlators/features/read-only/src/Makefile
-		xlators/features/mac-compat/Makefile
-		xlators/features/mac-compat/src/Makefile
-		xlators/features/quiesce/Makefile
-		xlators/features/quiesce/src/Makefile
+                xlators/features/read-only/Makefile
+                xlators/features/read-only/src/Makefile
+                xlators/features/compress/Makefile
+                xlators/features/compress/src/Makefile
+                xlators/features/mac-compat/Makefile
+                xlators/features/mac-compat/src/Makefile
+                xlators/features/quiesce/Makefile
+                xlators/features/quiesce/src/Makefile
                 xlators/features/index/Makefile
                 xlators/features/index/src/Makefile
-		xlators/encryption/Makefile
-		xlators/encryption/rot-13/Makefile
-		xlators/encryption/rot-13/src/Makefile
+                xlators/features/protect/Makefile
+                xlators/features/protect/src/Makefile
+                xlators/features/gfid-access/Makefile
+                xlators/features/gfid-access/src/Makefile
+                xlators/playground/Makefile
+                xlators/playground/template/Makefile
+                xlators/playground/template/src/Makefile
+                xlators/encryption/Makefile
+                xlators/encryption/rot-13/Makefile
+                xlators/encryption/rot-13/src/Makefile
+                xlators/encryption/crypt/Makefile
+                xlators/encryption/crypt/src/Makefile
+		xlators/features/qemu-block/Makefile
+		xlators/features/qemu-block/src/Makefile
                 xlators/system/Makefile
                 xlators/system/posix-acl/Makefile
                 xlators/system/posix-acl/src/Makefile
-                cli/Makefile
-                cli/src/Makefile
-		doc/Makefile
-		doc/examples/Makefile
-		doc/hacker-guide/Makefile
-		extras/Makefile
-		extras/init.d/Makefile
-		extras/init.d/glusterd.plist
-                extras/init.d/glusterd-Debian
-                extras/init.d/glusterd-Redhat
-                extras/init.d/glusterd-SuSE
-		extras/benchmarking/Makefile
-		contrib/fuse-util/Makefile
-		contrib/uuid/uuid_types.h
                 xlators/nfs/Makefile
                 xlators/nfs/server/Makefile
                 xlators/nfs/server/src/Makefile
                 xlators/mgmt/Makefile
                 xlators/mgmt/glusterd/Makefile
                 xlators/mgmt/glusterd/src/Makefile
-		glusterfs.spec])
+                cli/Makefile
+                cli/src/Makefile
+                doc/Makefile
+                extras/Makefile
+                extras/init.d/Makefile
+                extras/init.d/glusterd.plist
+                extras/init.d/glusterd-Debian
+                extras/init.d/glusterd-Redhat
+                extras/init.d/glusterd-SuSE
+                extras/systemd/Makefile
+                extras/systemd/glusterd.service
+                extras/benchmarking/Makefile
+                extras/hook-scripts/Makefile
+                extras/ocf/Makefile
+                extras/ocf/glusterd
+                extras/ocf/volume
+                extras/LinuxRPM/Makefile
+                extras/geo-rep/Makefile
+                contrib/fuse-util/Makefile
+                contrib/uuid/uuid_types.h
+                glusterfs-api.pc
+                libgfchangelog.pc
+                api/Makefile
+                api/src/Makefile
+                api/examples/Makefile
+                api/examples/setup.py
+		geo-replication/Makefile
+		geo-replication/src/Makefile
+		geo-replication/syncdaemon/Makefile
+                glusterfs.spec])
 
 AC_CANONICAL_HOST
 
 AC_PROG_CC
+AC_DISABLE_STATIC
 AC_PROG_LIBTOOL
 
+AC_ARG_WITH(pkgconfigdir,
+            [  --with-pkgconfigdir=DIR      pkgconfig file in DIR @<:@LIBDIR/pkgconfig@:>@],
+            [pkgconfigdir=$withval],
+            [pkgconfigdir='${libdir}/pkgconfig'])
+AC_SUBST(pkgconfigdir)
+
 AC_ARG_WITH(mountutildir,
             [  --with-mountutildir=DIR mount helper utility in DIR @<:@/sbin@:>@],
             [mountutildir=$withval],
             [mountutildir='/sbin'])
 AC_SUBST(mountutildir)
 
+AC_ARG_WITH(systemddir,
+            [  --with-systemddir=DIR systemd service files in DIR @<:@/usr/lib/systemd/system@:>@],
+            [systemddir=$withval],
+            [systemddir='/usr/lib/systemd/system'])
+AC_SUBST(systemddir)
+
 AC_ARG_WITH(initdir,
             [  --with-initdir=DIR init.d scripts in DIR @<:@/etc/init.d@:>@],
             [initdir=$withval],
@@ -170,6 +211,13 @@ AC_ARG_WITH(launchddir,
             [launchddir='/Library/LaunchDaemons'])
 AC_SUBST(launchddir)
 
+AC_ARG_WITH([ocf],
+            [AS_HELP_STRING([--without-ocf], [build OCF-compliant cluster resource agents])],
+            ,
+            [OCF_SUBDIR='ocf'],
+            )
+AC_SUBST(OCF_SUBDIR)
+
 # LEX needs a check
 AC_PROG_LEX
 if test  "x${LEX}" != "xflex" -a "x${FLEX}" != "xlex"; then
@@ -200,8 +248,10 @@ fi
 
 AC_CHECK_TOOL([LD],[ld])
 
+AC_CHECK_LIB([crypto], [MD5], , AC_MSG_ERROR([OpenSSL crypto library is required to build glusterfs]))
+
 AC_CHECK_LIB([pthread], [pthread_mutex_init], , AC_MSG_ERROR([Posix threads library is required to build glusterfs]))
-		 
+
 AC_CHECK_FUNC([dlopen], [has_dlopen=yes], AC_CHECK_LIB([dl], [dlopen], , AC_MSG_ERROR([Dynamic linking library required to build glusterfs])))
 
 
@@ -209,6 +259,10 @@ AC_CHECK_HEADERS([sys/xattr.h])
 
 AC_CHECK_HEADERS([sys/extattr.h])
 
+AC_CHECK_HEADERS([openssl/md5.h])
+
+AC_CHECK_HEADERS([linux/falloc.h])
+
 case $host_os in
   darwin*)
     if ! test "`/usr/bin/sw_vers | grep ProductVersion: | cut -f 2 | cut -d. -f2`" -ge 5; then
@@ -242,8 +296,8 @@ fi
 
 # FUSE section
 AC_ARG_ENABLE([fuse-client],
-	      AC_HELP_STRING([--disable-fuse-client],
-			     [Do not build the fuse client. NOTE: you cannot mount glusterfs without the client]))
+              AC_HELP_STRING([--disable-fuse-client],
+                             [Do not build the fuse client. NOTE: you cannot mount glusterfs without the client]))
 
 BUILD_FUSE_CLIENT=no
 if test "x$enable_fuse_client" != "xno"; then
@@ -251,60 +305,153 @@ if test "x$enable_fuse_client" != "xno"; then
    BUILD_FUSE_CLIENT="yes"
 fi
 
+AC_ARG_ENABLE([bd-xlator],
+              AC_HELP_STRING([--enable-bd-xlator], [Build BD xlator]))
+
+if test "x$enable_bd_xlator" != "xno"; then
+  AC_CHECK_LIB([lvm2app],
+               [lvm_init,lvm_lv_from_name],
+               [HAVE_BD_LIB="yes"],
+               [HAVE_BD_LIB="no"])
+
+if test "x$HAVE_BD_LIB" = "xyes"; then
+    # lvm_lv_from_name() has been made public with lvm2-2.02.79
+    AC_CHECK_DECLS(
+                  [lvm_lv_from_name],
+                  [NEED_LVM_LV_FROM_NAME_DECL="no"],
+                  [NEED_LVM_LV_FROM_NAME_DECL="yes"],
+                  [[#include <lvm2app.h>]])
+  fi
+fi
+
+if test "x$enable_bd_xlator" = "xyes" -a "x$HAVE_BD_LIB" = "xno"; then
+   echo "BD xlator requested but required lvm2 development library not found."
+   exit 1
+fi
+
+BUILD_BD_XLATOR=no
+if test "x${enable-bd-xlator}" != "xno" -a "x${HAVE_BD_LIB}" = "xyes"; then
+  BUILD_BD_XLATOR=yes
+  AC_DEFINE(HAVE_BD_XLATOR, 1, [define if lvm2app library found and bd xlator
+                            enabled])
+  if test "x$NEED_LVM_LV_FROM_NAME_DECL" = "xyes"; then
+    AC_DEFINE(NEED_LVM_LV_FROM_NAME_DECL, 1, [defined if lvm_lv_from_name()
+                  was not found in the lvm2app.h header, but can be linked])
+  fi
+fi
+
+AM_CONDITIONAL([ENABLE_BD_XLATOR], [test x$BUILD_BD_XLATOR = xyes])
+
+# start encryption/crypt section
+
+AC_CHECK_HEADERS([openssl/cmac.h], [have_cmac_h=yes], [have_cmac_h=no])
+
+AC_ARG_ENABLE([crypt-xlator],
+	AC_HELP_STRING([--enable-crypt-xlator], [Build crypt encryption xlator]))
+
+if test "x$enable_crypt_xlator" = "xyes" -a "x$have_cmac_h" = "xno"; then
+   echo "Encryption xlator requires OpenSSL with cmac.h"
+   exit 1
+fi
+
+BUILD_CRYPT_XLATOR=no
+if test "x$enable_crypt_xlator" != "xno" -a "x$have_cmac_h" = "xyes"; then
+   BUILD_CRYPT_XLATOR=yes
+   AC_DEFINE(HAVE_CRYPT_XLATOR, 1, [enable building crypt encryption xlator])
+fi
+
+AM_CONDITIONAL([ENABLE_CRYPT_XLATOR], [test x$BUILD_CRYPT_XLATOR = xyes])
+
 AC_SUBST(FUSE_CLIENT_SUBDIR)
 # end FUSE section
 
 
 # FUSERMOUNT section
 AC_ARG_ENABLE([fusermount],
-              AC_HELP_STRING([--enable-fusermount],
-                             [Build fusermount]))
+              AC_HELP_STRING([--disable-fusermount],
+                             [Use system's fusermount]))
 
-BUILD_FUSERMOUNT="no"
-if test "x$enable_fusermount" = "xyes"; then
-  FUSERMOUNT_SUBDIR="contrib/fuse-util"
-  BUILD_FUSERMOUNT="yes"
+BUILD_FUSERMOUNT="yes"
+if test "x$enable_fusermount" = "xno"; then
+  BUILD_FUSERMOUNT="no"
+else
   AC_DEFINE(GF_FUSERMOUNT, 1, [Use our own fusermount])
+  FUSERMOUNT_SUBDIR="contrib/fuse-util"
 fi
 
 AC_SUBST(FUSERMOUNT_SUBDIR)
 #end FUSERMOUNT section
 
+# QEMU_BLOCK section
+
+AC_ARG_ENABLE([qemu-block],
+                AC_HELP_STRING([--enable-qemu-block],
+                        [Build QEMU Block formats translator]))
+
+if test "x$enable_qemu_block" != "xno"; then
+  PKG_CHECK_MODULES([GLIB], [glib-2.0],
+                    [HAVE_GLIB_2="yes"],
+  		    [HAVE_GLIB_2="no"])
+fi
+
+if test "x$enable_qemu_block" = "xyes" -a "x$HAVE_GLIB_2" = "xno"; then
+   echo "QEMU Block formats translator requires libglib-2.0, but missing."
+   exit 1
+fi
+
+BUILD_QEMU_BLOCK=no
+if test "x${enable_qemu_block}" != "xno" -a "x${HAVE_GLIB_2}" = "xyes"; then
+  BUILD_QEMU_BLOCK=yes
+  AC_DEFINE(HAVE_QEMU_BLOCK, 1, [define if libglib-2.0 library found and QEMU
+                  	     Block translator enabled])
+fi
+
+AM_CONDITIONAL([ENABLE_QEMU_BLOCK], [test x$BUILD_QEMU_BLOCK = xyes])
+
+# end QEMU_BLOCK section
 
 # EPOLL section
 AC_ARG_ENABLE([epoll],
-	      AC_HELP_STRING([--disable-epoll],
-			     [Use poll instead of epoll.]))
+              AC_HELP_STRING([--disable-epoll],
+                             [Use poll instead of epoll.]))
 
 BUILD_EPOLL=no
 if test "x$enable_epoll" != "xno"; then
    AC_CHECK_HEADERS([sys/epoll.h],
                     [BUILD_EPOLL=yes],
-		    [BUILD_EPOLL=no])
+                    [BUILD_EPOLL=no])
 fi
 # end EPOLL section
 
 
 # IBVERBS section
 AC_ARG_ENABLE([ibverbs],
-	      AC_HELP_STRING([--disable-ibverbs],
-			     [Do not build the ibverbs transport]))
+              AC_HELP_STRING([--disable-ibverbs],
+                             [Do not build the ibverbs transport]))
 
 if test "x$enable_ibverbs" != "xno"; then
   AC_CHECK_LIB([ibverbs],
                [ibv_get_device_list],
-	       [HAVE_LIBIBVERBS="yes"],
-	       [HAVE_LIBIBVERBS="no"])
+               [HAVE_LIBIBVERBS="yes"],
+               [HAVE_LIBIBVERBS="no"])
+  AC_CHECK_LIB([rdmacm], [rdma_create_id], [HAVE_RDMACM="yes"], [HAVE_RDMACM="no"])
 fi
 
-if test "x$enable_ibverbs" = "xyes" -a "x$HAVE_LIBIBVERBS" = "xno"; then
-   echo "ibverbs requested but not found."
-   exit 1
+if test "x$enable_ibverbs" = "xyes"; then
+   if test "x$HAVE_LIBIBVERBS" = "xno"; then
+      echo "ibverbs-transport requested, but libibverbs is not present."
+      exit 1
+   fi
+
+   if test "x$HAVE_RDMACM" = "xno"; then
+      echo "ibverbs-transport requested, but librdmacm is not present."
+      exit 1
+   fi
 fi
 
 BUILD_RDMA=no
 BUILD_IBVERBS=no
-if test "x$enable_ibverbs" != "xno" -a "x$HAVE_LIBIBVERBS" = "xyes"; then
+if test "x$enable_ibverbs" != "xno" -a "x$HAVE_LIBIBVERBS" = "xyes" -a "x$HAVE_RDMACM" = "xyes"; then
   IBVERBS_SUBDIR=ib-verbs
   BUILD_IBVERBS=yes
   RDMA_SUBDIR=rdma
@@ -318,8 +465,8 @@ AC_SUBST(RDMA_SUBDIR)
 
 # SYNCDAEMON section
 AC_ARG_ENABLE([georeplication],
-	      AC_HELP_STRING([--disable-georeplication],
-			     [Do not install georeplication components]))
+              AC_HELP_STRING([--disable-georeplication],
+                             [Do not install georeplication components]))
 
 BUILD_SYNCDAEMON=no
 case $host_os in
@@ -331,12 +478,12 @@ case $host_os in
        ;;
      *)
 #disabling geo replication for non-linux platforms
-	enable_georeplication=no
+        enable_georeplication=no
         ;;
 esac
 SYNCDAEMON_COMPILE=0
 if test "x$enable_georeplication" != "xno"; then
-  SYNCDAEMON_SUBDIR=utils
+  SYNCDAEMON_SUBDIR=geo-replication
   SYNCDAEMON_COMPILE=1
 
   BUILD_SYNCDAEMON="yes"
@@ -360,15 +507,67 @@ AC_SUBST(SYNCDAEMON_COMPILE)
 AC_SUBST(SYNCDAEMON_SUBDIR)
 # end SYNCDAEMON section
 
-#check if libxml is present if so enable HAVE_LIB_XML
-echo -n "checking if libxml2 is present... "
+# CDC xlator - check if libz is present if so enable HAVE_LIB_Z
+echo -n "checking if libz is present... "
 
-PKG_CHECK_MODULES([LIBXML2], [libxml-2.0 >= 2.6.19],
-                  [echo "yes (features requiring libxml2 enabled)"  AC_DEFINE(HAVE_LIB_XML, 1, [define if libxml2 is present])],
+PKG_CHECK_MODULES([ZLIB], [zlib >= 1.2.0],
+                  [echo "yes (features requiring zlib enabled)"  AC_DEFINE(HAVE_LIB_Z, 1, [define if zlib is present])],
                   [echo "no"] )
 
-AC_SUBST(LIBXML2_CFLAGS)
-AC_SUBST(LIBXML2_LIBS)
+AC_SUBST(LIBZ_CFLAGS)
+AC_SUBST(LIBZ_LIBS)
+# end CDC xlator secion
+
+# check for systemtap/dtrace
+BUILD_SYSTEMTAP=no
+AC_MSG_CHECKING([whether to include systemtap tracing support])
+AC_ARG_ENABLE([systemtap],
+              [AS_HELP_STRING([--enable-systemtap],
+              [Enable inclusion of systemtap trace support])],
+              [ENABLE_SYSTEMTAP="${enableval}"], [ENABLE_SYSTEMTAP="def"])
+
+AM_CONDITIONAL([ENABLE_SYSTEMTAP], [test "x${ENABLE_SYSTEMTAP}" = "xyes"])
+AC_MSG_RESULT(${ENABLE_SYSTEMTAP})
+
+if test "x${ENABLE_SYSTEMTAP}" != "xno"; then
+    AC_CHECK_PROG(DTRACE, dtrace, "yes", "no")
+    AC_CHECK_HEADER([sys/sdt.h], [SDT_H_FOUND="yes"],
+                    [SDT_H_FOUND="no"])
+fi
+
+if test "x${ENABLE_SYSTEMTAP}" = "xyes"; then
+    if test "x${DTRACE}" = "xno"; then
+        AC_MSG_ERROR([dtrace not found])
+    elif test "$x{SDT_H_FOUND}" = "xno"; then
+        AC_MSG_ERROR([systemtap support needs sys/sdt.h header])
+    fi
+fi
+
+if test "x${DTRACE}" = "xyes" -a "x${SDT_H_FOUND}" = "xyes"; then
+    AC_MSG_CHECKING([x"${DTRACE}"xy"${SDT_H_FOUND}"y])
+    AC_DEFINE([HAVE_SYSTEMTAP], [1], [Define to 1 if using  probes.])
+    BUILD_SYSTEMTAP=yes
+fi
+# end of systemtap/dtrace
+
+# xml-output
+AC_ARG_ENABLE([xml-output],
+              AC_HELP_STRING([--disable-xml-output],
+                             [Disable the xml output]))
+BUILD_XML_OUTPUT="yes"
+if test "x$enable_xml_output" != "xno"; then
+    #check if libxml is present if so enable HAVE_LIB_XML
+    m4_ifdef([AM_PATH_XML2],[AM_PATH_XML2([2.6.19])], [no_xml=yes])
+    if test "x${no_xml}" = "x"; then
+        AC_DEFINE([HAVE_LIB_XML], [1], [Define to 1 if using libxml2.])
+    else
+        AC_MSG_WARN([libxml2 devel libraries not found disabling XML support])
+        BUILD_XML_OUTPUT="no"
+    fi
+else
+    BUILD_XML_OUTPUT="no"
+fi
+# end of xml-output
 
 dnl FreeBSD > 5 has execinfo as a Ported library for giving a workaround
 dnl solution to GCC backtrace functionality
@@ -393,17 +592,26 @@ dnl Linux, Solaris, Cygwin
 AC_CHECK_MEMBERS([struct stat.st_atim.tv_nsec])
 dnl FreeBSD, NetBSD
 AC_CHECK_MEMBERS([struct stat.st_atimespec.tv_nsec])
+case $host_os in
+        *netbsd*)
+        CFLAGS+=" -D_INCOMPLETE_XOPEN_C063"
+        ;;
+esac
 AC_CHECK_FUNC([linkat], [have_linkat=yes])
 if test "x${have_linkat}" = "xyes"; then
    AC_DEFINE(HAVE_LINKAT, 1, [define if found linkat])
 fi
 AC_SUBST(HAVE_LINKAT)
 
+dnl check for Monotonic clock
+AC_CHECK_FUNC([clock_gettime], [has_monotonic_clock=yes], AC_CHECK_LIB([rt], [clock_gettime], , AC_MSG_WARN([System doesn't have monotonic clock using contrib])))
+
 dnl Check for argp
 AC_CHECK_HEADER([argp.h], AC_DEFINE(HAVE_ARGP, 1, [have argp]))
 AC_CONFIG_SUBDIRS(argp-standalone)
+
 BUILD_ARGP_STANDALONE=no
-if test "x${ac_cv_header_argp_h}" = "xno"; then 
+if test "x${ac_cv_header_argp_h}" = "xno"; then
    BUILD_ARGP_STANDALONE=yes
    ARGP_STANDALONE_CPPFLAGS='-I${top_srcdir}/argp-standalone'
    ARGP_STANDALONE_LDADD='${top_builddir}/argp-standalone/libargp.a'
@@ -424,7 +632,18 @@ if test "x${have_fdatasync}" = "xyes"; then
    AC_DEFINE(HAVE_FDATASYNC, 1, [define if fdatasync exists])
 fi
 
-# Check the distribution where you are compiling glusterfs on 
+AC_CHECK_FUNC([fallocate], [have_fallocate=yes])
+if test "x${have_fallocate}" = "xyes"; then
+   AC_DEFINE(HAVE_FALLOCATE, 1, [define if fallocate exists])
+fi
+
+AC_CHECK_FUNC([posix_fallocate], [have_posix_fallocate=yes])
+if test "x${have_posix_fallocate}" = "xyes"; then
+   AC_DEFINE(HAVE_POSIX_FALLOCATE, 1, [define if posix_fallocate exists])
+fi
+
+
+# Check the distribution where you are compiling glusterfs on
 
 GF_DISTRIBUTION=
 AC_CHECK_FILE([/etc/debian_version])
@@ -446,56 +665,115 @@ AC_SUBST(GF_DISTRIBUTION)
 GF_HOST_OS=""
 GF_LDFLAGS="-rdynamic"
 
+# check for gcc -Werror=format-security
+saved_CFLAGS=$CFLAGS
+CFLAGS="-Wformat -Werror=format-security"
+AC_MSG_CHECKING([whether $CC accepts -Werror=format-security])
+AC_COMPILE_IFELSE([AC_LANG_PROGRAM()], [cc_werror_format_security=yes], [cc_werror_format_security=no])
+echo $cc_werror_format_security
+if test "x$cc_werror_format_security" = "xno"; then
+    CFLAGS="$saved_CFLAGS"
+else
+    CFLAGS="$saved_CFLAGS $CFLAGS"
+fi
+
+# check for gcc -Werror=implicit-function-declaration
+saved_CFLAGS=$CFLAGS
+CFLAGS="-Werror=implicit-function-declaration"
+AC_MSG_CHECKING([whether $CC accepts -Werror=implicit-function-declaration])
+AC_COMPILE_IFELSE([AC_LANG_PROGRAM()], [cc_werror_implicit=yes], [cc_werror_implicit=no])
+echo $cc_werror_implicit
+if test "x$cc_werror_implicit" = "xno"; then
+    CFLAGS="$saved_CFLAGS"
+else
+    CFLAGS="$saved_CFLAGS $CFLAGS"
+fi
+
 case $host_os in
      linux*)
-     dnl	GF_LINUX_HOST_OS=1
         GF_HOST_OS="GF_LINUX_HOST_OS"
-	GF_CFLAGS="${ARGP_STANDALONE_CPPFLAGS} -O0"
-	GF_GLUSTERFS_CFLAGS="${GF_CFLAGS}"
-	GF_LDADD="${ARGP_STANDALONE_LDADD}"
-	GF_FUSE_CFLAGS="-DFUSERMOUNT_DIR=\\\"\$(bindir)\\\""
-	;;
+        GF_CFLAGS="${ARGP_STANDALONE_CPPFLAGS} -O0"
+        GF_GLUSTERFS_CFLAGS="${GF_CFLAGS}"
+        GF_LDADD="${ARGP_STANDALONE_LDADD}"
+        GF_FUSE_CFLAGS="-DFUSERMOUNT_DIR=\\\"\$(bindir)\\\""
+        ;;
      solaris*)
         GF_HOST_OS="GF_SOLARIS_HOST_OS"
-	GF_CFLAGS="${ARGP_STANDALONE_CPPFLAGS} -D_REENTRANT -D_POSIX_PTHREAD_SEMANTICS -O0 -m64"
-	GF_LDFLAGS=""
-	GF_GLUSTERFS_CFLAGS="${GF_CFLAGS}"
-	GF_LDADD="${ARGP_STANDALONE_LDADD}"
-	GF_GLUSTERFS_LDFLAGS="-lnsl -lresolv -lsocket"
+        GF_CFLAGS="${ARGP_STANDALONE_CPPFLAGS} -D_REENTRANT -D_POSIX_PTHREAD_SEMANTICS -O0 -m64"
+        GF_LDFLAGS=""
+        GF_GLUSTERFS_CFLAGS="${GF_CFLAGS}"
+        GF_LDADD="${ARGP_STANDALONE_LDADD}"
+        GF_GLUSTERFS_LIBS="-lnsl -lresolv -lsocket"
         BUILD_FUSE_CLIENT=no
         FUSE_CLIENT_SUBDIR=""
-	;;
+        ;;
      *netbsd*)
-	GF_HOST_OS="GF_BSD_HOST_OS"
-	GF_CFLAGS="${ARGP_STANDALONE_CPPFLAGS} -D_INCOMPLETE_XOPEN_C063"
-	GF_GLUSTERFS_CFLAGS="${GF_CFLAGS}"
-	GF_LDADD="${ARGP_STANDALONE_LDADD}"
-	if test "x$ac_cv_header_execinfo_h" = "xyes"; then
-	   GF_GLUSTERFS_LDFLAGS="-lexecinfo"
-	fi
-	GF_FUSE_LDADD="-liconv -lperfuse"
-	BUILD_FUSE_CLIENT=no
-	;;
+        GF_HOST_OS="GF_BSD_HOST_OS"
+        GF_CFLAGS="${ARGP_STANDALONE_CPPFLAGS} -D_INCOMPLETE_XOPEN_C063"
+        GF_CFLAGS="${GF_CFLAGS} -DTHREAD_UNSAFE_BASENAME"
+        GF_CFLAGS="${GF_CFLAGS} -DTHREAD_UNSAFE_DIRNAME"
+        GF_GLUSTERFS_CFLAGS="${GF_CFLAGS}"
+        GF_LDADD="${ARGP_STANDALONE_LDADD}"
+        if test "x$ac_cv_header_execinfo_h" = "xyes"; then
+           GF_GLUSTERFS_LIBS="-lexecinfo"
+        fi
+        GF_FUSE_LDADD="-lperfuse"
+        BUILD_FUSE_CLIENT=yes
+        LEXLIB=""
+        ;;
      *bsd*)
         GF_HOST_OS="GF_BSD_HOST_OS"
-	GF_CFLAGS="${ARGP_STANDALONE_CPPFLAGS} -O0"
-	GF_GLUSTERFS_CFLAGS="${GF_CFLAGS}"
-	GF_LDADD="${ARGP_STANDALONE_LDADD}"
-	if test "x$ac_cv_header_execinfo_h" = "xyes"; then
-	   GF_GLUSTERFS_LDFLAGS="-lexecinfo"
-	fi				      
-	BUILD_FUSE_CLIENT=no
-	;;
+        GF_CFLAGS="${ARGP_STANDALONE_CPPFLAGS} -O0"
+        GF_CFLAGS="${GF_CFLAGS} -DTHREAD_UNSAFE_BASENAME"
+        GF_CFLAGS="${GF_CFLAGS} -DTHREAD_UNSAFE_DIRNAME"
+        GF_GLUSTERFS_CFLAGS="${GF_CFLAGS}"
+        GF_LDADD="${ARGP_STANDALONE_LDADD}"
+        if test "x$ac_cv_header_execinfo_h" = "xyes"; then
+           GF_GLUSTERFS_LIBS="-lexecinfo"
+        fi
+        BUILD_FUSE_CLIENT=no
+        ;;
      darwin*)
         GF_HOST_OS="GF_DARWIN_HOST_OS"
-	LIBTOOL=glibtool
-	GF_CFLAGS="${ARGP_STANDALONE_CPPFLAGS} -D__DARWIN_64_BIT_INO_T -bundle -undefined suppress -flat_namespace -D_XOPEN_SOURCE -O0"
-	GF_GLUSTERFS_CFLAGS="${ARGP_STANDALONE_CPPFLAGS} -D__DARWIN_64_BIT_INO_T -undefined suppress -flat_namespace -O0"
-	GF_LDADD="${ARGP_STANDALONE_LDADD}"
-	GF_FUSE_CFLAGS="-I\$(CONTRIBDIR)/macfuse"
-	;;
+        LIBTOOL=glibtool
+        GF_CFLAGS="${ARGP_STANDALONE_CPPFLAGS} -D__DARWIN_64_BIT_INO_T -bundle -undefined suppress -flat_namespace -D_XOPEN_SOURCE -O0"
+        GF_CFLAGS="${GF_CFLAGS} -DTHREAD_UNSAFE_BASENAME"
+        GF_CFLAGS="${GF_CFLAGS} -DTHREAD_UNSAFE_DIRNAME"
+        GF_GLUSTERFS_CFLAGS="${ARGP_STANDALONE_CPPFLAGS} -D__DARWIN_64_BIT_INO_T -undefined suppress -flat_namespace -O0"
+        GF_LDADD="${ARGP_STANDALONE_LDADD}"
+        GF_FUSE_CFLAGS="-I\$(CONTRIBDIR)/macfuse"
+        ;;
 esac
 
+# enable debug section
+AC_ARG_ENABLE([debug],
+              AC_HELP_STRING([--enable-debug],
+                             [Enable debug build options.]))
+
+BUILD_DEBUG=no
+if test "x$enable_debug" = "xyes"; then
+   BUILD_DEBUG=yes
+   CFLAGS=`echo $CFLAGS | sed -e s/O2/O0/`
+else
+   BUILD_DEBUG=no
+fi
+AC_SUBST(CFLAGS)
+# end enable debug section
+
+# syslog section
+AC_ARG_ENABLE([syslog],
+	      AC_HELP_STRING([--disable-syslog],
+			     [Disable syslog for logging]))
+
+USE_SYSLOG="yes"
+if test "x$enable_syslog" != "xno"; then
+  AC_DEFINE(GF_USE_SYSLOG, 1, [Use syslog for logging])
+else
+  USE_SYSLOG="no"
+fi
+AM_CONDITIONAL([ENABLE_SYSLOG], [test x$USE_SYSLOG = xyes])
+#end syslog section
+
 BUILD_READLINE=no
 AC_CHECK_LIB([readline -lcurses],[readline],[RLLIBS="-lreadline -lcurses"])
 AC_CHECK_LIB([readline -ltermcap],[readline],[RLLIBS="-lreadline -ltermcap"])
@@ -506,8 +784,66 @@ if test "x$RLLIBS" != "x"; then
    BUILD_READLINE=yes
 fi
 
+BUILD_LIBAIO=no
+AC_CHECK_LIB([aio],[io_setup],[LIBAIO="-laio"])
+
+if test "x$LIBAIO" != "x"; then
+   AC_DEFINE(HAVE_LIBAIO, 1, [libaio based POSIX enabled])
+   BUILD_LIBAIO=yes
+fi
+
+# glupy section
+BUILD_GLUPY=no
+have_python2=no
+have_Python_h=no
+
+AM_PATH_PYTHON()
+if echo $PYTHON_VERSION | grep ^2; then
+  have_python2=yes
+fi
+AC_CHECK_HEADERS([python$PYTHON_VERSION/Python.h],[have_Python_h=yes],[])
+AC_ARG_ENABLE([glupy],
+              AS_HELP_STRING([--enable-glupy],
+                             [build glupy]))
+case x$enable_glupy in
+   xyes)
+      if test "x$have_python2" = "xyes" -a "x$have_Python_h" = "xyes"; then
+         BUILD_GLUPY=yes
+      else
+         AC_MSG_ERROR([glupy requires python-devel/python-dev package and python2.x])
+      fi
+      ;;
+   xno)
+      ;;
+   *)
+      if test "x$have_python2" = "xyes" -a "x$have_Python_h" = "xyes"; then
+         BUILD_GLUPY=yes
+      else
+         AC_MSG_WARN([
+        ---------------------------------------------------------------------------------
+         cannot build glupy. python 2.x and python-devel/python-dev package are required.
+        ---------------------------------------------------------------------------------])
+      fi
+      ;;
+esac
+
+if test "x$BUILD_GLUPY" = "xyes"; then
+   BUILD_PYTHON_INC=`$PYTHON -c "from distutils import sysconfig; print sysconfig.get_python_inc()"`
+   BUILD_PYTHON_LIB=python$PYTHON_VERSION
+   GLUPY_SUBDIR=glupy
+   GLUPY_SUBDIR_MAKEFILE=xlators/features/glupy/Makefile
+   GLUPY_SUBDIR_SRC_MAKEFILE=xlators/features/glupy/src/Makefile
+   echo "building glupy with -isystem $BUILD_PYTHON_INC -l $BUILD_PYTHON_LIB"
+   AC_SUBST(BUILD_PYTHON_INC)
+   AC_SUBST(BUILD_PYTHON_LIB)
+   AC_SUBST(GLUPY_SUBDIR)
+   AC_SUBST(GLUPY_SUBDIR_MAKEFILE)
+   AC_SUBST(GLUPY_SUBDIR_SRC_MAKEFILE)
+fi
+# end glupy section
+
 AC_SUBST(GF_HOST_OS)
-AC_SUBST(GF_GLUSTERFS_LDFLAGS)
+AC_SUBST([GF_GLUSTERFS_LIBS])
 AC_SUBST(GF_GLUSTERFS_CFLAGS)
 AC_SUBST(GF_CFLAGS)
 AC_SUBST(GF_LDFLAGS)
@@ -515,27 +851,41 @@ AC_SUBST(GF_LDADD)
 AC_SUBST(GF_FUSE_LDADD)
 AC_SUBST(GF_FUSE_CFLAGS)
 AC_SUBST(RLLIBS)
+AC_SUBST(LIBAIO)
 AC_SUBST(AM_MAKEFLAGS)
 AC_SUBST(AM_LIBTOOLFLAGS)
 
 CONTRIBDIR='$(top_srcdir)/contrib'
 AC_SUBST(CONTRIBDIR)
 
-INCLUDES='-I$(top_srcdir)/libglusterfs/src -I$(CONTRIBDIR)/uuid'
-AC_SUBST(INCLUDES)
+GF_CPPDEFINES='-D_FILE_OFFSET_BITS=64 -D_GNU_SOURCE -D$(GF_HOST_OS)'
+GF_CPPINCLUDES='-I$(top_srcdir)/libglusterfs/src -I$(CONTRIBDIR)/uuid'
+GF_CPPFLAGS="$GF_CPPDEFINES $GF_CPPINCLUDES"
+AC_SUBST([GF_CPPFLAGS])
+
+AM_CONDITIONAL([GF_DARWIN_HOST_OS], test "${GF_HOST_OS}" = "GF_DARWIN_HOST_OS")
 
-AM_CONDITIONAL([GF_DARWIN_HOST_OS], test "${GF_HOST_OS}" = "GF_DARWIN_HOST_OS")	
+AM_CONDITIONAL([GF_INSTALL_VAR_LIB_GLUSTERD], test ! -d ${localstatedir}/lib/glusterd && test -d ${sysconfdir}/glusterd )
 
 AC_OUTPUT
 
 echo
 echo "GlusterFS configure summary"
 echo "==========================="
-echo "FUSE client        : $BUILD_FUSE_CLIENT"
-echo "Infiniband verbs   : $BUILD_IBVERBS"
-echo "epoll IO multiplex : $BUILD_EPOLL"
-echo "argp-standalone    : $BUILD_ARGP_STANDALONE"
-echo "fusermount         : $BUILD_FUSERMOUNT"
-echo "readline           : $BUILD_READLINE"
-echo "georeplication     : $BUILD_SYNCDAEMON"
+echo "FUSE client          : $BUILD_FUSE_CLIENT"
+echo "Infiniband verbs     : $BUILD_IBVERBS"
+echo "epoll IO multiplex   : $BUILD_EPOLL"
+echo "argp-standalone      : $BUILD_ARGP_STANDALONE"
+echo "fusermount           : $BUILD_FUSERMOUNT"
+echo "readline             : $BUILD_READLINE"
+echo "georeplication       : $BUILD_SYNCDAEMON"
+echo "Linux-AIO            : $BUILD_LIBAIO"
+echo "Enable Debug         : $BUILD_DEBUG"
+echo "systemtap            : $BUILD_SYSTEMTAP"
+echo "Block Device xlator  : $BUILD_BD_XLATOR"
+echo "glupy                : $BUILD_GLUPY"
+echo "Use syslog           : $USE_SYSLOG"
+echo "XML output           : $BUILD_XML_OUTPUT"
+echo "QEMU Block formats   : $BUILD_QEMU_BLOCK"
+echo "Encryption xlator    : $BUILD_CRYPT_XLATOR"
 echo
diff --git a/contrib/aclocal/mkdirp.m4 b/contrib/aclocal/mkdirp.m4
new file mode 100644
index 000000000..d2f7edd5c
--- /dev/null
+++ b/contrib/aclocal/mkdirp.m4
@@ -0,0 +1,146 @@
+# Excerpt from autoconf/autoconf/programs.m4
+# This file is part of Autoconf.                       -*- Autoconf -*-
+# Checking for programs.
+
+# Copyright (C) 1992, 1993, 1994, 1995, 1996, 1998, 1999, 2000, 2001,
+# 2002, 2003, 2004, 2005, 2006, 2007, 2008, 2009, 2010 Free Software
+# Foundation, Inc.
+
+# This file is part of Autoconf.  This program is free
+# software; you can redistribute it and/or modify it under the
+# terms of the GNU General Public License as published by the
+# Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# Under Section 7 of GPL version 3, you are granted additional
+# permissions described in the Autoconf Configure Script Exception,
+# version 3.0, as published by the Free Software Foundation.
+#
+# You should have received a copy of the GNU General Public License
+# and a copy of the Autoconf Configure Script Exception along with
+# this program; see the files COPYINGv3 and COPYING.EXCEPTION
+# respectively.  If not, see <http://www.gnu.org/licenses/>.
+
+# Written by David MacKenzie, with help from
+# Franc,ois Pinard, Karl Berry, Richard Pixley, Ian Lance Taylor,
+# Roland McGrath, Noah Friedman, david d zuhn, and many others.
+
+# AC_PROG_MKDIR_P
+# ---------------
+# Check whether `mkdir -p' is known to be thread-safe, and fall back to
+# install-sh -d otherwise.
+#
+# Automake 1.8 used `mkdir -m 0755 -p --' to ensure that directories
+# created by `make install' are always world readable, even if the
+# installer happens to have an overly restrictive umask (e.g. 077).
+# This was a mistake.  There are at least two reasons why we must not
+# use `-m 0755':
+#   - it causes special bits like SGID to be ignored,
+#   - it may be too restrictive (some setups expect 775 directories).
+#
+# Do not use -m 0755 and let people choose whatever they expect by
+# setting umask.
+#
+# We cannot accept any implementation of `mkdir' that recognizes `-p'.
+# Some implementations (such as Solaris 8's) are vulnerable to race conditions:
+# if a parallel make tries to run `mkdir -p a/b' and `mkdir -p a/c'
+# concurrently, both version can detect that a/ is missing, but only
+# one can create it and the other will error out.  Consequently we
+# restrict ourselves to known race-free implementations.
+#
+# Automake used to define mkdir_p as `mkdir -p .', in order to
+# allow $(mkdir_p) to be used without argument.  As in
+#   $(mkdir_p) $(somedir)
+# where $(somedir) is conditionally defined.  However we don't do
+# that for MKDIR_P.
+#  1. before we restricted the check to GNU mkdir, `mkdir -p .' was
+#     reported to fail in read-only directories.  The system where this
+#     happened has been forgotten.
+#  2. in practice we call $(MKDIR_P) on directories such as
+#       $(MKDIR_P) "$(DESTDIR)$(somedir)"
+#     and we don't want to create $(DESTDIR) if $(somedir) is empty.
+#     To support the latter case, we have to write
+#       test -z "$(somedir)" || $(MKDIR_P) "$(DESTDIR)$(somedir)"
+#     so $(MKDIR_P) always has an argument.
+#     We will have better chances of detecting a missing test if
+#     $(MKDIR_P) complains about missing arguments.
+#  3. $(MKDIR_P) is named after `mkdir -p' and we don't expect this
+#     to accept no argument.
+#  4. having something like `mkdir .' in the output is unsightly.
+#
+# On NextStep and OpenStep, the `mkdir' command does not
+# recognize any option.  It will interpret all options as
+# directories to create.
+AN_MAKEVAR([MKDIR_P], [AC_PROG_MKDIR_P])
+AC_DEFUN_ONCE([AC_PROG_MKDIR_P],
+[AC_REQUIRE([AC_CONFIG_AUX_DIR_DEFAULT])dnl
+
+AC_MSG_CHECKING([for a thread-safe mkdir -p])
+if test -z "$MKDIR_P"; then
+  AC_CACHE_VAL([ac_cv_path_mkdir],
+    [_AS_PATH_WALK([$PATH$PATH_SEPARATOR/opt/sfw/bin],
+      [for ac_prog in mkdir gmkdir; do
+	 for ac_exec_ext in '' $ac_executable_extensions; do
+	   AS_EXECUTABLE_P(["$as_dir/$ac_prog$ac_exec_ext"]) || continue
+	   case `"$as_dir/$ac_prog$ac_exec_ext" --version 2>&1` in #(
+	     'mkdir (GNU coreutils) '* | \
+	     'mkdir (coreutils) '* | \
+	     'mkdir (fileutils) '4.1*)
+	       ac_cv_path_mkdir=$as_dir/$ac_prog$ac_exec_ext
+	       break 3;;
+	   esac
+	 done
+       done])])
+  test -d ./--version && rmdir ./--version
+  if test "${ac_cv_path_mkdir+set}" = set; then
+    MKDIR_P="$ac_cv_path_mkdir -p"
+  else
+    # As a last resort, use the slow shell script.  Don't cache a
+    # value for MKDIR_P within a source directory, because that will
+    # break other packages using the cache if that directory is
+    # removed, or if the value is a relative name.
+    MKDIR_P="$ac_install_sh -d"
+  fi
+fi
+dnl status.m4 does special magic for MKDIR_P instead of AC_SUBST,
+dnl to get relative names right.  However, also AC_SUBST here so
+dnl that Automake versions before 1.10 will pick it up (they do not
+dnl trace AC_SUBST_TRACE).
+dnl FIXME: Remove this once we drop support for Automake < 1.10.
+AC_SUBST([MKDIR_P])dnl
+AC_MSG_RESULT([$MKDIR_P])
+])# AC_PROG_MKDIR_P
+
+
+# From automake/m4/mkdirp.m4
+##                                                          -*- Autoconf -*-
+# Copyright (C) 2003, 2004, 2005, 2006  Free Software Foundation, Inc.
+#
+# This file is free software; the Free Software Foundation
+# gives unlimited permission to copy and/or distribute it,
+# with or without modifications, as long as this notice is preserved.
+
+# AM_PROG_MKDIR_P
+# ---------------
+# Check for `mkdir -p'.
+AC_DEFUN([AM_PROG_MKDIR_P],
+[
+AC_REQUIRE([AC_PROG_MKDIR_P])dnl
+dnl Automake 1.8 to 1.9.6 used to define mkdir_p.  We now use MKDIR_P,
+dnl while keeping a definition of mkdir_p for backward compatibility.
+dnl @MKDIR_P@ is magic: AC_OUTPUT adjusts its value for each Makefile.
+dnl However we cannot define mkdir_p as $(MKDIR_P) for the sake of
+dnl Makefile.ins that do not define MKDIR_P, so we do our own
+dnl adjustment using top_builddir (which is defined more often than
+dnl MKDIR_P).
+AC_SUBST([mkdir_p], ["$MKDIR_P"])dnl
+case $mkdir_p in
+  [[\\/$]]* | ?:[[\\/]]*) ;;
+  */*) mkdir_p="\$(top_builddir)/$mkdir_p" ;;
+esac
+])
diff --git a/contrib/aclocal/python.m4 b/contrib/aclocal/python.m4
new file mode 100644
index 000000000..a39a90090
--- /dev/null
+++ b/contrib/aclocal/python.m4
@@ -0,0 +1,209 @@
+## ------------------------                                 -*- Autoconf -*-
+## Python file handling
+## From Andrew Dalke
+## Updated by James Henstridge
+## ------------------------
+# Copyright (C) 1999, 2000, 2001, 2002, 2003, 2004, 2005, 2008, 2009
+# Free Software Foundation, Inc.
+#
+# This file is free software; the Free Software Foundation
+# gives unlimited permission to copy and/or distribute it,
+# with or without modifications, as long as this notice is preserved.
+
+# AM_PATH_PYTHON([MINIMUM-VERSION], [ACTION-IF-FOUND], [ACTION-IF-NOT-FOUND])
+# ---------------------------------------------------------------------------
+# Adds support for distributing Python modules and packages.  To
+# install modules, copy them to $(pythondir), using the python_PYTHON
+# automake variable.  To install a package with the same name as the
+# automake package, install to $(pkgpythondir), or use the
+# pkgpython_PYTHON automake variable.
+#
+# The variables $(pyexecdir) and $(pkgpyexecdir) are provided as
+# locations to install python extension modules (shared libraries).
+# Another macro is required to find the appropriate flags to compile
+# extension modules.
+#
+# If your package is configured with a different prefix to python,
+# users will have to add the install directory to the PYTHONPATH
+# environment variable, or create a .pth file (see the python
+# documentation for details).
+#
+# If the MINIMUM-VERSION argument is passed, AM_PATH_PYTHON will
+# cause an error if the version of python installed on the system
+# doesn't meet the requirement.  MINIMUM-VERSION should consist of
+# numbers and dots only.
+AC_DEFUN([AM_PATH_PYTHON],
+ [
+  dnl Find a Python interpreter.  Python versions prior to 2.0 are not
+  dnl supported. (2.0 was released on October 16, 2000).
+  m4_define_default([_AM_PYTHON_INTERPRETER_LIST],
+                    [python python2 python3 python3.2 python3.1 python3.0 python2.7 python2.6 python2.5 python2.4 python2.3 python2.2 dnl
+python2.1 python2.0])
+
+  m4_if([$1],[],[
+    dnl No version check is needed.
+    # Find any Python interpreter.
+    if test -z "$PYTHON"; then
+      AC_PATH_PROGS([PYTHON], _AM_PYTHON_INTERPRETER_LIST, :)
+    fi
+    am_display_PYTHON=python
+  ], [
+    dnl A version check is needed.
+    if test -n "$PYTHON"; then
+      # If the user set $PYTHON, use it and don't search something else.
+      AC_MSG_CHECKING([whether $PYTHON version >= $1])
+      AM_PYTHON_CHECK_VERSION([$PYTHON], [$1],
+			      [AC_MSG_RESULT(yes)],
+			      [AC_MSG_ERROR(too old)])
+      am_display_PYTHON=$PYTHON
+    else
+      # Otherwise, try each interpreter until we find one that satisfies
+      # VERSION.
+      AC_CACHE_CHECK([for a Python interpreter with version >= $1],
+	[am_cv_pathless_PYTHON],[
+	for am_cv_pathless_PYTHON in _AM_PYTHON_INTERPRETER_LIST none; do
+	  test "$am_cv_pathless_PYTHON" = none && break
+	  AM_PYTHON_CHECK_VERSION([$am_cv_pathless_PYTHON], [$1], [break])
+	done])
+      # Set $PYTHON to the absolute path of $am_cv_pathless_PYTHON.
+      if test "$am_cv_pathless_PYTHON" = none; then
+	PYTHON=:
+      else
+        AC_PATH_PROG([PYTHON], [$am_cv_pathless_PYTHON])
+      fi
+      am_display_PYTHON=$am_cv_pathless_PYTHON
+    fi
+  ])
+
+  if test "$PYTHON" = :; then
+  dnl Run any user-specified action, or abort.
+    m4_default([$3], [AC_MSG_ERROR([no suitable Python interpreter found])])
+  else
+
+  dnl Query Python for its version number.  Getting [:3] seems to be
+  dnl the best way to do this; it's what "site.py" does in the standard
+  dnl library.
+
+  AC_CACHE_CHECK([for $am_display_PYTHON version], [am_cv_python_version],
+    [am_cv_python_version=`$PYTHON -c "import sys; sys.stdout.write(sys.version[[:3]])"`])
+  AC_SUBST([PYTHON_VERSION], [$am_cv_python_version])
+
+  dnl Use the values of $prefix and $exec_prefix for the corresponding
+  dnl values of PYTHON_PREFIX and PYTHON_EXEC_PREFIX.  These are made
+  dnl distinct variables so they can be overridden if need be.  However,
+  dnl general consensus is that you shouldn't need this ability.
+
+  AC_SUBST([PYTHON_PREFIX], ['${prefix}'])
+  AC_SUBST([PYTHON_EXEC_PREFIX], ['${exec_prefix}'])
+
+  dnl At times (like when building shared libraries) you may want
+  dnl to know which OS platform Python thinks this is.
+
+  AC_CACHE_CHECK([for $am_display_PYTHON platform], [am_cv_python_platform],
+    [am_cv_python_platform=`$PYTHON -c "import sys; sys.stdout.write(sys.platform)"`])
+  AC_SUBST([PYTHON_PLATFORM], [$am_cv_python_platform])
+
+
+  dnl Set up 4 directories:
+
+  dnl pythondir -- where to install python scripts.  This is the
+  dnl   site-packages directory, not the python standard library
+  dnl   directory like in previous automake betas.  This behavior
+  dnl   is more consistent with lispdir.m4 for example.
+  dnl Query distutils for this directory.  distutils does not exist in
+  dnl Python 1.5, so we fall back to the hardcoded directory if it
+  dnl doesn't work.
+  AC_CACHE_CHECK([for $am_display_PYTHON script directory],
+    [am_cv_python_pythondir],
+    [if test "x$prefix" = xNONE
+     then
+       am_py_prefix=$ac_default_prefix
+     else
+       am_py_prefix=$prefix
+     fi
+     am_cv_python_pythondir=`$PYTHON -c "import sys; from distutils import sysconfig; sys.stdout.write(sysconfig.get_python_lib(0,0,prefix='$am_py_prefix'))" 2>/dev/null ||
+     echo "$PYTHON_PREFIX/lib/python$PYTHON_VERSION/site-packages"`
+     case $am_cv_python_pythondir in
+     $am_py_prefix*)
+       am__strip_prefix=`echo "$am_py_prefix" | sed 's|.|.|g'`
+       am_cv_python_pythondir=`echo "$am_cv_python_pythondir" | sed "s,^$am__strip_prefix,$PYTHON_PREFIX,"`
+       ;;
+     *)
+       case $am_py_prefix in
+         /usr|/System*) ;;
+         *)
+	  am_cv_python_pythondir=$PYTHON_PREFIX/lib/python$PYTHON_VERSION/site-packages
+	  ;;
+       esac
+       ;;
+     esac
+    ])
+  AC_SUBST([pythondir], [$am_cv_python_pythondir])
+
+  dnl pkgpythondir -- $PACKAGE directory under pythondir.  Was
+  dnl   PYTHON_SITE_PACKAGE in previous betas, but this naming is
+  dnl   more consistent with the rest of automake.
+
+  AC_SUBST([pkgpythondir], [\${pythondir}/$PACKAGE])
+
+  dnl pyexecdir -- directory for installing python extension modules
+  dnl   (shared libraries)
+  dnl Query distutils for this directory.  distutils does not exist in
+  dnl Python 1.5, so we fall back to the hardcoded directory if it
+  dnl doesn't work.
+  AC_CACHE_CHECK([for $am_display_PYTHON extension module directory],
+    [am_cv_python_pyexecdir],
+    [if test "x$exec_prefix" = xNONE
+     then
+       am_py_exec_prefix=$am_py_prefix
+     else
+       am_py_exec_prefix=$exec_prefix
+     fi
+     am_cv_python_pyexecdir=`$PYTHON -c "import sys; from distutils import sysconfig; sys.stdout.write(sysconfig.get_python_lib(1,0,prefix='$am_py_exec_prefix'))" 2>/dev/null ||
+     echo "$PYTHON_EXEC_PREFIX/lib/python$PYTHON_VERSION/site-packages"`
+     case $am_cv_python_pyexecdir in
+     $am_py_exec_prefix*)
+       am__strip_prefix=`echo "$am_py_exec_prefix" | sed 's|.|.|g'`
+       am_cv_python_pyexecdir=`echo "$am_cv_python_pyexecdir" | sed "s,^$am__strip_prefix,$PYTHON_EXEC_PREFIX,"`
+       ;;
+     *)
+       case $am_py_exec_prefix in
+         /usr|/System*) ;;
+         *)
+	   am_cv_python_pyexecdir=$PYTHON_EXEC_PREFIX/lib/python$PYTHON_VERSION/site-packages
+	   ;;
+       esac
+       ;;
+     esac
+    ])
+  AC_SUBST([pyexecdir], [$am_cv_python_pyexecdir])
+
+  dnl pkgpyexecdir -- $(pyexecdir)/$(PACKAGE)
+
+  AC_SUBST([pkgpyexecdir], [\${pyexecdir}/$PACKAGE])
+
+  dnl Run any user-specified action.
+  $2
+  fi
+
+])
+
+
+# AM_PYTHON_CHECK_VERSION(PROG, VERSION, [ACTION-IF-TRUE], [ACTION-IF-FALSE])
+# ---------------------------------------------------------------------------
+# Run ACTION-IF-TRUE if the Python interpreter PROG has version >= VERSION.
+# Run ACTION-IF-FALSE otherwise.
+# This test uses sys.hexversion instead of the string equivalent (first
+# word of sys.version), in order to cope with versions such as 2.2c1.
+# This supports Python 2.0 or higher. (2.0 was released on October 16, 2000).
+AC_DEFUN([AM_PYTHON_CHECK_VERSION],
+ [prog="import sys
+# split strings by '.' and convert to numeric.  Append some zeros
+# because we need at least 4 digits for the hex conversion.
+# map returns an iterator in Python 3.0 and a list in 2.x
+minver = list(map(int, '$2'.split('.'))) + [[0, 0, 0]]
+minverhex = 0
+# xrange is not present in Python 3.0 and range returns an iterator
+for i in list(range(0, 4)): minverhex = (minverhex << 8) + minver[[i]]
+sys.exit(sys.hexversion < minverhex)"
+  AS_IF([AM_RUN_LOG([$1 -c "$prog"])], [$3], [$4])])
diff --git a/contrib/fuse-include/fuse-mount.h b/contrib/fuse-include/fuse-mount.h
index 9f83faf02..9358ac810 100644
--- a/contrib/fuse-include/fuse-mount.h
+++ b/contrib/fuse-include/fuse-mount.h
@@ -8,5 +8,6 @@
 */
 
 void gf_fuse_unmount (const char *mountpoint, int fd);
-int gf_fuse_mount (const char *mountpoint, char *fsname, char *mnt_param,
-                   pid_t *mtab_pid);
+int gf_fuse_mount (const char *mountpoint, char *fsname,
+                   unsigned long mountflags, char *mnt_param,
+                   pid_t *mtab_pid, int status_fd);
diff --git a/contrib/fuse-include/fuse_kernel.h b/contrib/fuse-include/fuse_kernel.h
index 9ae25d6f9..60bb2f9f7 100644
--- a/contrib/fuse-include/fuse_kernel.h
+++ b/contrib/fuse-include/fuse_kernel.h
@@ -60,23 +60,75 @@
  * 7.13
  *  - make max number of background requests and congestion threshold
  *    tunables
+ *
+ * 7.14
+ *  - add splice support to fuse device
+ *
+ * 7.15
+ *  - add store notify
+ *  - add retrieve notify
+ *
+ * 7.16
+ *  - add BATCH_FORGET request
+ *  - FUSE_IOCTL_UNRESTRICTED shall now return with array of 'struct
+ *    fuse_ioctl_iovec' instead of ambiguous 'struct iovec'
+ *  - add FUSE_IOCTL_32BIT flag
+ *
+ * 7.17
+ *  - add FUSE_FLOCK_LOCKS and FUSE_RELEASE_FLOCK_UNLOCK
+ *
+ * 7.18
+ *  - add FUSE_IOCTL_DIR flag
+ *  - add FUSE_NOTIFY_DELETE
+ *
+ * 7.19
+ *  - add FUSE_FALLOCATE
+ *
+ * 7.20
+ *  - add FUSE_AUTO_INVAL_DATA
+ *
+ * 7.21
+ *  - add FUSE_READDIRPLUS
+ *  - send the requested events in POLL request
+ *
+ * 7.22
+ *  - add FUSE_ASYNC_DIO
  */
 
 #ifndef _LINUX_FUSE_H
 #define _LINUX_FUSE_H
 
-#include <sys/types.h>
-#define __u64 uint64_t
-#define __s64 int64_t
-#define __u32 uint32_t
-#define __s32 int32_t
-#define __u16 uint16_t
+#ifdef __KERNEL__
+#include <linux/types.h>
+#else
+#include <stdint.h>
+#endif
+
+/*
+ * Version negotiation:
+ *
+ * Both the kernel and userspace send the version they support in the
+ * INIT request and reply respectively.
+ *
+ * If the major versions match then both shall use the smallest
+ * of the two minor versions for communication.
+ *
+ * If the kernel supports a larger major version, then userspace shall
+ * reply with the major version it supports, ignore the rest of the
+ * INIT message and expect a new INIT message from the kernel with a
+ * matching major version.
+ *
+ * If the library supports a larger major version, then it shall fall
+ * back to the major protocol version sent by the kernel for
+ * communication and reply with that major version (and an arbitrary
+ * supported minor version).
+ */
 
 /** Version number of this interface */
 #define FUSE_KERNEL_VERSION 7
 
 /** Minor version number of this interface */
-#define FUSE_KERNEL_MINOR_VERSION 13
+#define FUSE_KERNEL_MINOR_VERSION 22
 
 /** The node ID of the root inode */
 #define FUSE_ROOT_ID 1
@@ -85,42 +137,42 @@
    userspace works under 64bit kernels */
 
 struct fuse_attr {
-	__u64	ino;
-	__u64	size;
-	__u64	blocks;
-	__u64	atime;
-	__u64	mtime;
-	__u64	ctime;
-	__u32	atimensec;
-	__u32	mtimensec;
-	__u32	ctimensec;
-	__u32	mode;
-	__u32	nlink;
-	__u32	uid;
-	__u32	gid;
-	__u32	rdev;
-	__u32	blksize;
-	__u32	padding;
+	uint64_t	ino;
+	uint64_t	size;
+	uint64_t	blocks;
+	uint64_t	atime;
+	uint64_t	mtime;
+	uint64_t	ctime;
+	uint32_t	atimensec;
+	uint32_t	mtimensec;
+	uint32_t	ctimensec;
+	uint32_t	mode;
+	uint32_t	nlink;
+	uint32_t	uid;
+	uint32_t	gid;
+	uint32_t	rdev;
+	uint32_t	blksize;
+	uint32_t	padding;
 };
 
 struct fuse_kstatfs {
-	__u64	blocks;
-	__u64	bfree;
-	__u64	bavail;
-	__u64	files;
-	__u64	ffree;
-	__u32	bsize;
-	__u32	namelen;
-	__u32	frsize;
-	__u32	padding;
-	__u32	spare[6];
+	uint64_t	blocks;
+	uint64_t	bfree;
+	uint64_t	bavail;
+	uint64_t	files;
+	uint64_t	ffree;
+	uint32_t	bsize;
+	uint32_t	namelen;
+	uint32_t	frsize;
+	uint32_t	padding;
+	uint32_t	spare[6];
 };
 
 struct fuse_file_lock {
-	__u64	start;
-	__u64	end;
-	__u32	type;
-	__u32	pid; /* tgid */
+	uint64_t	start;
+	uint64_t	end;
+	uint32_t	type;
+	uint32_t	pid; /* tgid */
 };
 
 /**
@@ -151,8 +203,22 @@ struct fuse_file_lock {
 /**
  * INIT request/reply flags
  *
+ * FUSE_ASYNC_READ: asynchronous read requests
+ * FUSE_POSIX_LOCKS: remote locking for POSIX file locks
+ * FUSE_FILE_OPS: kernel sends file handle for fstat, etc... (not yet supported)
+ * FUSE_ATOMIC_O_TRUNC: handles the O_TRUNC open flag in the filesystem
  * FUSE_EXPORT_SUPPORT: filesystem handles lookups of "." and ".."
+ * FUSE_BIG_WRITES: filesystem can handle write size larger than 4kB
  * FUSE_DONT_MASK: don't apply umask to file mode on create operations
+ * FUSE_SPLICE_WRITE: kernel supports splice write on the device
+ * FUSE_SPLICE_MOVE: kernel supports splice move on the device
+ * FUSE_SPLICE_READ: kernel supports splice read on the device
+ * FUSE_FLOCK_LOCKS: remote locking for BSD style file locks
+ * FUSE_HAS_IOCTL_DIR: kernel supports ioctl on directories
+ * FUSE_AUTO_INVAL_DATA: automatically invalidate cached pages
+ * FUSE_DO_READDIRPLUS: do READDIRPLUS (READDIR+LOOKUP in one)
+ * FUSE_READDIRPLUS_AUTO: adaptive readdirplus
+ * FUSE_ASYNC_DIO: asynchronous direct I/O submission
  */
 #define FUSE_ASYNC_READ		(1 << 0)
 #define FUSE_POSIX_LOCKS	(1 << 1)
@@ -161,6 +227,15 @@ struct fuse_file_lock {
 #define FUSE_EXPORT_SUPPORT	(1 << 4)
 #define FUSE_BIG_WRITES		(1 << 5)
 #define FUSE_DONT_MASK		(1 << 6)
+#define FUSE_SPLICE_WRITE	(1 << 7)
+#define FUSE_SPLICE_MOVE	(1 << 8)
+#define FUSE_SPLICE_READ	(1 << 9)
+#define FUSE_FLOCK_LOCKS	(1 << 10)
+#define FUSE_HAS_IOCTL_DIR	(1 << 11)
+#define FUSE_AUTO_INVAL_DATA	(1 << 12)
+#define FUSE_DO_READDIRPLUS	(1 << 13)
+#define FUSE_READDIRPLUS_AUTO	(1 << 14)
+#define FUSE_ASYNC_DIO		(1 << 15)
 
 /**
  * CUSE INIT request/reply flags
@@ -173,6 +248,7 @@ struct fuse_file_lock {
  * Release flags
  */
 #define FUSE_RELEASE_FLUSH	(1 << 0)
+#define FUSE_RELEASE_FLOCK_UNLOCK	(1 << 1)
 
 /**
  * Getattr flags
@@ -204,12 +280,16 @@ struct fuse_file_lock {
  * FUSE_IOCTL_COMPAT: 32bit compat ioctl on 64bit machine
  * FUSE_IOCTL_UNRESTRICTED: not restricted to well-formed ioctls, retry allowed
  * FUSE_IOCTL_RETRY: retry with new iovecs
+ * FUSE_IOCTL_32BIT: 32bit ioctl
+ * FUSE_IOCTL_DIR: is a directory
  *
  * FUSE_IOCTL_MAX_IOV: maximum of in_iovecs + out_iovecs
  */
 #define FUSE_IOCTL_COMPAT	(1 << 0)
 #define FUSE_IOCTL_UNRESTRICTED	(1 << 1)
 #define FUSE_IOCTL_RETRY	(1 << 2)
+#define FUSE_IOCTL_32BIT	(1 << 3)
+#define FUSE_IOCTL_DIR		(1 << 4)
 
 #define FUSE_IOCTL_MAX_IOV	256
 
@@ -259,6 +339,10 @@ enum fuse_opcode {
 	FUSE_DESTROY       = 38,
 	FUSE_IOCTL         = 39,
 	FUSE_POLL          = 40,
+	FUSE_NOTIFY_REPLY  = 41,
+	FUSE_BATCH_FORGET  = 42,
+	FUSE_FALLOCATE     = 43,
+	FUSE_READDIRPLUS   = 44,
 
 	/* CUSE specific operations */
 	CUSE_INIT          = 4096,
@@ -268,6 +352,9 @@ enum fuse_notify_code {
 	FUSE_NOTIFY_POLL   = 1,
 	FUSE_NOTIFY_INVAL_INODE = 2,
 	FUSE_NOTIFY_INVAL_ENTRY = 3,
+	FUSE_NOTIFY_STORE = 4,
+	FUSE_NOTIFY_RETRIEVE = 5,
+	FUSE_NOTIFY_DELETE = 6,
 	FUSE_NOTIFY_CODE_MAX,
 };
 
@@ -277,133 +364,143 @@ enum fuse_notify_code {
 #define FUSE_COMPAT_ENTRY_OUT_SIZE 120
 
 struct fuse_entry_out {
-	__u64	nodeid;		/* Inode ID */
-	__u64	generation;	/* Inode generation: nodeid:gen must
-				   be unique for the fs's lifetime */
-	__u64	entry_valid;	/* Cache timeout for the name */
-	__u64	attr_valid;	/* Cache timeout for the attributes */
-	__u32	entry_valid_nsec;
-	__u32	attr_valid_nsec;
+	uint64_t	nodeid;		/* Inode ID */
+	uint64_t	generation;	/* Inode generation: nodeid:gen must
+					   be unique for the fs's lifetime */
+	uint64_t	entry_valid;	/* Cache timeout for the name */
+	uint64_t	attr_valid;	/* Cache timeout for the attributes */
+	uint32_t	entry_valid_nsec;
+	uint32_t	attr_valid_nsec;
 	struct fuse_attr attr;
 };
 
 struct fuse_forget_in {
-	__u64	nlookup;
+	uint64_t	nlookup;
+};
+
+struct fuse_forget_one {
+	uint64_t	nodeid;
+	uint64_t	nlookup;
+};
+
+struct fuse_batch_forget_in {
+	uint32_t	count;
+	uint32_t	dummy;
 };
 
 struct fuse_getattr_in {
-	__u32	getattr_flags;
-	__u32	dummy;
-	__u64	fh;
+	uint32_t	getattr_flags;
+	uint32_t	dummy;
+	uint64_t	fh;
 };
 
 #define FUSE_COMPAT_ATTR_OUT_SIZE 96
 
 struct fuse_attr_out {
-	__u64	attr_valid;	/* Cache timeout for the attributes */
-	__u32	attr_valid_nsec;
-	__u32	dummy;
+	uint64_t	attr_valid;	/* Cache timeout for the attributes */
+	uint32_t	attr_valid_nsec;
+	uint32_t	dummy;
 	struct fuse_attr attr;
 };
 
 #define FUSE_COMPAT_MKNOD_IN_SIZE 8
 
 struct fuse_mknod_in {
-	__u32	mode;
-	__u32	rdev;
-	__u32	umask;
-	__u32	padding;
+	uint32_t	mode;
+	uint32_t	rdev;
+	uint32_t	umask;
+	uint32_t	padding;
 };
 
 struct fuse_mkdir_in {
-	__u32	mode;
-	__u32	umask;
+	uint32_t	mode;
+	uint32_t	umask;
 };
 
 struct fuse_rename_in {
-	__u64	newdir;
+	uint64_t	newdir;
 };
 
 struct fuse_link_in {
-	__u64	oldnodeid;
+	uint64_t	oldnodeid;
 };
 
 struct fuse_setattr_in {
-	__u32	valid;
-	__u32	padding;
-	__u64	fh;
-	__u64	size;
-	__u64	lock_owner;
-	__u64	atime;
-	__u64	mtime;
-	__u64	unused2;
-	__u32	atimensec;
-	__u32	mtimensec;
-	__u32	unused3;
-	__u32	mode;
-	__u32	unused4;
-	__u32	uid;
-	__u32	gid;
-	__u32	unused5;
+	uint32_t	valid;
+	uint32_t	padding;
+	uint64_t	fh;
+	uint64_t	size;
+	uint64_t	lock_owner;
+	uint64_t	atime;
+	uint64_t	mtime;
+	uint64_t	unused2;
+	uint32_t	atimensec;
+	uint32_t	mtimensec;
+	uint32_t	unused3;
+	uint32_t	mode;
+	uint32_t	unused4;
+	uint32_t	uid;
+	uint32_t	gid;
+	uint32_t	unused5;
 };
 
 struct fuse_open_in {
-	__u32	flags;
-	__u32	unused;
+	uint32_t	flags;
+	uint32_t	unused;
 };
 
 struct fuse_create_in {
-	__u32	flags;
-	__u32	mode;
-	__u32	umask;
-	__u32	padding;
+	uint32_t	flags;
+	uint32_t	mode;
+	uint32_t	umask;
+	uint32_t	padding;
 };
 
 struct fuse_open_out {
-	__u64	fh;
-	__u32	open_flags;
-	__u32	padding;
+	uint64_t	fh;
+	uint32_t	open_flags;
+	uint32_t	padding;
 };
 
 struct fuse_release_in {
-	__u64	fh;
-	__u32	flags;
-	__u32	release_flags;
-	__u64	lock_owner;
+	uint64_t	fh;
+	uint32_t	flags;
+	uint32_t	release_flags;
+	uint64_t	lock_owner;
 };
 
 struct fuse_flush_in {
-	__u64	fh;
-	__u32	unused;
-	__u32	padding;
-	__u64	lock_owner;
+	uint64_t	fh;
+	uint32_t	unused;
+	uint32_t	padding;
+	uint64_t	lock_owner;
 };
 
 struct fuse_read_in {
-	__u64	fh;
-	__u64	offset;
-	__u32	size;
-	__u32	read_flags;
-	__u64	lock_owner;
-	__u32	flags;
-	__u32	padding;
+	uint64_t	fh;
+	uint64_t	offset;
+	uint32_t	size;
+	uint32_t	read_flags;
+	uint64_t	lock_owner;
+	uint32_t	flags;
+	uint32_t	padding;
 };
 
 #define FUSE_COMPAT_WRITE_IN_SIZE 24
 
 struct fuse_write_in {
-	__u64	fh;
-	__u64	offset;
-	__u32	size;
-	__u32	write_flags;
-	__u64	lock_owner;
-	__u32	flags;
-	__u32	padding;
+	uint64_t	fh;
+	uint64_t	offset;
+	uint32_t	size;
+	uint32_t	write_flags;
+	uint64_t	lock_owner;
+	uint32_t	flags;
+	uint32_t	padding;
 };
 
 struct fuse_write_out {
-	__u32	size;
-	__u32	padding;
+	uint32_t	size;
+	uint32_t	padding;
 };
 
 #define FUSE_COMPAT_STATFS_SIZE 48
@@ -413,32 +510,32 @@ struct fuse_statfs_out {
 };
 
 struct fuse_fsync_in {
-	__u64	fh;
-	__u32	fsync_flags;
-	__u32	padding;
+	uint64_t	fh;
+	uint32_t	fsync_flags;
+	uint32_t	padding;
 };
 
 struct fuse_setxattr_in {
-	__u32	size;
-	__u32	flags;
+	uint32_t	size;
+	uint32_t	flags;
 };
 
 struct fuse_getxattr_in {
-	__u32	size;
-	__u32	padding;
+	uint32_t	size;
+	uint32_t	padding;
 };
 
 struct fuse_getxattr_out {
-	__u32	size;
-	__u32	padding;
+	uint32_t	size;
+	uint32_t	padding;
 };
 
 struct fuse_lk_in {
-	__u64	fh;
-	__u64	owner;
+	uint64_t	fh;
+	uint64_t	owner;
 	struct fuse_file_lock lk;
-	__u32	lk_flags;
-	__u32	padding;
+	uint32_t	lk_flags;
+	uint32_t	padding;
 };
 
 struct fuse_lk_out {
@@ -446,134 +543,190 @@ struct fuse_lk_out {
 };
 
 struct fuse_access_in {
-	__u32	mask;
-	__u32	padding;
+	uint32_t	mask;
+	uint32_t	padding;
 };
 
 struct fuse_init_in {
-	__u32	major;
-	__u32	minor;
-	__u32	max_readahead;
-	__u32	flags;
+	uint32_t	major;
+	uint32_t	minor;
+	uint32_t	max_readahead;
+	uint32_t	flags;
 };
 
 struct fuse_init_out {
-	__u32	major;
-	__u32	minor;
-	__u32	max_readahead;
-	__u32	flags;
-	__u16   max_background;
-	__u16   congestion_threshold;
-	__u32	max_write;
+	uint32_t	major;
+	uint32_t	minor;
+	uint32_t	max_readahead;
+	uint32_t	flags;
+	uint16_t	max_background;
+	uint16_t	congestion_threshold;
+	uint32_t	max_write;
 };
 
 #define CUSE_INIT_INFO_MAX 4096
 
 struct cuse_init_in {
-	__u32	major;
-	__u32	minor;
-	__u32	unused;
-	__u32	flags;
+	uint32_t	major;
+	uint32_t	minor;
+	uint32_t	unused;
+	uint32_t	flags;
 };
 
 struct cuse_init_out {
-	__u32	major;
-	__u32	minor;
-	__u32	unused;
-	__u32	flags;
-	__u32	max_read;
-	__u32	max_write;
-	__u32	dev_major;		/* chardev major */
-	__u32	dev_minor;		/* chardev minor */
-	__u32	spare[10];
+	uint32_t	major;
+	uint32_t	minor;
+	uint32_t	unused;
+	uint32_t	flags;
+	uint32_t	max_read;
+	uint32_t	max_write;
+	uint32_t	dev_major;		/* chardev major */
+	uint32_t	dev_minor;		/* chardev minor */
+	uint32_t	spare[10];
 };
 
 struct fuse_interrupt_in {
-	__u64	unique;
+	uint64_t	unique;
 };
 
 struct fuse_bmap_in {
-	__u64	block;
-	__u32	blocksize;
-	__u32	padding;
+	uint64_t	block;
+	uint32_t	blocksize;
+	uint32_t	padding;
 };
 
 struct fuse_bmap_out {
-	__u64	block;
+	uint64_t	block;
 };
 
 struct fuse_ioctl_in {
-	__u64	fh;
-	__u32	flags;
-	__u32	cmd;
-	__u64	arg;
-	__u32	in_size;
-	__u32	out_size;
+	uint64_t	fh;
+	uint32_t	flags;
+	uint32_t	cmd;
+	uint64_t	arg;
+	uint32_t	in_size;
+	uint32_t	out_size;
+};
+
+struct fuse_ioctl_iovec {
+	uint64_t	base;
+	uint64_t	len;
 };
 
 struct fuse_ioctl_out {
-	__s32	result;
-	__u32	flags;
-	__u32	in_iovs;
-	__u32	out_iovs;
+	int32_t		result;
+	uint32_t	flags;
+	uint32_t	in_iovs;
+	uint32_t	out_iovs;
 };
 
 struct fuse_poll_in {
-	__u64	fh;
-	__u64	kh;
-	__u32	flags;
-	__u32   padding;
+	uint64_t	fh;
+	uint64_t	kh;
+	uint32_t	flags;
+	uint32_t	events;
 };
 
 struct fuse_poll_out {
-	__u32	revents;
-	__u32	padding;
+	uint32_t	revents;
+	uint32_t	padding;
 };
 
 struct fuse_notify_poll_wakeup_out {
-	__u64	kh;
+	uint64_t	kh;
+};
+
+struct fuse_fallocate_in {
+	uint64_t	fh;
+	uint64_t	offset;
+	uint64_t	length;
+	uint32_t	mode;
+	uint32_t	padding;
 };
 
 struct fuse_in_header {
-	__u32	len;
-	__u32	opcode;
-	__u64	unique;
-	__u64	nodeid;
-	__u32	uid;
-	__u32	gid;
-	__u32	pid;
-	__u32	padding;
+	uint32_t	len;
+	uint32_t	opcode;
+	uint64_t	unique;
+	uint64_t	nodeid;
+	uint32_t	uid;
+	uint32_t	gid;
+	uint32_t	pid;
+	uint32_t	padding;
 };
 
 struct fuse_out_header {
-	__u32	len;
-	__s32	error;
-	__u64	unique;
+	uint32_t	len;
+	int32_t		error;
+	uint64_t	unique;
 };
 
 struct fuse_dirent {
-	__u64	ino;
-	__u64	off;
-	__u32	namelen;
-	__u32	type;
-	char name[0];
+	uint64_t	ino;
+	uint64_t	off;
+	uint32_t	namelen;
+	uint32_t	type;
+	char name[];
 };
 
 #define FUSE_NAME_OFFSET offsetof(struct fuse_dirent, name)
-#define FUSE_DIRENT_ALIGN(x) (((x) + sizeof(__u64) - 1) & ~(sizeof(__u64) - 1))
+#define FUSE_DIRENT_ALIGN(x) \
+	(((x) + sizeof(uint64_t) - 1) & ~(sizeof(uint64_t) - 1))
 #define FUSE_DIRENT_SIZE(d) \
 	FUSE_DIRENT_ALIGN(FUSE_NAME_OFFSET + (d)->namelen)
 
+struct fuse_direntplus {
+	struct fuse_entry_out entry_out;
+	struct fuse_dirent dirent;
+};
+
+#define FUSE_NAME_OFFSET_DIRENTPLUS \
+	offsetof(struct fuse_direntplus, dirent.name)
+#define FUSE_DIRENTPLUS_SIZE(d) \
+	FUSE_DIRENT_ALIGN(FUSE_NAME_OFFSET_DIRENTPLUS + (d)->dirent.namelen)
+
 struct fuse_notify_inval_inode_out {
-	__u64	ino;
-	__s64	off;
-	__s64	len;
+	uint64_t	ino;
+	int64_t		off;
+	int64_t		len;
 };
 
 struct fuse_notify_inval_entry_out {
-	__u64	parent;
-	__u32	namelen;
-	__u32	padding;
+	uint64_t	parent;
+	uint32_t	namelen;
+	uint32_t	padding;
+};
+
+struct fuse_notify_delete_out {
+	uint64_t	parent;
+	uint64_t	child;
+	uint32_t	namelen;
+	uint32_t	padding;
+};
+
+struct fuse_notify_store_out {
+	uint64_t	nodeid;
+	uint64_t	offset;
+	uint32_t	size;
+	uint32_t	padding;
+};
+
+struct fuse_notify_retrieve_out {
+	uint64_t	notify_unique;
+	uint64_t	nodeid;
+	uint64_t	offset;
+	uint32_t	size;
+	uint32_t	padding;
+};
+
+/* Matches the size of fuse_write_in */
+struct fuse_notify_retrieve_in {
+	uint64_t	dummy1;
+	uint64_t	offset;
+	uint32_t	size;
+	uint32_t	dummy2;
+	uint64_t	dummy3;
+	uint64_t	dummy4;
 };
 
 #endif /* _LINUX_FUSE_H */
diff --git a/contrib/fuse-util/mount_util.h b/contrib/fuse-include/mount_util.h
index f392f99f1..f392f99f1 100644
--- a/contrib/fuse-util/mount_util.h
+++ b/contrib/fuse-include/mount_util.h
diff --git a/contrib/fuse-lib/mount-common.c b/contrib/fuse-lib/mount-common.c
new file mode 100644
index 000000000..fd6cce44e
--- /dev/null
+++ b/contrib/fuse-lib/mount-common.c
@@ -0,0 +1,265 @@
+/*
+  FUSE: Filesystem in Userspace
+  Copyright (C) 2001-2007  Miklos Szeredi <miklos@szeredi.hu>
+  Copyright (c) 2010 Gluster, Inc. <http://www.gluster.com>
+
+  This program can be distributed under the terms of the GNU LGPLv2.
+  See the file COPYING.LIB.
+*/
+
+#include "mount-gluster-compat.h"
+
+/*
+ * These functions (and gf_fuse_umount() in mount.c)
+ * were originally taken from libfuse as of commit 7960e99e
+ * (http://fuse.git.sourceforge.net/git/gitweb.cgi?p=fuse/fuse;a=commit;h=7960e99e)
+ * almost verbatim. What has been changed upon adoption:
+ * - style adopted to that of glusterfs
+ * - s/fprintf/gf_log/
+ * - s/free/FREE/, s/malloc/MALLOC/
+ * - there are some other minor things
+ *
+ * For changes that were made later and syncs with upstream,
+ * see the commit log and per-function comments.
+ */
+
+#ifndef __NetBSD__
+/* FUSE: cherry-picked bd99f9cf */
+static int
+mtab_needs_update (const char *mnt)
+{
+        int res;
+        struct stat stbuf;
+
+        /* If mtab is within new mount, don't touch it */
+        if (strncmp (mnt, _PATH_MOUNTED, strlen (mnt)) == 0 &&
+            _PATH_MOUNTED[strlen (mnt)] == '/')
+                return 0;
+
+        /*
+         * Skip mtab update if /etc/mtab:
+         *
+         *  - doesn't exist,
+         *  - is a symlink,
+         *  - is on a read-only filesystem.
+         */
+        res = lstat (_PATH_MOUNTED, &stbuf);
+        if (res == -1) {
+                if (errno == ENOENT)
+                        return 0;
+        } else {
+                uid_t ruid;
+                int err;
+
+                if (S_ISLNK (stbuf.st_mode))
+                        return 0;
+
+                ruid = getuid ();
+                if (ruid != 0)
+                        setreuid (0, -1);
+
+                res = access (_PATH_MOUNTED, W_OK);
+                err = (res == -1) ? errno : 0;
+                if (ruid != 0)
+                        setreuid (ruid, -1);
+
+                if (err == EROFS)
+                        return 0;
+        }
+
+        return 1;
+}
+#else /* __NetBSD__ */
+#define mtab_needs_update(x) 1
+#endif /* __NetBSD__ */
+
+/* FUSE: called add_mount_legacy(); R.I.P. as of cbd3a2a8 */
+int
+fuse_mnt_add_mount (const char *progname, const char *fsname,
+                    const char *mnt, const char *type, const char *opts)
+{
+        int res;
+        int status;
+        sigset_t blockmask;
+        sigset_t oldmask;
+
+        if (!mtab_needs_update (mnt))
+                return 0;
+
+        sigemptyset (&blockmask);
+        sigaddset (&blockmask, SIGCHLD);
+        res = sigprocmask (SIG_BLOCK, &blockmask, &oldmask);
+        if (res == -1) {
+                GFFUSE_LOGERR ("%s: sigprocmask: %s",
+                               progname, strerror (errno));
+                return -1;
+        }
+
+        res = fork ();
+        if (res == -1) {
+                GFFUSE_LOGERR ("%s: fork: %s", progname, strerror (errno));
+                goto out_restore;
+        }
+        if (res == 0) {
+                char templ[] = "/tmp/fusermountXXXXXX";
+                char *tmp;
+
+                sigprocmask (SIG_SETMASK, &oldmask, NULL);
+                setuid (geteuid ());
+
+                /*
+                 * hide in a directory, where mount isn't able to resolve
+                 * fsname as a valid path
+                 */
+                tmp = mkdtemp (templ);
+                if (!tmp) {
+                        GFFUSE_LOGERR ("%s: failed to create temporary directory",
+                                       progname);
+                        exit (1);
+                }
+                if (chdir (tmp)) {
+                        GFFUSE_LOGERR ("%s: failed to chdir to %s: %s",
+                                       progname, tmp, strerror (errno));
+                        exit (1);
+                }
+                rmdir (tmp);
+                execl (_PATH_MOUNT, _PATH_MOUNT, "-i", "-f", "-t", type,
+                       "-o", opts, fsname, mnt, NULL);
+                GFFUSE_LOGERR ("%s: failed to execute %s: %s",
+                               progname, _PATH_MOUNT, strerror (errno));
+                exit (1);
+        }
+
+        res = waitpid (res, &status, 0);
+        if (res == -1)
+                GFFUSE_LOGERR ("%s: waitpid: %s", progname, strerror (errno));
+        res = (res != -1 && status == 0) ? 0 : -1;
+
+ out_restore:
+        sigprocmask (SIG_SETMASK, &oldmask, NULL);
+        return res;
+}
+
+char *
+fuse_mnt_resolve_path (const char *progname, const char *orig)
+{
+        char buf[PATH_MAX];
+        char *copy;
+        char *dst;
+        char *end;
+        char *lastcomp;
+        const char *toresolv;
+
+        if (!orig[0]) {
+                GFFUSE_LOGERR ("%s: invalid mountpoint '%s'", progname, orig);
+                return NULL;
+        }
+
+        copy = strdup (orig);
+        if (copy == NULL) {
+                GFFUSE_LOGERR ("%s: failed to allocate memory", progname);
+                return NULL;
+        }
+
+        toresolv = copy;
+        lastcomp = NULL;
+        for (end = copy + strlen (copy) - 1; end > copy && *end == '/'; end --);
+        if (end[0] != '/') {
+                char *tmp;
+                end[1] = '\0';
+                tmp = strrchr (copy, '/');
+                if (tmp == NULL) {
+                        lastcomp = copy;
+                        toresolv = ".";
+                } else {
+                        lastcomp = tmp + 1;
+                        if (tmp == copy)
+                                toresolv = "/";
+                }
+                if (strcmp (lastcomp, ".") == 0 || strcmp (lastcomp, "..") == 0) {
+                        lastcomp = NULL;
+                        toresolv = copy;
+                }
+                else if (tmp)
+                        tmp[0] = '\0';
+        }
+        if (realpath (toresolv, buf) == NULL) {
+                GFFUSE_LOGERR ("%s: bad mount point %s: %s", progname, orig,
+                               strerror (errno));
+                FREE (copy);
+                return NULL;
+        }
+        if (lastcomp == NULL)
+                dst = strdup (buf);
+        else {
+                dst = (char *) MALLOC (strlen (buf) + 1 + strlen (lastcomp) + 1);
+                if (dst) {
+                        unsigned buflen = strlen (buf);
+                        if (buflen && buf[buflen-1] == '/')
+                                sprintf (dst, "%s%s", buf, lastcomp);
+                        else
+                                sprintf (dst, "%s/%s", buf, lastcomp);
+                }
+        }
+        FREE (copy);
+        if (dst == NULL)
+                GFFUSE_LOGERR ("%s: failed to allocate memory", progname);
+        return dst;
+}
+
+/* FUSE: to support some changes that were reverted since
+ * then, it was split in two (fuse_mnt_umount() and
+ * exec_umount()); however the actual code is same as here
+ * since 0197ce40
+ */
+int
+fuse_mnt_umount (const char *progname, const char *abs_mnt,
+                 const char *rel_mnt, int lazy)
+{
+        int res;
+        int status;
+        sigset_t blockmask;
+        sigset_t oldmask;
+
+        if (!mtab_needs_update (abs_mnt)) {
+                res = umount2 (rel_mnt, lazy ? 2 : 0);
+                if (res == -1)
+                        GFFUSE_LOGERR ("%s: failed to unmount %s: %s",
+                                       progname, abs_mnt, strerror (errno));
+                return res;
+        }
+
+        sigemptyset (&blockmask);
+        sigaddset (&blockmask, SIGCHLD);
+        res = sigprocmask (SIG_BLOCK, &blockmask, &oldmask);
+        if (res == -1) {
+                GFFUSE_LOGERR ("%s: sigprocmask: %s", progname,
+                               strerror (errno));
+                return -1;
+        }
+
+        res = fork ();
+        if (res == -1) {
+                GFFUSE_LOGERR ("%s: fork: %s", progname, strerror (errno));
+                goto out_restore;
+        }
+        if (res == 0) {
+                sigprocmask (SIG_SETMASK, &oldmask, NULL);
+                setuid (geteuid ());
+                execl ("/bin/umount", "/bin/umount", "-i", rel_mnt,
+                      lazy ? "-l" : NULL, NULL);
+                GFFUSE_LOGERR ("%s: failed to execute /bin/umount: %s",
+                               progname, strerror (errno));
+                exit (1);
+        }
+        res = waitpid (res, &status, 0);
+        if (res == -1)
+                GFFUSE_LOGERR ("%s: waitpid: %s", progname, strerror (errno));
+
+        if (status != 0)
+                res = -1;
+
+ out_restore:
+        sigprocmask (SIG_SETMASK, &oldmask, NULL);
+        return res;
+}
diff --git a/contrib/fuse-lib/mount-gluster-compat.h b/contrib/fuse-lib/mount-gluster-compat.h
new file mode 100644
index 000000000..4fc20623b
--- /dev/null
+++ b/contrib/fuse-lib/mount-gluster-compat.h
@@ -0,0 +1,56 @@
+/*
+  FUSE: Filesystem in Userspace
+  Copyright (C) 2001-2007  Miklos Szeredi <miklos@szeredi.hu>
+  Copyright (c) 2010 Gluster, Inc. <http://www.gluster.com>
+
+  This program can be distributed under the terms of the GNU LGPLv2.
+  See the file COPYING.LIB.
+*/
+
+#ifndef _CONFIG_H
+#define _CONFIG_H
+#include "config.h"
+#endif
+
+#include <stdio.h>
+#include <stdlib.h>
+#include <unistd.h>
+#include <stddef.h>
+#include <limits.h>
+#include <fcntl.h>
+#include <errno.h>
+#include <dirent.h>
+#include <signal.h>
+#ifndef __NetBSD__
+#include <mntent.h>
+#endif /* __NetBSD__ */
+#include <sys/stat.h>
+#include <sys/poll.h>
+#include <sys/un.h>
+#include <sys/wait.h>
+#include <sys/mount.h>
+
+#ifdef __NetBSD__
+#include <perfuse.h>
+#define umount2(dir, flags) unmount(dir, ((flags) != 0) ? MNT_FORCE : 0)
+#define MS_RDONLY MNT_RDONLY
+#endif
+
+#ifdef linux
+#define _PATH_MOUNT "/bin/mount"
+#else /* NetBSD, MacOS X */
+#define _PATH_MOUNT "/sbin/mount"
+#endif
+
+#ifdef FUSE_UTIL
+#define MALLOC(size) malloc (size)
+#define FREE(ptr) free (ptr)
+#define GFFUSE_LOGERR(...) fprintf (stderr, ## __VA_ARGS__)
+#else /* FUSE_UTIL */
+#include "glusterfs.h"
+#include "logging.h"
+#include "common-utils.h"
+
+#define GFFUSE_LOGERR(...) \
+        gf_log ("glusterfs-fuse", GF_LOG_ERROR, ## __VA_ARGS__)
+#endif /* !FUSE_UTIL */
diff --git a/contrib/fuse-lib/mount.c b/contrib/fuse-lib/mount.c
index 800fd193e..922d9e464 100644
--- a/contrib/fuse-lib/mount.c
+++ b/contrib/fuse-lib/mount.c
@@ -7,342 +7,140 @@
   See the file COPYING.LIB.
 */
 
-#ifndef _CONFIG_H
-#define _CONFIG_H
-#include "config.h"
-#endif
-
-#include <stdio.h>
-#include <stdlib.h>
-#include <unistd.h>
-#include <stddef.h>
-#include <limits.h>
-#include <fcntl.h>
-#include <errno.h>
-#include <dirent.h>
-#include <signal.h>
-#ifndef __NetBSD__
-#include <mntent.h>
-#endif /* __NetBSD__ */
-#include <sys/stat.h>
-#include <sys/poll.h>
-#include <sys/socket.h>
-#include <sys/un.h>
-#include <sys/wait.h>
-#include <sys/mount.h>
-
-#ifdef __NetBSD__
-#include <perfuse.h>
-#define umount2(dir, flags) unmount(dir, ((flags) != 0) ? MNT_FORCE : 0)
-#endif
-
-#ifdef linux
-#define _PATH_MOUNT "/bin/mount"
-#else /* NetBSD, MacOS X */
-#define _PATH_MOUNT "/sbin/mount"
-#endif
-
-#ifdef FUSE_UTIL
-#define MALLOC(size) malloc (size)
-#define FREE(ptr) free (ptr)
-#define GFFUSE_LOGERR(...) fprintf (stderr, ## __VA_ARGS__)
-#else /* FUSE_UTIL */
-#include "glusterfs.h"
-#include "logging.h"
-#include "common-utils.h"
+#include "mount_util.h"
+#include "mount-gluster-compat.h"
 
 #ifdef GF_FUSERMOUNT
 #define FUSERMOUNT_PROG FUSERMOUNT_DIR "/fusermount-glusterfs"
 #else
 #define FUSERMOUNT_PROG "fusermount"
 #endif
-#define FUSE_COMMFD_ENV "_FUSE_COMMFD"
+#define FUSE_DEVFD_ENV "_FUSE_DEVFD"
 
-#define GFFUSE_LOGERR(...) \
-        gf_log ("glusterfs-fuse", GF_LOG_ERROR, ## __VA_ARGS__)
-#endif /* !FUSE_UTIL */
 
-/*
- * Functions below, until following note, were taken from libfuse
- * (http://git.gluster.com/?p=users/csaba/fuse.git;a=commit;h=b988bbf9)
- * almost verbatim. What has been changed:
- * - style adopted to that of glusterfs
- * - s/fprintf/gf_log/
- * - s/free/FREE/, s/malloc/MALLOC/
- * - there are some other minor things
- */
-
-#ifndef __NetBSD__
-static int
-mtab_needs_update (const char *mnt)
+/* FUSE: function is called fuse_kern_unmount() */
+void
+gf_fuse_unmount (const char *mountpoint, int fd)
 {
         int res;
-        struct stat stbuf;
-
-        /* If mtab is within new mount, don't touch it */
-        if (strncmp (mnt, _PATH_MOUNTED, strlen (mnt)) == 0 &&
-            _PATH_MOUNTED[strlen (mnt)] == '/')
-                return 0;
-
-        /*
-         * Skip mtab update if /etc/mtab:
-         *
-         *  - doesn't exist,
-         *  - is a symlink,
-         *  - is on a read-only filesystem.
-         */
-        res = lstat (_PATH_MOUNTED, &stbuf);
-        if (res == -1) {
-                if (errno == ENOENT)
-                        return 0;
-        } else {
-                if (S_ISLNK (stbuf.st_mode))
-                        return 0;
-
-                res = access (_PATH_MOUNTED, W_OK);
-                if (res == -1 && errno == EROFS)
-                        return 0;
-        }
+        int pid;
 
-        return 1;
-}
-#else /* __NetBSD__ */
-#define mtab_needs_update(x) 1
-#endif /* __NetBSD__ */
+        if (!mountpoint)
+                return;
 
-#ifndef FUSE_UTIL
-static
-#endif
-int
-fuse_mnt_add_mount (const char *progname, const char *fsname,
-                    const char *mnt, const char *type, const char *opts,
-                    pid_t *mtab_pid)
-{
-        int res;
-        int status;
-        sigset_t blockmask;
-        sigset_t oldmask;
-
-        if (!mtab_needs_update (mnt))
-                return 0;
-
-        sigemptyset (&blockmask);
-        sigaddset (&blockmask, SIGCHLD);
-        res = sigprocmask (SIG_BLOCK, &blockmask, &oldmask);
-        if (res == -1) {
-                GFFUSE_LOGERR ("%s: sigprocmask: %s",
-                               progname, strerror (errno));
-                return -1;
-        }
+        if (fd != -1) {
+                struct pollfd pfd;
 
-        res = fork ();
-        if (res == -1) {
-                GFFUSE_LOGERR ("%s: fork: %s", progname, strerror (errno));
-                goto out_restore;
-        }
-        if (res == 0) {
-                char templ[] = "/tmp/fusermountXXXXXX";
-                char *tmp;
-
-                if (!mtab_pid) {
-                        /* mtab update done async, just log if fails */
-                        res = fork ();
-                        if (res)
-                                exit (res == -1 ? 1 : 0);
-                        res = fork ();
-                        if (res) {
-                                if (res != -1) {
-                                        if (!(res == waitpid (res, &status, 0)
-                                              && status == 0))
-                                                GFFUSE_LOGERR ("%s: /etc/mtab "
-                                                               "update failed",
-                                                               progname);
-                                }
-                                exit (0);
-                        }
-                }
+                pfd.fd = fd;
+                pfd.events = 0;
+                res = poll (&pfd, 1, 0);
+                /* If file poll returns POLLERR on the device file descriptor,
+                   then the filesystem is already unmounted */
+                if (res == 1 && (pfd.revents & POLLERR))
+                        return;
 
-                sigprocmask (SIG_SETMASK, &oldmask, NULL);
-                setuid (geteuid ());
-
-                /*
-                 * hide in a directory, where mount isn't able to resolve
-                 * fsname as a valid path
-                 */
-                tmp = mkdtemp (templ);
-                if (!tmp) {
-                        GFFUSE_LOGERR ("%s: failed to create temporary directory",
-                                       progname);
-                        exit (1);
-                }
-                if (chdir (tmp)) {
-                        GFFUSE_LOGERR ("%s: failed to chdir to %s: %s",
-                                       progname, tmp, strerror (errno));
-                        exit (1);
-                }
-                rmdir (tmp);
-                execl (_PATH_MOUNT, _PATH_MOUNT, "-i", "-f", "-t", type,
-                       "-o", opts, fsname, mnt, NULL);
-                GFFUSE_LOGERR ("%s: failed to execute %s: %s",
-                               progname, _PATH_MOUNT, strerror (errno));
-                exit (1);
+                /* Need to close file descriptor, otherwise synchronous umount
+                   would recurse into filesystem, and deadlock */
+                close (fd);
         }
-        if (mtab_pid) {
-                *mtab_pid = res;
-                res = 0;
-        } else {
-                if (!(res == waitpid (res, &status, 0) && status == 0))
-                        res = -1;
+
+        if (geteuid () == 0) {
+                fuse_mnt_umount ("fuse", mountpoint, mountpoint, 1);
+                return;
         }
-        if (res == -1)
-                GFFUSE_LOGERR ("%s: waitpid: %s", progname, strerror (errno));
 
- out_restore:
-        sigprocmask (SIG_SETMASK, &oldmask, NULL);
-        return res;
-}
+        res = umount2 (mountpoint, 2);
+        if (res == 0)
+                return;
 
-#ifndef FUSE_UTIL
-static
-#endif
-char
-*fuse_mnt_resolve_path (const char *progname, const char *orig)
-{
-        char buf[PATH_MAX];
-        char *copy;
-        char *dst;
-        char *end;
-        char *lastcomp;
-        const char *toresolv;
-
-        if (!orig[0]) {
-                GFFUSE_LOGERR ("%s: invalid mountpoint '%s'", progname, orig);
-                return NULL;
-        }
+        pid = fork ();
+        if (pid == -1)
+                return;
 
-        copy = strdup (orig);
-        if (copy == NULL) {
-                GFFUSE_LOGERR ("%s: failed to allocate memory", progname);
-                return NULL;
-        }
+        if (pid == 0) {
+                const char *argv[] = { FUSERMOUNT_PROG, "-u", "-q", "-z",
+                                       "--", mountpoint, NULL };
 
-        toresolv = copy;
-        lastcomp = NULL;
-        for (end = copy + strlen (copy) - 1; end > copy && *end == '/'; end --);
-        if (end[0] != '/') {
-                char *tmp;
-                end[1] = '\0';
-                tmp = strrchr (copy, '/');
-                if (tmp == NULL) {
-                        lastcomp = copy;
-                        toresolv = ".";
-                } else {
-                        lastcomp = tmp + 1;
-                        if (tmp == copy)
-                                toresolv = "/";
-                }
-                if (strcmp (lastcomp, ".") == 0 || strcmp (lastcomp, "..") == 0) {
-                        lastcomp = NULL;
-                        toresolv = copy;
-                }
-                else if (tmp)
-                        tmp[0] = '\0';
-        }
-        if (realpath (toresolv, buf) == NULL) {
-                GFFUSE_LOGERR ("%s: bad mount point %s: %s", progname, orig,
-                               strerror (errno));
-                FREE (copy);
-                return NULL;
-        }
-        if (lastcomp == NULL)
-                dst = strdup (buf);
-        else {
-                dst = (char *) MALLOC (strlen (buf) + 1 + strlen (lastcomp) + 1);
-                if (dst) {
-                        unsigned buflen = strlen (buf);
-                        if (buflen && buf[buflen-1] == '/')
-                                sprintf (dst, "%s%s", buf, lastcomp);
-                        else
-                                sprintf (dst, "%s/%s", buf, lastcomp);
-                }
+                execvp (FUSERMOUNT_PROG, (char **)argv);
+                _exit (1);
         }
-        FREE (copy);
-        if (dst == NULL)
-                GFFUSE_LOGERR ("%s: failed to allocate memory", progname);
-        return dst;
+        waitpid (pid, NULL, 0);
 }
 
-#ifndef FUSE_UTIL
-/* return value:
- * >= 0         => fd
- * -1         => error
- */
-static int
-receive_fd (int fd)
+
+/* gluster-specific routines */
+
+static char *
+escape (char *s)
 {
-        struct msghdr msg;
-        struct iovec iov;
-        char buf[1];
-        int rv;
-        size_t ccmsg[CMSG_SPACE (sizeof (int)) / sizeof (size_t)];
-        struct cmsghdr *cmsg;
-        int *recv_fd;
-
-        iov.iov_base = buf;
-        iov.iov_len = 1;
-
-        msg.msg_name = 0;
-        msg.msg_namelen = 0;
-        msg.msg_iov = &iov;
-        msg.msg_iovlen = 1;
-        /* old BSD implementations should use msg_accrights instead of
-         * msg_control; the interface is different. */
-        msg.msg_control = ccmsg;
-        msg.msg_controllen = sizeof (ccmsg);
-
-        while (((rv = recvmsg (fd, &msg, 0)) == -1) && errno == EINTR);
-        if (rv == -1) {
-                GFFUSE_LOGERR ("recvmsg failed: %s", strerror (errno));
-                return -1;
-        }
-        if (!rv) {
-                /* EOF */
-                return -1;
+        size_t len = 0;
+        char *p = NULL;
+        char *q = NULL;
+        char *e = NULL;
+
+        for (p = s; *p; p++) {
+                if (*p == ',')
+                       len++;
+                len++;
         }
 
-        cmsg = CMSG_FIRSTHDR (&msg);
-        /*
-         * simplify condition expression
-         */
-        if (cmsg->cmsg_type != SCM_RIGHTS) {
-                GFFUSE_LOGERR ("got control message of unknown type %d",
-                               cmsg->cmsg_type);
-                return -1;
+        e = CALLOC (1, len + 1);
+        if (!e)
+                return NULL;
+
+        for (p = s, q = e; *p; p++, q++) {
+                if (*p == ',') {
+                        *q = '\\';
+                        q++;
+                }
+                *q = *p;
         }
 
-        recv_fd = (int *) CMSG_DATA (cmsg);
-        return (*recv_fd);
+        return e;
 }
 
 static int
-fuse_mount_fusermount (const char *mountpoint, const char *opts)
+fuse_mount_fusermount (const char *mountpoint, char *fsname,
+                       unsigned long mountflags, char *mnt_param,
+                       int fd)
 {
-        int fds[2], pid;
-        int res;
-        int rv;
+        int  pid = -1;
+        int  res = 0;
+        int  ret = -1;
+        char *fm_mnt_params = NULL;
+        char *efsname = NULL;
+
+#ifndef GF_FUSERMOUNT
+        GFFUSE_LOGERR ("Mounting via helper utility "
+                       "(unprivileged mounting) is supported "
+                       "only if glusterfs is compiled with "
+                       "--enable-fusermount");
+        return -1;
+#endif
+
+        efsname = escape (fsname);
+        if (!efsname) {
+                GFFUSE_LOGERR ("Out of memory");
 
-        res = socketpair (PF_UNIX, SOCK_STREAM, 0, fds);
-        if (res == -1) {
-                GFFUSE_LOGERR ("socketpair() failed: %s", strerror (errno));
                 return -1;
         }
+        ret = asprintf (&fm_mnt_params,
+                        "%s%s,fsname=%s,nonempty,subtype=glusterfs",
+                        (mountflags & MS_RDONLY) ? "ro," : "",
+                        mnt_param, efsname);
+        FREE (efsname);
+        if (ret == -1) {
+                GFFUSE_LOGERR ("Out of memory");
+
+                goto out;
+        }
 
+        /* fork to exec fusermount */
         pid = fork ();
         if (pid == -1) {
                 GFFUSE_LOGERR ("fork() failed: %s", strerror (errno));
-                close (fds[0]);
-                close (fds[1]);
-                return -1;
+                ret = -1;
+                goto out;
         }
 
         if (pid == 0) {
@@ -351,214 +149,37 @@ fuse_mount_fusermount (const char *mountpoint, const char *opts)
                 int a = 0;
 
                 argv[a++] = FUSERMOUNT_PROG;
-                if (opts) {
-                        argv[a++] = "-o";
-                        argv[a++] = opts;
-                }
+                argv[a++] = "-o";
+                argv[a++] = fm_mnt_params;
                 argv[a++] = "--";
                 argv[a++] = mountpoint;
                 argv[a++] = NULL;
 
-                close (fds[1]);
-                fcntl (fds[0], F_SETFD, 0);
-                snprintf (env, sizeof (env), "%i", fds[0]);
-                setenv (FUSE_COMMFD_ENV, env, 1);
+                snprintf (env, sizeof (env), "%i", fd);
+                setenv (FUSE_DEVFD_ENV, env, 1);
                 execvp (FUSERMOUNT_PROG, (char **)argv);
                 GFFUSE_LOGERR ("failed to exec fusermount: %s",
                                strerror (errno));
                 _exit (1);
         }
 
-        close (fds[0]);
-        rv = receive_fd (fds[1]);
-        close (fds[1]);
-        waitpid (pid, NULL, 0); /* bury zombie */
-
-        return rv;
-}
-#endif
-
-#ifndef FUSE_UTIL
-static
-#endif
-int
-fuse_mnt_umount (const char *progname, const char *abs_mnt,
-                 const char *rel_mnt, int lazy)
-{
-        int res;
-        int status;
-        sigset_t blockmask;
-        sigset_t oldmask;
-
-        if (!mtab_needs_update (abs_mnt)) {
-                res = umount2 (rel_mnt, lazy ? 2 : 0);
-                if (res == -1)
-                        GFFUSE_LOGERR ("%s: failed to unmount %s: %s",
-                                       progname, abs_mnt, strerror (errno));
-                return res;
-        }
-
-        sigemptyset (&blockmask);
-        sigaddset (&blockmask, SIGCHLD);
-        res = sigprocmask (SIG_BLOCK, &blockmask, &oldmask);
-        if (res == -1) {
-                GFFUSE_LOGERR ("%s: sigprocmask: %s", progname,
-                               strerror (errno));
-                return -1;
-        }
-
-        res = fork ();
-        if (res == -1) {
-                GFFUSE_LOGERR ("%s: fork: %s", progname, strerror (errno));
-                goto out_restore;
-        }
-        if (res == 0) {
-                sigprocmask (SIG_SETMASK, &oldmask, NULL);
-                setuid (geteuid ());
-                execl ("/bin/umount", "/bin/umount", "-i", rel_mnt,
-                      lazy ? "-l" : NULL, NULL);
-                GFFUSE_LOGERR ("%s: failed to execute /bin/umount: %s",
-                               progname, strerror (errno));
-                exit (1);
-        }
-        res = waitpid (res, &status, 0);
-        if (res == -1)
-                GFFUSE_LOGERR ("%s: waitpid: %s", progname, strerror (errno));
-
-        if (status != 0)
-                res = -1;
-
- out_restore:
-        sigprocmask (SIG_SETMASK, &oldmask, NULL);
-        return res;
-}
-
-#ifdef FUSE_UTIL
-int
-fuse_mnt_check_empty (const char *progname, const char *mnt,
-                      mode_t rootmode, off_t rootsize)
-{
-        int isempty = 1;
-
-        if (S_ISDIR (rootmode)) {
-                struct dirent *ent;
-                DIR *dp = opendir (mnt);
-                if (dp == NULL) {
-                        fprintf (stderr,
-                                 "%s: failed to open mountpoint for reading: %s\n",
-                                 progname, strerror (errno));
-                        return -1;
-                }
-                while ((ent = readdir (dp)) != NULL) {
-                        if (strcmp (ent->d_name, ".") != 0 &&
-                            strcmp (ent->d_name, "..") != 0) {
-                                isempty = 0;
-                                break;
-                        }
-                }
-                closedir (dp);
-        } else if (rootsize)
-                isempty = 0;
-
-        if (!isempty) {
-                fprintf (stderr, "%s: mountpoint is not empty\n", progname);
-                fprintf (stderr, "%s: if you are sure this is safe, "
-                         "use the 'nonempty' mount option\n", progname);
-                return -1;
-        }
-        return 0;
-}
-
-int
-fuse_mnt_check_fuseblk (void)
-{
-        char buf[256];
-        FILE *f = fopen ("/proc/filesystems", "r");
-        if (!f)
-                return 1;
-
-        while (fgets (buf, sizeof (buf), f))
-                if (strstr (buf, "fuseblk\n")) {
-                        fclose (f);
-                        return 1;
-                }
-
-        fclose (f);
-        return 0;
-}
-#endif
-
-#ifndef FUSE_UTIL
-void
-gf_fuse_unmount (const char *mountpoint, int fd)
-{
-        int res;
-        int pid;
-
-        if (!mountpoint)
-                return;
-
-        if (fd != -1) {
-                struct pollfd pfd;
-
-                pfd.fd = fd;
-                pfd.events = 0;
-                res = poll (&pfd, 1, 0);
-                /* If file poll returns POLLERR on the device file descriptor,
-                   then the filesystem is already unmounted */
-                if (res == 1 && (pfd.revents & POLLERR))
-                        return;
-
-                /* Need to close file descriptor, otherwise synchronous umount
-                   would recurse into filesystem, and deadlock */
-                close (fd);
-        }
-
-        if (geteuid () == 0) {
-                fuse_mnt_umount ("fuse", mountpoint, mountpoint, 1);
-                return;
-        }
-
-        res = umount2 (mountpoint, 2);
-        if (res == 0)
-                return;
-
-        pid = fork ();
-        if (pid == -1)
-                return;
-
-        if (pid == 0) {
-                const char *argv[] = { FUSERMOUNT_PROG, "-u", "-q", "-z",
-                                       "--", mountpoint, NULL };
-
-                execvp (FUSERMOUNT_PROG, (char **)argv);
-                _exit (1);
-        }
-        waitpid (pid, NULL, 0);
+        ret = waitpid (pid, &res, 0);
+        ret = (ret == pid && res == 0) ? 0 : -1;
+ out:
+        FREE (fm_mnt_params);
+        return ret;
 }
-#endif
-
-/*
- * Functions below are loosely modelled after similar functions of libfuse
- */
 
-#ifndef FUSE_UTIL
 static int
-fuse_mount_sys (const char *mountpoint, char *fsname, char *mnt_param, pid_t *mtab_pid)
+fuse_mount_sys (const char *mountpoint, char *fsname,
+                unsigned long mountflags, char *mnt_param, int fd)
 {
-        int fd = -1, ret = -1;
+        int ret = -1;
         unsigned mounted = 0;
         char *mnt_param_mnt = NULL;
         char *fstype = "fuse.glusterfs";
         char *source = fsname;
 
-        fd = open ("/dev/fuse", O_RDWR);
-        if (fd == -1) {
-                GFFUSE_LOGERR ("cannot open /dev/fuse (%s)", strerror (errno));
-
-                return -1;
-        }
-
         ret = asprintf (&mnt_param_mnt,
                         "%s,fd=%i,rootmode=%o,user_id=%i,group_id=%i",
                         mnt_param, fd, S_IFDIR, getuid (), getgid ());
@@ -567,7 +188,7 @@ fuse_mount_sys (const char *mountpoint, char *fsname, char *mnt_param, pid_t *mt
 
                 goto out;
         }
-        ret = mount (source, mountpoint, fstype, 0,
+        ret = mount (source, mountpoint, fstype, mountflags,
                      mnt_param_mnt);
         if (ret == -1 && errno == ENODEV) {
                 /* fs subtype support was added by 79c0b2df aka
@@ -580,7 +201,7 @@ fuse_mount_sys (const char *mountpoint, char *fsname, char *mnt_param, pid_t *mt
 
                         goto out;
                 }
-                ret = mount (source, mountpoint, fstype, 0,
+                ret = mount (source, mountpoint, fstype, mountflags,
                              mnt_param_mnt);
         }
         if (ret == -1)
@@ -588,8 +209,10 @@ fuse_mount_sys (const char *mountpoint, char *fsname, char *mnt_param, pid_t *mt
         else
                 mounted = 1;
 
+#ifndef __NetBSD__
         if (geteuid () == 0) {
                 char *newmnt = fuse_mnt_resolve_path ("fuse", mountpoint);
+                char *mnt_param_mtab = NULL;
 
                 if (!newmnt) {
                         ret = -1;
@@ -597,8 +220,17 @@ fuse_mount_sys (const char *mountpoint, char *fsname, char *mnt_param, pid_t *mt
                         goto out;
                 }
 
-                ret = fuse_mnt_add_mount ("fuse", source, newmnt, fstype,
-                                          mnt_param, mtab_pid);
+                ret = asprintf (&mnt_param_mtab, "%s%s",
+                                mountflags & MS_RDONLY ? "ro," : "",
+                                mnt_param);
+                if (ret == -1)
+                        GFFUSE_LOGERR ("Out of memory");
+                else {
+                        ret = fuse_mnt_add_mount ("fuse", source, newmnt,
+                                                  fstype, mnt_param_mtab);
+                        FREE (mnt_param_mtab);
+                }
+
                 FREE (newmnt);
                 if (ret == -1) {
                         GFFUSE_LOGERR ("failed to add mtab entry");
@@ -606,95 +238,76 @@ fuse_mount_sys (const char *mountpoint, char *fsname, char *mnt_param, pid_t *mt
                         goto out;
                 }
         }
+#endif /* __NetBSD__ */
 
- out:
+out:
         if (ret == -1) {
                 if (mounted)
                         umount2 (mountpoint, 2); /* lazy umount */
-                close (fd);
-                fd = -1;
         }
         FREE (mnt_param_mnt);
         if (source != fsname)
                 FREE (source);
-        return fd;
-}
-
-static char *
-escape (char *s)
-{
-        size_t len = 0;
-        char *p = NULL;
-        char *q = NULL;
-        char *e = NULL;
-
-        for (p = s; *p; p++) {
-                if (*p == ',')
-                       len++;
-                len++;
-        }
-
-        e = CALLOC (1, len + 1);
-        if (!e)
-                return NULL;
-
-        for (p = s, q = e; *p; p++, q++) {
-                if (*p == ',') {
-                        *q = '\\';
-                        q++;
-                }
-                *q = *p;
-        }
 
-        return e;
+        return ret;
 }
 
 int
-gf_fuse_mount (const char *mountpoint, char *fsname, char *mnt_param,
-               pid_t *mtab_pid)
+gf_fuse_mount (const char *mountpoint, char *fsname,
+               unsigned long mountflags, char *mnt_param,
+               pid_t *mnt_pid, int status_fd)
 {
-        int fd = -1, rv = -1;
-        char *fm_mnt_params = NULL, *p = NULL;
-        char *efsname = NULL;
+        int   fd  = -1;
+        pid_t pid = -1;
+        int   ret = -1;
 
-        fd = fuse_mount_sys (mountpoint, fsname, mnt_param, mtab_pid);
+        fd = open ("/dev/fuse", O_RDWR);
         if (fd == -1) {
-                gf_log ("glusterfs-fuse", GF_LOG_INFO,
-                        "direct mount failed (%s), "
-                        "retry to mount via fusermount",
-                        strerror (errno));
-
-                efsname = escape (fsname);
-                if (!efsname) {
-                        GFFUSE_LOGERR ("Out of memory");
-
-                        return -1;
-                }
-                rv = asprintf (&fm_mnt_params,
-                               "%s,fsname=%s,nonempty,subtype=glusterfs",
-                               mnt_param, efsname);
-                FREE (efsname);
-                if (rv == -1) {
-                        GFFUSE_LOGERR ("Out of memory");
+                GFFUSE_LOGERR ("cannot open /dev/fuse (%s)",
+                                strerror (errno));
+                return -1;
+        }
 
-                        return -1;
+        /* start mount agent */
+        pid = fork();
+        switch (pid) {
+        case 0:
+                /* hello it's mount agent */
+                if (!mnt_pid) {
+                        /* daemonize mount agent, caller is
+                         * not interested in waiting for it
+                         */
+                        pid = fork ();
+                        if (pid)
+                                exit (pid == -1 ? 1 : 0);
                 }
 
-                fd = fuse_mount_fusermount (mountpoint, fm_mnt_params);
-                if (fd == -1) {
-                        p = fm_mnt_params + strlen (fm_mnt_params);
-                        while (*--p != ',');
-                        *p = '\0';
+                ret = fuse_mount_sys (mountpoint, fsname, mountflags, mnt_param, fd);
+                if (ret == -1) {
+                        gf_log ("glusterfs-fuse", GF_LOG_INFO,
+                                "direct mount failed (%s), "
+                                "retry to mount via fusermount",
+                                strerror (errno));
 
-                        fd = fuse_mount_fusermount (mountpoint, fm_mnt_params);
+                        ret = fuse_mount_fusermount (mountpoint, fsname, mountflags,
+                                                     mnt_param, fd);
                 }
 
-                FREE (fm_mnt_params);
+                if (ret == -1)
+                        GFFUSE_LOGERR ("mount of %s to %s (%s) failed",
+                                       fsname, mountpoint, mnt_param);
 
-                if (fd == -1)
-                       GFFUSE_LOGERR ("mount failed");
+                if (status_fd >= 0)
+                        (void)write (status_fd, &ret, sizeof (ret));
+                exit (!!ret);
+                /* bye mount agent */
+        case -1:
+                close (fd);
+                fd = -1;
         }
 
+        if (mnt_pid)
+               *mnt_pid = pid;
+
         return fd;
 }
-#endif
diff --git a/contrib/fuse-util/Makefile.am b/contrib/fuse-util/Makefile.am
index a72e3832b..971d3d220 100644
--- a/contrib/fuse-util/Makefile.am
+++ b/contrib/fuse-util/Makefile.am
@@ -1,9 +1,11 @@
 bin_PROGRAMS = fusermount-glusterfs
 
-fusermount_glusterfs_SOURCES = fusermount.c $(CONTRIBDIR)/fuse-lib/mount.c
-noinst_HEADERS = mount_util.h
+fusermount_glusterfs_SOURCES = fusermount.c mount_util.c $(CONTRIBDIR)/fuse-lib/mount-common.c
+noinst_HEADERS = $(CONTRIBDIR)/fuse-include/mount_util.h
 
-AM_CFLAGS = -Wall -D_FILE_OFFSET_BITS=64 -DFUSE_UTIL $(GF_CFLAGS) -D_GNU_SOURCE
+AM_CPPFLAGS = $(GF_CPPFLAGS) -DFUSE_UTIL -I$(CONTRIBDIR)/fuse-include
+
+AM_CFLAGS = -Wall $(GF_CFLAGS)
 
 install-exec-hook:
 	-chown root $(DESTDIR)$(bindir)/fusermount-glusterfs
diff --git a/contrib/fuse-util/fusermount.c b/contrib/fuse-util/fusermount.c
index 39da9b6a0..0ff8d9039 100644
--- a/contrib/fuse-util/fusermount.c
+++ b/contrib/fuse-util/fusermount.c
@@ -19,6 +19,7 @@
 #include <errno.h>
 #include <fcntl.h>
 #include <pwd.h>
+#include <limits.h>
 #include <mntent.h>
 #include <sys/wait.h>
 #include <sys/stat.h>
@@ -28,6 +29,7 @@
 #include <sys/utsname.h>
 #include <sched.h>
 
+#define FUSE_DEVFD_ENV		"_FUSE_DEVFD"
 #define FUSE_COMMFD_ENV		"_FUSE_COMMFD"
 
 #define FUSE_DEV_OLD "/proc/fs/fuse/dev"
@@ -114,8 +116,16 @@ static int lock_umount(void)
 
 static void unlock_umount(int mtablock)
 {
-	lockf(mtablock, F_ULOCK, 0);
-	close(mtablock);
+	if (mtablock >= 0) {
+		int res;
+
+		res = lockf(mtablock, F_ULOCK, 0);
+		if (res < 0) {
+			fprintf(stderr, "%s: error releasing lock: %s\n",
+				progname, strerror(errno));
+		}
+		close(mtablock);
+	}
 }
 
 static int add_mount(const char *source, const char *mnt, const char *type,
@@ -238,7 +248,7 @@ static int check_is_mount_child(void *p)
 	}
 
 	count = 0;
-	while ((entp = getmntent(fp)) != NULL)
+	while (getmntent(fp) != NULL)
 		count++;
 	endmntent(fp);
 
@@ -325,7 +335,7 @@ static int check_is_mount(const char *last, const char *mnt)
 	return 0;
 }
 
-static int chdir_to_parent(char *copy, const char **lastp, int *currdir_fd)
+static int chdir_to_parent(char *copy, const char **lastp)
 {
 	char *tmp;
 	const char *parent;
@@ -350,14 +360,6 @@ static int chdir_to_parent(char *copy, const char **lastp, int *currdir_fd)
 		parent = "/";
 	}
 
-	*currdir_fd = open(".", O_RDONLY);
-	if (*currdir_fd == -1) {
-		fprintf(stderr,
-			"%s: failed to open current directory: %s\n",
-			progname, strerror(errno));
-		return -1;
-	}
-
 	res = chdir(parent);
 	if (res == -1) {
 		fprintf(stderr, "%s: failed to chdir to %s: %s\n",
@@ -382,7 +384,6 @@ static int chdir_to_parent(char *copy, const char **lastp, int *currdir_fd)
 
 static int unmount_fuse_locked(const char *mnt, int quiet, int lazy)
 {
-	int currdir_fd = -1;
 	char *copy;
 	const char *last;
 	int res;
@@ -399,7 +400,7 @@ static int unmount_fuse_locked(const char *mnt, int quiet, int lazy)
 		return -1;
 	}
 
-	res = chdir_to_parent(copy, &last, &currdir_fd);
+	res = chdir_to_parent(copy, &last);
 	if (res == -1)
 		goto out;
 
@@ -411,10 +412,6 @@ static int unmount_fuse_locked(const char *mnt, int quiet, int lazy)
 
 out:
 	free(copy);
-	if (currdir_fd != -1) {
-		fchdir(currdir_fd);
-		close(currdir_fd);
-	}
 
 	return res;
 }
@@ -508,13 +505,13 @@ static void read_conf(void)
 		int isnewline = 1;
 		while (fgets(line, sizeof(line), fp) != NULL) {
 			if (isnewline) {
-				if (line[strlen(line)-1] == '\n') {
+				if (strlen(line) && line[strlen(line)-1] == '\n') {
 					strip_line(line);
 					parse_line(line, linenum);
 				} else {
 					isnewline = 0;
 				}
-			} else if(line[strlen(line)-1] == '\n') {
+			} else if(strlen(line) && line[strlen(line)-1] == '\n') {
 				fprintf(stderr, "%s: reading %s: line %i too long\n", progname, FUSE_CONF, linenum);
 
 				isnewline = 1;
@@ -609,7 +606,7 @@ static int add_option(char **optsp, const char *opt, unsigned expand)
 static int get_mnt_opts(int flags, char *opts, char **mnt_optsp)
 {
 	int i;
-	int l;
+	size_t l;
 
 	if (!(flags & MS_RDONLY) && add_option(mnt_optsp, "rw", 0) == -1)
 		return -1;
@@ -624,7 +621,7 @@ static int get_mnt_opts(int flags, char *opts, char **mnt_optsp)
 		return -1;
 	/* remove comma from end of opts*/
 	l = strlen(*mnt_optsp);
-	if ((*mnt_optsp)[l-1] == ',')
+	if (l && (*mnt_optsp)[l-1] == ',')
 		(*mnt_optsp)[l-1] = '\0';
 	if (getuid() != 0) {
 		const char *user = get_user_name();
@@ -653,8 +650,7 @@ static int get_string_opt(const char *s, unsigned len, const char *opt,
 	unsigned opt_len = strlen(opt);
 	char *d;
 
-	if (*val)
-		free(*val);
+	free(*val);
 	*val = (char *) malloc(len - opt_len + 1);
 	if (!*val) {
 		fprintf(stderr, "%s: failed to allocate memory\n", progname);
@@ -823,15 +819,14 @@ static int do_mount(const char *mnt, char **typep, mode_t rootmode,
 			fprintf(stderr, "%s: mount failed: %s\n", progname,
 				strerror(errno_save));
 		goto err;
-	} else {
-		*sourcep = source;
-		*typep = type;
-		*mnt_optsp = mnt_opts;
 	}
+	*sourcep = source;
+	*typep = type;
+	*mnt_optsp = mnt_opts;
 	free(fsname);
 	free(optbuf);
 
-	return res;
+	return 0;
 
 err:
 	free(fsname);
@@ -874,8 +869,7 @@ static int check_version(const char *dev)
 	return 0;
 }
 
-static int check_perm(const char **mntp, struct stat *stbuf, int *currdir_fd,
-		      int *mountpoint_fd)
+static int check_perm(const char **mntp, struct stat *stbuf, int *mountpoint_fd)
 {
 	int res;
 	const char *mnt = *mntp;
@@ -893,13 +887,6 @@ static int check_perm(const char **mntp, struct stat *stbuf, int *currdir_fd,
 		return 0;
 
 	if (S_ISDIR(stbuf->st_mode)) {
-		*currdir_fd = open(".", O_RDONLY);
-		if (*currdir_fd == -1) {
-			fprintf(stderr,
-				"%s: failed to open current directory: %s\n",
-				progname, strerror(errno));
-			return -1;
-		}
 		res = chdir(mnt);
 		if (res == -1) {
 			fprintf(stderr,
@@ -1016,8 +1003,36 @@ static int open_fuse_device(char **devp)
 	return -1;
 }
 
+static int check_fuse_device(char *devfd, char **devp)
+{
+	int res;
+	char *devlink;
 
-static int mount_fuse(const char *mnt, const char *opts)
+	res = asprintf(&devlink, "/proc/self/fd/%s", devfd);
+	if (res == -1) {
+		fprintf(stderr, "%s: failed to allocate memory\n", progname);
+		return -1;
+	}
+
+	*devp = (char *) calloc(1, PATH_MAX + 1);
+	if (!*devp) {
+		fprintf(stderr, "%s: failed to allocate memory\n", progname);
+		free(devlink);
+		return -1;
+	}
+
+	res = readlink (devlink, *devp, PATH_MAX);
+	free (devlink);
+	if (res == -1) {
+		fprintf(stderr, "%s: specified fuse fd is invalid\n",
+			progname);
+		return -1;
+	}
+
+	return atoi(devfd);
+}
+
+static int mount_fuse(const char *mnt, const char *opts, char *devfd)
 {
 	int res;
 	int fd;
@@ -1027,10 +1042,9 @@ static int mount_fuse(const char *mnt, const char *opts)
 	char *source = NULL;
 	char *mnt_opts = NULL;
 	const char *real_mnt = mnt;
-	int currdir_fd = -1;
 	int mountpoint_fd = -1;
 
-	fd = open_fuse_device(&dev);
+	fd = devfd ? check_fuse_device(devfd, &dev) : open_fuse_device(&dev);
 	if (fd == -1)
 		return -1;
 
@@ -1041,15 +1055,13 @@ static int mount_fuse(const char *mnt, const char *opts)
 		int mount_count = count_fuse_fs();
 		if (mount_count >= mount_max) {
 			fprintf(stderr, "%s: too many FUSE filesystems mounted; mount_max=N can be set in /etc/fuse.conf\n", progname);
-			close(fd);
-			return -1;
+			goto fail_close_fd;
 		}
 	}
 
 	res = check_version(dev);
 	if (res != -1) {
-		res = check_perm(&real_mnt, &stbuf, &currdir_fd,
-				 &mountpoint_fd);
+		res = check_perm(&real_mnt, &stbuf, &mountpoint_fd);
 		restore_privs();
 		if (res != -1)
 			res = do_mount(real_mnt, &type, stbuf.st_mode & S_IFMT,
@@ -1058,33 +1070,38 @@ static int mount_fuse(const char *mnt, const char *opts)
 	} else
 		restore_privs();
 
-	if (currdir_fd != -1) {
-		fchdir(currdir_fd);
-		close(currdir_fd);
-	}
 	if (mountpoint_fd != -1)
 		close(mountpoint_fd);
 
+	if (res == -1)
+		goto fail_close_fd;
+
+	res = chdir("/");
 	if (res == -1) {
-		close(fd);
-		return -1;
+		fprintf(stderr, "%s: failed to chdir to '/'\n", progname);
+		goto fail_close_fd;
 	}
 
 	if (geteuid() == 0) {
 		res = add_mount(source, mnt, type, mnt_opts);
 		if (res == -1) {
-			umount2(mnt, 2); /* lazy umount */
-			close(fd);
-			return -1;
+			/* Can't clean up mount in a non-racy way */
+			goto fail_close_fd;
 		}
 	}
 
+out_free:
 	free(source);
 	free(type);
 	free(mnt_opts);
 	free(dev);
 
 	return fd;
+
+fail_close_fd:
+	close(fd);
+	fd = -1;
+	goto out_free;
 }
 
 static int send_fd(int sock_fd, int fd)
@@ -1154,6 +1171,7 @@ int main(int argc, char *argv[])
 	static int unmount = 0;
 	static int lazy = 0;
 	static int quiet = 0;
+	char *devfd;
 	char *commfd;
 	int cfd;
 	const char *opts = "";
@@ -1222,6 +1240,13 @@ int main(int argc, char *argv[])
 
 	drop_privs();
 	mnt = fuse_mnt_resolve_path(progname, origmnt);
+	if (mnt != NULL) {
+		res = chdir("/");
+		if (res == -1) {
+			fprintf(stderr, "%s: failed to chdir to '/'\n", progname);
+			exit(1);
+		}
+	}
 	restore_privs();
 	if (mnt == NULL)
 		exit(1);
@@ -1242,21 +1267,26 @@ int main(int argc, char *argv[])
 		return 0;
 	}
 
-	commfd = getenv(FUSE_COMMFD_ENV);
-	if (commfd == NULL) {
-		fprintf(stderr, "%s: old style mounting not supported\n",
-			progname);
-		exit(1);
+	devfd = getenv(FUSE_DEVFD_ENV);
+	if (devfd == NULL) {
+		commfd = getenv(FUSE_COMMFD_ENV);
+		if (commfd == NULL) {
+			fprintf(stderr, "%s: old style mounting not supported\n",
+				progname);
+			exit(1);
+		}
 	}
 
-	fd = mount_fuse(mnt, opts);
+	fd = mount_fuse(mnt, opts, devfd);
 	if (fd == -1)
 		exit(1);
 
-	cfd = atoi(commfd);
-	res = send_fd(cfd, fd);
-	if (res == -1)
-		exit(1);
+	if (devfd == NULL) {
+		cfd = atoi(commfd);
+		res = send_fd(cfd, fd);
+		if (res == -1)
+			exit(1);
+	}
 
 	return 0;
 }
diff --git a/contrib/fuse-util/mount_util.c b/contrib/fuse-util/mount_util.c
new file mode 100644
index 000000000..911b84445
--- /dev/null
+++ b/contrib/fuse-util/mount_util.c
@@ -0,0 +1,64 @@
+/*
+  FUSE: Filesystem in Userspace
+  Copyright (C) 2001-2007  Miklos Szeredi <miklos@szeredi.hu>
+
+  This program can be distributed under the terms of the GNU LGPLv2.
+  See the file COPYING.LIB.
+*/
+
+#include <dirent.h>
+#include <stdio.h>
+#include <string.h>
+#include <errno.h>
+#include <sys/types.h>
+#include <sys/stat.h>
+
+int fuse_mnt_check_empty(const char *progname, const char *mnt,
+			 mode_t rootmode, off_t rootsize)
+{
+	int isempty = 1;
+
+	if (S_ISDIR(rootmode)) {
+		struct dirent *ent;
+		DIR *dp = opendir(mnt);
+		if (dp == NULL) {
+			fprintf(stderr,
+				"%s: failed to open mountpoint for reading: %s\n",
+				progname, strerror(errno));
+			return -1;
+		}
+		while ((ent = readdir(dp)) != NULL) {
+			if (strcmp(ent->d_name, ".") != 0 &&
+			    strcmp(ent->d_name, "..") != 0) {
+				isempty = 0;
+				break;
+			}
+		}
+		closedir(dp);
+	} else if (rootsize)
+		isempty = 0;
+
+	if (!isempty) {
+		fprintf(stderr, "%s: mountpoint is not empty\n", progname);
+		fprintf(stderr, "%s: if you are sure this is safe, use the 'nonempty' mount option\n", progname);
+		return -1;
+	}
+	return 0;
+}
+
+int fuse_mnt_check_fuseblk(void)
+{
+	char buf[256];
+	FILE *f = fopen("/proc/filesystems", "r");
+	if (!f)
+		return 1;
+
+	while (fgets(buf, sizeof(buf), f))
+		if (strstr(buf, "fuseblk\n")) {
+			fclose(f);
+			return 1;
+		}
+
+	fclose(f);
+	return 0;
+}
diff --git a/contrib/libgen/basename_r.c b/contrib/libgen/basename_r.c
new file mode 100644
index 000000000..2c3a87afe
--- /dev/null
+++ b/contrib/libgen/basename_r.c
@@ -0,0 +1,40 @@
+/*
+ * borrowed from glibc-2.12.1/string/basename.c
+ * Modified to return "." for NULL or "", as required for SUSv2.
+ */
+#include <string.h>
+#include <stdlib.h>
+#ifdef THREAD_UNSAFE_BASENAME
+
+/* Return the name-within-directory of a file name.
+   Copyright (C) 1996,97,98,2002 Free Software Foundation, Inc.
+   This file is part of the GNU C Library.
+
+   The GNU C Library is free software; you can redistribute it and/or
+   modify it under the terms of the GNU Lesser General Public
+   License as published by the Free Software Foundation; either
+   version 2.1 of the License, or (at your option) any later version.
+
+   The GNU C Library is distributed in the hope that it will be useful,
+   but WITHOUT ANY WARRANTY; without even the implied warranty of
+   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+   Lesser General Public License for more details.
+
+   You should have received a copy of the GNU Lesser General Public
+   License along with the GNU C Library; if not, write to the Free
+   Software Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA
+   02111-1307 USA.  */
+
+char *
+basename_r (filename)
+     const char *filename;
+{
+  char *p;
+
+  if ((filename == NULL) || (*filename == '\0'))
+    return ".";
+
+  p = strrchr (filename, '/');
+  return p ? p + 1 : (char *) filename;
+}
+#endif /* THREAD_UNSAFE_BASENAME */
diff --git a/contrib/libgen/dirname_r.c b/contrib/libgen/dirname_r.c
new file mode 100644
index 000000000..131cbcf2a
--- /dev/null
+++ b/contrib/libgen/dirname_r.c
@@ -0,0 +1,243 @@
+/*
+ * Borrowed from glibc-2.12.1/string/memrchr.c
+ * Based on strlen implementation by Torbjorn Granlund (tege@sics.se),
+ * Removed code for long bigger than 32 bytes, renamed __ptr_t as void *
+ * changed reg_char type to char.
+ */
+#include <string.h>
+#include <stdlib.h>
+#ifdef THREAD_UNSAFE_DIRNAME
+
+/* memrchr -- find the last occurrence of a byte in a memory block
+   Copyright (C) 1991, 93, 96, 97, 99, 2000 Free Software Foundation, Inc.
+   This file is part of the GNU C Library.
+   Based on strlen implementation by Torbjorn Granlund (tege@sics.se),
+   with help from Dan Sahlin (dan@sics.se) and
+   commentary by Jim Blandy (jimb@ai.mit.edu);
+   adaptation to memchr suggested by Dick Karpinski (dick@cca.ucsf.edu),
+   and implemented by Roland McGrath (roland@ai.mit.edu).
+
+   The GNU C Library is free software; you can redistribute it and/or
+   modify it under the terms of the GNU Lesser General Public
+   License as published by the Free Software Foundation; either
+   version 2.1 of the License, or (at your option) any later version.
+
+   The GNU C Library is distributed in the hope that it will be useful,
+   but WITHOUT ANY WARRANTY; without even the implied warranty of
+   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+   Lesser General Public License for more details.
+
+   You should have received a copy of the GNU Lesser General Public
+   License along with the GNU C Library; if not, write to the Free
+   Software Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA
+   02111-1307 USA.  */
+
+void *
+__memrchr (s, c_in, n)
+     const void * s;
+     int c_in;
+     size_t n;
+{
+  const unsigned char *char_ptr;
+  const unsigned long int *longword_ptr;
+  unsigned long int longword, magic_bits, charmask;
+  unsigned char c;
+
+  c = (unsigned char) c_in;
+
+  /* Handle the last few characters by reading one character at a time.
+     Do this until CHAR_PTR is aligned on a longword boundary.  */
+  for (char_ptr = (const unsigned char *) s + n;
+       n > 0 && ((unsigned long int) char_ptr
+		 & (sizeof (longword) - 1)) != 0;
+       --n)
+    if (*--char_ptr == c)
+      return (void *) char_ptr;
+
+  /* All these elucidatory comments refer to 4-byte longwords,
+     but the theory applies equally well to 8-byte longwords.  */
+
+  longword_ptr = (const unsigned long int *) char_ptr;
+
+  /* Bits 31, 24, 16, and 8 of this number are zero.  Call these bits
+     the "holes."  Note that there is a hole just to the left of
+     each byte, with an extra at the end:
+
+     bits:  01111110 11111110 11111110 11111111
+     bytes: AAAAAAAA BBBBBBBB CCCCCCCC DDDDDDDD
+
+     The 1-bits make sure that carries propagate to the next 0-bit.
+     The 0-bits provide holes for carries to fall into.  */
+
+  if (sizeof (longword) != 4 && sizeof (longword) != 8)
+    abort ();
+
+  magic_bits = 0x7efefeff;
+
+  /* Set up a longword, each of whose bytes is C.  */
+  charmask = c | (c << 8);
+  charmask |= charmask << 16;
+
+  /* Instead of the traditional loop which tests each character,
+     we will test a longword at a time.  The tricky part is testing
+     if *any of the four* bytes in the longword in question are zero.  */
+  while (n >= sizeof (longword))
+    {
+      /* We tentatively exit the loop if adding MAGIC_BITS to
+	 LONGWORD fails to change any of the hole bits of LONGWORD.
+
+	 1) Is this safe?  Will it catch all the zero bytes?
+	 Suppose there is a byte with all zeros.  Any carry bits
+	 propagating from its left will fall into the hole at its
+	 least significant bit and stop.  Since there will be no
+	 carry from its most significant bit, the LSB of the
+	 byte to the left will be unchanged, and the zero will be
+	 detected.
+
+	 2) Is this worthwhile?  Will it ignore everything except
+	 zero bytes?  Suppose every byte of LONGWORD has a bit set
+	 somewhere.  There will be a carry into bit 8.  If bit 8
+	 is set, this will carry into bit 16.  If bit 8 is clear,
+	 one of bits 9-15 must be set, so there will be a carry
+	 into bit 16.  Similarly, there will be a carry into bit
+	 24.  If one of bits 24-30 is set, there will be a carry
+	 into bit 31, so all of the hole bits will be changed.
+
+	 The one misfire occurs when bits 24-30 are clear and bit
+	 31 is set; in this case, the hole at bit 31 is not
+	 changed.  If we had access to the processor carry flag,
+	 we could close this loophole by putting the fourth hole
+	 at bit 32!
+
+	 So it ignores everything except 128's, when they're aligned
+	 properly.
+
+	 3) But wait!  Aren't we looking for C, not zero?
+	 Good point.  So what we do is XOR LONGWORD with a longword,
+	 each of whose bytes is C.  This turns each byte that is C
+	 into a zero.  */
+
+      longword = *--longword_ptr ^ charmask;
+
+      /* Add MAGIC_BITS to LONGWORD.  */
+      if ((((longword + magic_bits)
+
+	    /* Set those bits that were unchanged by the addition.  */
+	    ^ ~longword)
+
+	   /* Look at only the hole bits.  If any of the hole bits
+	      are unchanged, most likely one of the bytes was a
+	      zero.  */
+	   & ~magic_bits) != 0)
+	{
+	  /* Which of the bytes was C?  If none of them were, it was
+	     a misfire; continue the search.  */
+
+	  const unsigned char *cp = (const unsigned char *) longword_ptr;
+
+	  if (cp[3] == c)
+	    return (void *) &cp[3];
+	  if (cp[2] == c)
+	    return (void *) &cp[2];
+	  if (cp[1] == c)
+	    return (void *) &cp[1];
+	  if (cp[0] == c)
+	    return (void *) cp;
+	}
+
+      n -= sizeof (longword);
+    }
+
+  char_ptr = (const unsigned char *) longword_ptr;
+
+  while (n-- > 0)
+    {
+      if (*--char_ptr == c)
+	return (void *) char_ptr;
+    }
+
+  return 0;
+}
+
+/*
+ * Borrowed from glibc-2.12.1/misc/dirname.c
+ */
+
+/* dirname - return directory part of PATH.
+   Copyright (C) 1996, 2000, 2001, 2002 Free Software Foundation, Inc.
+   This file is part of the GNU C Library.
+   Contributed by Ulrich Drepper <drepper@cygnus.com>, 1996.
+
+   The GNU C Library is free software; you can redistribute it and/or
+   modify it under the terms of the GNU Lesser General Public
+   License as published by the Free Software Foundation; either
+   version 2.1 of the License, or (at your option) any later version.
+
+   The GNU C Library is distributed in the hope that it will be useful,
+   but WITHOUT ANY WARRANTY; without even the implied warranty of
+   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+   Lesser General Public License for more details.
+
+   You should have received a copy of the GNU Lesser General Public
+   License along with the GNU C Library; if not, write to the Free
+   Software Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA
+   02111-1307 USA.  */
+
+char *
+dirname_r (char *path)
+{
+  static const char dot[] = ".";
+  char *last_slash;
+
+  /* Find last '/'.  */
+  last_slash = path != NULL ? strrchr (path, '/') : NULL;
+
+  if (last_slash != NULL && last_slash != path && last_slash[1] == '\0')
+    {
+      /* Determine whether all remaining characters are slashes.  */
+      char *runp;
+
+      for (runp = last_slash; runp != path; --runp)
+        if (runp[-1] != '/')
+          break;
+
+      /* The '/' is the last character, we have to look further.  */
+      if (runp != path)
+        last_slash = __memrchr (path, '/', runp - path);
+    }
+
+  if (last_slash != NULL)
+    {
+      /* Determine whether all remaining characters are slashes.  */
+      char *runp;
+
+      for (runp = last_slash; runp != path; --runp)
+        if (runp[-1] != '/')
+          break;
+
+      /* Terminate the path.  */
+      if (runp == path)
+        {
+          /* The last slash is the first character in the string.  We have to
+             return "/".  As a special case we have to return "//" if there
+             are exactly two slashes at the beginning of the string.  See
+             XBD 4.10 Path Name Resolution for more information.  */
+          if (last_slash == path + 1)
+            ++last_slash;
+          else
+            last_slash = path + 1;
+        }
+      else
+        last_slash = runp;
+
+      last_slash[0] = '\0';
+    }
+  else
+    /* This assignment is ill-designed but the XPG specs require to
+       return a string containing "." in any case no directory part is
+       found and so a static and constant string is required.  */
+    path = (char *) dot;
+
+  return path;
+}
+#endif /* THREAD_UNSAFE_DIRNAME */
diff --git a/contrib/md5/md5.c b/contrib/md5/md5.c
deleted file mode 100644
index 5f0d0d157..000000000
--- a/contrib/md5/md5.c
+++ /dev/null
@@ -1,310 +0,0 @@
-/*
- * RFC 1321 compliant MD5 implementation
- *
- * Copyright (C) 2001-2003 Christophe Devine
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License as published by
- * the Free Software Foundation; either version 3 of the License, or
- * (at your option) any later version.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
- * GNU General Public License for more details.
- *
- * You should have received a copy of the GNU General Public License along
- * with this program; if not, visit the http://fsf.org website.
- */
-
-
-#include <inttypes.h>
-#include <string.h>
-
-#include "md5.h"
-
-void md5_begin(md_context *ctx)
-{
-	ctx->A = 0x67452301;
-	ctx->B = 0xEFCDAB89;
-	ctx->C = 0x98BADCFE;
-	ctx->D = 0x10325476;
-
-	ctx->totalN = ctx->totalN2 = 0;
-}
-
-static void md5_process(md_context *ctx, const uint8_t data[CSUM_CHUNK])
-{
-	uint32_t X[16], A, B, C, D;
-
-	A = ctx->A;
-	B = ctx->B;
-	C = ctx->C;
-	D = ctx->D;
-
-	X[0] = IVAL(data, 0);
-	X[1] = IVAL(data, 4);
-	X[2] = IVAL(data, 8);
-	X[3] = IVAL(data, 12);
-	X[4] = IVAL(data, 16);
-	X[5] = IVAL(data, 20);
-	X[6] = IVAL(data, 24);
-	X[7] = IVAL(data, 28);
-	X[8] = IVAL(data, 32);
-	X[9] = IVAL(data, 36);
-	X[10] = IVAL(data, 40);
-	X[11] = IVAL(data, 44);
-	X[12] = IVAL(data, 48);
-	X[13] = IVAL(data, 52);
-	X[14] = IVAL(data, 56);
-	X[15] = IVAL(data, 60);
-
-#define S(x,n) ((x << n) | ((x & 0xFFFFFFFF) >> (32 - n)))
-
-#define P(a,b,c,d,k,s,t) a += F(b,c,d) + X[k] + t, a = S(a,s) + b
-
-#define F(x,y,z) (z ^ (x & (y ^ z)))
-
-	P(A, B, C, D,  0,  7, 0xD76AA478);
-	P(D, A, B, C,  1, 12, 0xE8C7B756);
-	P(C, D, A, B,  2, 17, 0x242070DB);
-	P(B, C, D, A,  3, 22, 0xC1BDCEEE);
-	P(A, B, C, D,  4,  7, 0xF57C0FAF);
-	P(D, A, B, C,  5, 12, 0x4787C62A);
-	P(C, D, A, B,  6, 17, 0xA8304613);
-	P(B, C, D, A,  7, 22, 0xFD469501);
-	P(A, B, C, D,  8,  7, 0x698098D8);
-	P(D, A, B, C,  9, 12, 0x8B44F7AF);
-	P(C, D, A, B, 10, 17, 0xFFFF5BB1);
-	P(B, C, D, A, 11, 22, 0x895CD7BE);
-	P(A, B, C, D, 12,  7, 0x6B901122);
-	P(D, A, B, C, 13, 12, 0xFD987193);
-	P(C, D, A, B, 14, 17, 0xA679438E);
-	P(B, C, D, A, 15, 22, 0x49B40821);
-
-#undef F
-#define F(x,y,z) (y ^ (z & (x ^ y)))
-
-	P(A, B, C, D,  1,  5, 0xF61E2562);
-	P(D, A, B, C,  6,  9, 0xC040B340);
-	P(C, D, A, B, 11, 14, 0x265E5A51);
-	P(B, C, D, A,  0, 20, 0xE9B6C7AA);
-	P(A, B, C, D,  5,  5, 0xD62F105D);
-	P(D, A, B, C, 10,  9, 0x02441453);
-	P(C, D, A, B, 15, 14, 0xD8A1E681);
-	P(B, C, D, A,  4, 20, 0xE7D3FBC8);
-	P(A, B, C, D,  9,  5, 0x21E1CDE6);
-	P(D, A, B, C, 14,  9, 0xC33707D6);
-	P(C, D, A, B,  3, 14, 0xF4D50D87);
-	P(B, C, D, A,  8, 20, 0x455A14ED);
-	P(A, B, C, D, 13,  5, 0xA9E3E905);
-	P(D, A, B, C,  2,  9, 0xFCEFA3F8);
-	P(C, D, A, B,  7, 14, 0x676F02D9);
-	P(B, C, D, A, 12, 20, 0x8D2A4C8A);
-
-#undef F
-#define F(x,y,z) (x ^ y ^ z)
-
-	P(A, B, C, D,  5,  4, 0xFFFA3942);
-	P(D, A, B, C,  8, 11, 0x8771F681);
-	P(C, D, A, B, 11, 16, 0x6D9D6122);
-	P(B, C, D, A, 14, 23, 0xFDE5380C);
-	P(A, B, C, D,  1,  4, 0xA4BEEA44);
-	P(D, A, B, C,  4, 11, 0x4BDECFA9);
-	P(C, D, A, B,  7, 16, 0xF6BB4B60);
-	P(B, C, D, A, 10, 23, 0xBEBFBC70);
-	P(A, B, C, D, 13,  4, 0x289B7EC6);
-	P(D, A, B, C,  0, 11, 0xEAA127FA);
-	P(C, D, A, B,  3, 16, 0xD4EF3085);
-	P(B, C, D, A,  6, 23, 0x04881D05);
-	P(A, B, C, D,  9,  4, 0xD9D4D039);
-	P(D, A, B, C, 12, 11, 0xE6DB99E5);
-	P(C, D, A, B, 15, 16, 0x1FA27CF8);
-	P(B, C, D, A,  2, 23, 0xC4AC5665);
-
-#undef F
-#define F(x,y,z) (y ^ (x | ~z))
-
-	P(A, B, C, D,  0,  6, 0xF4292244);
-	P(D, A, B, C,  7, 10, 0x432AFF97);
-	P(C, D, A, B, 14, 15, 0xAB9423A7);
-	P(B, C, D, A,  5, 21, 0xFC93A039);
-	P(A, B, C, D, 12,  6, 0x655B59C3);
-	P(D, A, B, C,  3, 10, 0x8F0CCC92);
-	P(C, D, A, B, 10, 15, 0xFFEFF47D);
-	P(B, C, D, A,  1, 21, 0x85845DD1);
-	P(A, B, C, D,  8,  6, 0x6FA87E4F);
-	P(D, A, B, C, 15, 10, 0xFE2CE6E0);
-	P(C, D, A, B,  6, 15, 0xA3014314);
-	P(B, C, D, A, 13, 21, 0x4E0811A1);
-	P(A, B, C, D,  4,  6, 0xF7537E82);
-	P(D, A, B, C, 11, 10, 0xBD3AF235);
-	P(C, D, A, B,  2, 15, 0x2AD7D2BB);
-	P(B, C, D, A,  9, 21, 0xEB86D391);
-
-#undef F
-
-	ctx->A += A;
-	ctx->B += B;
-	ctx->C += C;
-	ctx->D += D;
-}
-
-void md5_update(md_context *ctx, const uint8_t *input, uint32_t length)
-{
-	uint32_t left, fill;
-
-	if (!length)
-		return;
-
-	left = ctx->totalN & 0x3F;
-	fill = CSUM_CHUNK - left;
-
-	ctx->totalN += length;
-	ctx->totalN &= 0xFFFFFFFF;
-
-	if (ctx->totalN < length)
-		ctx->totalN2++;
-
-	if (left && length >= fill) {
-		memcpy(ctx->buffer + left, input, fill);
-		md5_process(ctx, ctx->buffer);
-		length -= fill;
-		input  += fill;
-		left = 0;
-	}
-
-	while (length >= CSUM_CHUNK) {
-		md5_process(ctx, input);
-		length -= CSUM_CHUNK;
-		input  += CSUM_CHUNK;
-	}
-
-	if (length)
-		memcpy(ctx->buffer + left, input, length);
-}
-
-static uint8_t md5_padding[CSUM_CHUNK] = { 0x80 };
-
-void md5_result(md_context *ctx, uint8_t digest[MD5_DIGEST_LEN])
-{
-	uint32_t last, padn;
-	uint32_t high, low;
-	uint8_t msglen[8];
-
-	high = (ctx->totalN >> 29)
-	     | (ctx->totalN2 <<  3);
-	low  = (ctx->totalN <<  3);
-
-	SIVAL(msglen, 0, low);
-	SIVAL(msglen, 4, high);
-
-	last = ctx->totalN & 0x3F;
-	padn = last < 56 ? 56 - last : 120 - last;
-
-	md5_update(ctx, md5_padding, padn);
-	md5_update(ctx, msglen, 8);
-
-	SIVAL(digest, 0, ctx->A);
-	SIVAL(digest, 4, ctx->B);
-	SIVAL(digest, 8, ctx->C);
-	SIVAL(digest, 12, ctx->D);
-}
-
-void get_md5(uint8_t *out, const uint8_t *input, int n)
-{
-	md_context ctx;
-	md5_begin(&ctx);
-	md5_update(&ctx, input, n);
-	md5_result(&ctx, out);
-}
-
-#ifdef TEST_MD5
-
-#include <stdlib.h>
-#include <stdio.h>
-
-/*
- * those are the standard RFC 1321 test vectors
- */
-
-static struct {
-    char *str, *md5;
-} tests[] = {
- { "",
-   "d41d8cd98f00b204e9800998ecf8427e" },
- { "a",
-   "0cc175b9c0f1b6a831c399e269772661" },
- { "abc",
-   "900150983cd24fb0d6963f7d28e17f72" },
- { "message digest",
-   "f96b697d7cb7938d525a2f31aaf161d0" },
- { "abcdefghijklmnopqrstuvwxyz",
-   "c3fcd3d76192e4007dfb496cca67e13b" },
- { "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789",
-   "d174ab98d277d9f5a5611c2c9f419d9f" },
- { "12345678901234567890123456789012345678901234567890123456789012345678901234567890",
-   "57edf4a22be3c955ac49da2e2107b67a" },
- { NULL, NULL }
-};
-
-int main(int argc, char *argv[])
-{
-	FILE *f;
-	int i, j;
-	char output[33];
-	md_context ctx;
-	uint8_t buf[1000];
-	uint8_t md5sum[MD5_DIGEST_LEN];
-
-	if (argc < 2) {
-		printf("\nMD5 Validation Tests:\n\n");
-
-		for (i = 0; tests[i].str; i++) {
-			char *str = tests[i].str;
-			char *chk = tests[i].md5;
-
-			printf("  Test %d ", i + 1);
-
-			get_md5(md5sum, str, strlen(str));
-
-			for (j = 0; j < MD5_DIGEST_LEN; j++)
-				sprintf(output + j * 2, "%02x", md5sum[j]);
-
-			if (memcmp(output, chk, 32)) {
-				printf("failed!\n");
-				return 1;
-			}
-
-			printf("passed.\n");
-		}
-
-		printf("\n");
-		return 0;
-	}
-
-	while (--argc) {
-		if (!(f = fopen(*++argv, "rb"))) {
-			perror("fopen");
-			return 1;
-		}
-
-		md5_begin(&ctx);
-
-		while ((i = fread(buf, 1, sizeof buf, f)) > 0)
-			md5_update(&ctx, buf, i);
-
-		fclose(f);
-
-		md5_result(&ctx, md5sum);
-
-		for (j = 0; j < MD5_DIGEST_LEN; j++)
-			printf("%02x", md5sum[j]);
-
-		printf("  %s\n", *argv);
-	}
-
-	return 0;
-}
-
-#endif
diff --git a/contrib/md5/md5.h b/contrib/md5/md5.h
deleted file mode 100644
index ba8f08dbc..000000000
--- a/contrib/md5/md5.h
+++ /dev/null
@@ -1,78 +0,0 @@
-/* rsync-3.0.6/byteorder.h */
-
-/*
- * Simple byteorder handling.
- *
- * Copyright (C) 1992-1995 Andrew Tridgell
- * Copyright (C) 2007-2008 Wayne Davison
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License as published by
- * the Free Software Foundation; either version 3 of the License, or
- * (at your option) any later version.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
- * GNU General Public License for more details.
- *
- * You should have received a copy of the GNU General Public License along
- * with this program; if not, visit the http://fsf.org website.
- */
-
-#undef CAREFUL_ALIGNMENT
-
-/* We know that the x86 can handle misalignment and has the same
- * byte order (LSB-first) as the 32-bit numbers we transmit. */
-
-#ifdef __i386__
-#define CAREFUL_ALIGNMENT 0
-#endif
-
-#ifndef CAREFUL_ALIGNMENT
-#define CAREFUL_ALIGNMENT 1
-#endif
-
-#define CVAL(buf,pos) (((unsigned char *)(buf))[pos])
-#define UVAL(buf,pos) ((uint32_t)CVAL(buf,pos))
-#define SCVAL(buf,pos,val) (CVAL(buf,pos) = (val))
-
-#if CAREFUL_ALIGNMENT
-#define PVAL(buf,pos) (UVAL(buf,pos)|UVAL(buf,(pos)+1)<<8)
-#define IVAL(buf,pos) (PVAL(buf,pos)|PVAL(buf,(pos)+2)<<16)
-#define SSVALX(buf,pos,val) (CVAL(buf,pos)=(val)&0xFF,CVAL(buf,pos+1)=(val)>>8)
-#define SIVALX(buf,pos,val) (SSVALX(buf,pos,val&0xFFFF),SSVALX(buf,pos+2,val>>16))
-#define SIVAL(buf,pos,val) SIVALX((buf),(pos),((uint32_t)(val)))
-#else
-
-/* this handles things for architectures like the 386 that can handle
-   alignment errors */
-
-/*
-   WARNING: This section is dependent on the length of int32
-   being correct. set CAREFUL_ALIGNMENT if it is not.
-*/
-
-#define IVAL(buf,pos) (*(uint32_t *)((char *)(buf) + (pos)))
-#define SIVAL(buf,pos,val) IVAL(buf,pos)=((uint32_t)(val))
-#endif
-
-/* The include file for both the MD4 and MD5 routines. */
-
-#define MD5_DIGEST_LEN 16
-#define MAX_DIGEST_LEN MD5_DIGEST_LEN
-
-#define CSUM_CHUNK 64
-
-typedef struct {
-	uint32_t A, B, C, D;
-	uint32_t totalN;          /* bit count, lower 32 bits */
-	uint32_t totalN2;         /* bit count, upper 32 bits */
-	uint8_t buffer[CSUM_CHUNK];
-} md_context;
-
-void md5_begin(md_context *ctx);
-void md5_update(md_context *ctx, const uint8_t *input, uint32_t length);
-void md5_result(md_context *ctx, uint8_t digest[MD5_DIGEST_LEN]);
-
-void get_md5(uint8_t digest[MD5_DIGEST_LEN], const uint8_t *input, int n);
diff --git a/contrib/qemu/block.c b/contrib/qemu/block.c
new file mode 100644
index 000000000..b56024113
--- /dev/null
+++ b/contrib/qemu/block.c
@@ -0,0 +1,4604 @@
+/*
+ * QEMU System Emulator block driver
+ *
+ * Copyright (c) 2003 Fabrice Bellard
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
+ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+ * THE SOFTWARE.
+ */
+#include "config-host.h"
+#include "qemu-common.h"
+#include "trace.h"
+#include "monitor/monitor.h"
+#include "block/block_int.h"
+#include "block/blockjob.h"
+#include "qemu/module.h"
+#include "qapi/qmp/qjson.h"
+#include "sysemu/sysemu.h"
+#include "qemu/notify.h"
+#include "block/coroutine.h"
+#include "qmp-commands.h"
+#include "qemu/timer.h"
+
+#ifdef CONFIG_BSD
+#include <sys/types.h>
+#include <sys/stat.h>
+#include <sys/ioctl.h>
+#include <sys/queue.h>
+#ifndef __DragonFly__
+#include <sys/disk.h>
+#endif
+#endif
+
+#ifdef _WIN32
+#include <windows.h>
+#endif
+
+#define NOT_DONE 0x7fffffff /* used while emulated sync operation in progress */
+
+typedef enum {
+    BDRV_REQ_COPY_ON_READ = 0x1,
+    BDRV_REQ_ZERO_WRITE   = 0x2,
+} BdrvRequestFlags;
+
+static void bdrv_dev_change_media_cb(BlockDriverState *bs, bool load);
+static BlockDriverAIOCB *bdrv_aio_readv_em(BlockDriverState *bs,
+        int64_t sector_num, QEMUIOVector *qiov, int nb_sectors,
+        BlockDriverCompletionFunc *cb, void *opaque);
+static BlockDriverAIOCB *bdrv_aio_writev_em(BlockDriverState *bs,
+        int64_t sector_num, QEMUIOVector *qiov, int nb_sectors,
+        BlockDriverCompletionFunc *cb, void *opaque);
+static int coroutine_fn bdrv_co_readv_em(BlockDriverState *bs,
+                                         int64_t sector_num, int nb_sectors,
+                                         QEMUIOVector *iov);
+static int coroutine_fn bdrv_co_writev_em(BlockDriverState *bs,
+                                         int64_t sector_num, int nb_sectors,
+                                         QEMUIOVector *iov);
+static int coroutine_fn bdrv_co_do_readv(BlockDriverState *bs,
+    int64_t sector_num, int nb_sectors, QEMUIOVector *qiov,
+    BdrvRequestFlags flags);
+static int coroutine_fn bdrv_co_do_writev(BlockDriverState *bs,
+    int64_t sector_num, int nb_sectors, QEMUIOVector *qiov,
+    BdrvRequestFlags flags);
+static BlockDriverAIOCB *bdrv_co_aio_rw_vector(BlockDriverState *bs,
+                                               int64_t sector_num,
+                                               QEMUIOVector *qiov,
+                                               int nb_sectors,
+                                               BlockDriverCompletionFunc *cb,
+                                               void *opaque,
+                                               bool is_write);
+static void coroutine_fn bdrv_co_do_rw(void *opaque);
+static int coroutine_fn bdrv_co_do_write_zeroes(BlockDriverState *bs,
+    int64_t sector_num, int nb_sectors);
+
+static bool bdrv_exceed_bps_limits(BlockDriverState *bs, int nb_sectors,
+        bool is_write, double elapsed_time, uint64_t *wait);
+static bool bdrv_exceed_iops_limits(BlockDriverState *bs, bool is_write,
+        double elapsed_time, uint64_t *wait);
+static bool bdrv_exceed_io_limits(BlockDriverState *bs, int nb_sectors,
+        bool is_write, int64_t *wait);
+
+static QTAILQ_HEAD(, BlockDriverState) bdrv_states =
+    QTAILQ_HEAD_INITIALIZER(bdrv_states);
+
+static QLIST_HEAD(, BlockDriver) bdrv_drivers =
+    QLIST_HEAD_INITIALIZER(bdrv_drivers);
+
+/* If non-zero, use only whitelisted block drivers */
+static int use_bdrv_whitelist;
+
+#ifdef _WIN32
+static int is_windows_drive_prefix(const char *filename)
+{
+    return (((filename[0] >= 'a' && filename[0] <= 'z') ||
+             (filename[0] >= 'A' && filename[0] <= 'Z')) &&
+            filename[1] == ':');
+}
+
+int is_windows_drive(const char *filename)
+{
+    if (is_windows_drive_prefix(filename) &&
+        filename[2] == '\0')
+        return 1;
+    if (strstart(filename, "\\\\.\\", NULL) ||
+        strstart(filename, "//./", NULL))
+        return 1;
+    return 0;
+}
+#endif
+
+/* throttling disk I/O limits */
+void bdrv_io_limits_disable(BlockDriverState *bs)
+{
+    bs->io_limits_enabled = false;
+
+    while (qemu_co_queue_next(&bs->throttled_reqs));
+
+    if (bs->block_timer) {
+        qemu_del_timer(bs->block_timer);
+        qemu_free_timer(bs->block_timer);
+        bs->block_timer = NULL;
+    }
+
+    bs->slice_start = 0;
+    bs->slice_end   = 0;
+}
+
+static void bdrv_block_timer(void *opaque)
+{
+    BlockDriverState *bs = opaque;
+
+    qemu_co_queue_next(&bs->throttled_reqs);
+}
+
+void bdrv_io_limits_enable(BlockDriverState *bs)
+{
+    qemu_co_queue_init(&bs->throttled_reqs);
+    bs->block_timer = qemu_new_timer_ns(vm_clock, bdrv_block_timer, bs);
+    bs->io_limits_enabled = true;
+}
+
+bool bdrv_io_limits_enabled(BlockDriverState *bs)
+{
+    BlockIOLimit *io_limits = &bs->io_limits;
+    return io_limits->bps[BLOCK_IO_LIMIT_READ]
+         || io_limits->bps[BLOCK_IO_LIMIT_WRITE]
+         || io_limits->bps[BLOCK_IO_LIMIT_TOTAL]
+         || io_limits->iops[BLOCK_IO_LIMIT_READ]
+         || io_limits->iops[BLOCK_IO_LIMIT_WRITE]
+         || io_limits->iops[BLOCK_IO_LIMIT_TOTAL];
+}
+
+static void bdrv_io_limits_intercept(BlockDriverState *bs,
+                                     bool is_write, int nb_sectors)
+{
+    int64_t wait_time = -1;
+
+    if (!qemu_co_queue_empty(&bs->throttled_reqs)) {
+        qemu_co_queue_wait(&bs->throttled_reqs);
+    }
+
+    /* In fact, we hope to keep each request's timing, in FIFO mode. The next
+     * throttled requests will not be dequeued until the current request is
+     * allowed to be serviced. So if the current request still exceeds the
+     * limits, it will be inserted to the head. All requests followed it will
+     * be still in throttled_reqs queue.
+     */
+
+    while (bdrv_exceed_io_limits(bs, nb_sectors, is_write, &wait_time)) {
+        qemu_mod_timer(bs->block_timer,
+                       wait_time + qemu_get_clock_ns(vm_clock));
+        qemu_co_queue_wait_insert_head(&bs->throttled_reqs);
+    }
+
+    qemu_co_queue_next(&bs->throttled_reqs);
+}
+
+/* check if the path starts with "<protocol>:" */
+static int path_has_protocol(const char *path)
+{
+    const char *p;
+
+#ifdef _WIN32
+    if (is_windows_drive(path) ||
+        is_windows_drive_prefix(path)) {
+        return 0;
+    }
+    p = path + strcspn(path, ":/\\");
+#else
+    p = path + strcspn(path, ":/");
+#endif
+
+    return *p == ':';
+}
+
+int path_is_absolute(const char *path)
+{
+#ifdef _WIN32
+    /* specific case for names like: "\\.\d:" */
+    if (is_windows_drive(path) || is_windows_drive_prefix(path)) {
+        return 1;
+    }
+    return (*path == '/' || *path == '\\');
+#else
+    return (*path == '/');
+#endif
+}
+
+/* if filename is absolute, just copy it to dest. Otherwise, build a
+   path to it by considering it is relative to base_path. URL are
+   supported. */
+void path_combine(char *dest, int dest_size,
+                  const char *base_path,
+                  const char *filename)
+{
+    const char *p, *p1;
+    int len;
+
+    if (dest_size <= 0)
+        return;
+    if (path_is_absolute(filename)) {
+        pstrcpy(dest, dest_size, filename);
+    } else {
+        p = strchr(base_path, ':');
+        if (p)
+            p++;
+        else
+            p = base_path;
+        p1 = strrchr(base_path, '/');
+#ifdef _WIN32
+        {
+            const char *p2;
+            p2 = strrchr(base_path, '\\');
+            if (!p1 || p2 > p1)
+                p1 = p2;
+        }
+#endif
+        if (p1)
+            p1++;
+        else
+            p1 = base_path;
+        if (p1 > p)
+            p = p1;
+        len = p - base_path;
+        if (len > dest_size - 1)
+            len = dest_size - 1;
+        memcpy(dest, base_path, len);
+        dest[len] = '\0';
+        pstrcat(dest, dest_size, filename);
+    }
+}
+
+void bdrv_get_full_backing_filename(BlockDriverState *bs, char *dest, size_t sz)
+{
+    if (bs->backing_file[0] == '\0' || path_has_protocol(bs->backing_file)) {
+        pstrcpy(dest, sz, bs->backing_file);
+    } else {
+        path_combine(dest, sz, bs->filename, bs->backing_file);
+    }
+}
+
+void bdrv_register(BlockDriver *bdrv)
+{
+    /* Block drivers without coroutine functions need emulation */
+    if (!bdrv->bdrv_co_readv) {
+        bdrv->bdrv_co_readv = bdrv_co_readv_em;
+        bdrv->bdrv_co_writev = bdrv_co_writev_em;
+
+        /* bdrv_co_readv_em()/brdv_co_writev_em() work in terms of aio, so if
+         * the block driver lacks aio we need to emulate that too.
+         */
+        if (!bdrv->bdrv_aio_readv) {
+            /* add AIO emulation layer */
+            bdrv->bdrv_aio_readv = bdrv_aio_readv_em;
+            bdrv->bdrv_aio_writev = bdrv_aio_writev_em;
+        }
+    }
+
+    QLIST_INSERT_HEAD(&bdrv_drivers, bdrv, list);
+}
+
+/* create a new block device (by default it is empty) */
+BlockDriverState *bdrv_new(const char *device_name)
+{
+    BlockDriverState *bs;
+
+    bs = g_malloc0(sizeof(BlockDriverState));
+    pstrcpy(bs->device_name, sizeof(bs->device_name), device_name);
+    if (device_name[0] != '\0') {
+        QTAILQ_INSERT_TAIL(&bdrv_states, bs, list);
+    }
+    bdrv_iostatus_disable(bs);
+    notifier_list_init(&bs->close_notifiers);
+    notifier_with_return_list_init(&bs->before_write_notifiers);
+
+    return bs;
+}
+
+void bdrv_add_close_notifier(BlockDriverState *bs, Notifier *notify)
+{
+    notifier_list_add(&bs->close_notifiers, notify);
+}
+
+BlockDriver *bdrv_find_format(const char *format_name)
+{
+    BlockDriver *drv1;
+    QLIST_FOREACH(drv1, &bdrv_drivers, list) {
+        if (!strcmp(drv1->format_name, format_name)) {
+            return drv1;
+        }
+    }
+    return NULL;
+}
+
+static int bdrv_is_whitelisted(BlockDriver *drv, bool read_only)
+{
+    static const char *whitelist_rw[] = {
+        CONFIG_BDRV_RW_WHITELIST
+    };
+    static const char *whitelist_ro[] = {
+        CONFIG_BDRV_RO_WHITELIST
+    };
+    const char **p;
+
+    if (!whitelist_rw[0] && !whitelist_ro[0]) {
+        return 1;               /* no whitelist, anything goes */
+    }
+
+    for (p = whitelist_rw; *p; p++) {
+        if (!strcmp(drv->format_name, *p)) {
+            return 1;
+        }
+    }
+    if (read_only) {
+        for (p = whitelist_ro; *p; p++) {
+            if (!strcmp(drv->format_name, *p)) {
+                return 1;
+            }
+        }
+    }
+    return 0;
+}
+
+BlockDriver *bdrv_find_whitelisted_format(const char *format_name,
+                                          bool read_only)
+{
+    BlockDriver *drv = bdrv_find_format(format_name);
+    return drv && bdrv_is_whitelisted(drv, read_only) ? drv : NULL;
+}
+
+typedef struct CreateCo {
+    BlockDriver *drv;
+    char *filename;
+    QEMUOptionParameter *options;
+    int ret;
+} CreateCo;
+
+static void coroutine_fn bdrv_create_co_entry(void *opaque)
+{
+    CreateCo *cco = opaque;
+    assert(cco->drv);
+
+    cco->ret = cco->drv->bdrv_create(cco->filename, cco->options);
+}
+
+int bdrv_create(BlockDriver *drv, const char* filename,
+    QEMUOptionParameter *options)
+{
+    int ret;
+
+    Coroutine *co;
+    CreateCo cco = {
+        .drv = drv,
+        .filename = g_strdup(filename),
+        .options = options,
+        .ret = NOT_DONE,
+    };
+
+    if (!drv->bdrv_create) {
+        ret = -ENOTSUP;
+        goto out;
+    }
+
+    if (qemu_in_coroutine()) {
+        /* Fast-path if already in coroutine context */
+        bdrv_create_co_entry(&cco);
+    } else {
+        co = qemu_coroutine_create(bdrv_create_co_entry);
+        qemu_coroutine_enter(co, &cco);
+        while (cco.ret == NOT_DONE) {
+            qemu_aio_wait();
+        }
+    }
+
+    ret = cco.ret;
+
+out:
+    g_free(cco.filename);
+    return ret;
+}
+
+int bdrv_create_file(const char* filename, QEMUOptionParameter *options)
+{
+    BlockDriver *drv;
+
+    drv = bdrv_find_protocol(filename, true);
+    if (drv == NULL) {
+        return -ENOENT;
+    }
+
+    return bdrv_create(drv, filename, options);
+}
+
+/*
+ * Create a uniquely-named empty temporary file.
+ * Return 0 upon success, otherwise a negative errno value.
+ */
+int get_tmp_filename(char *filename, int size)
+{
+#ifdef _WIN32
+    char temp_dir[MAX_PATH];
+    /* GetTempFileName requires that its output buffer (4th param)
+       have length MAX_PATH or greater.  */
+    assert(size >= MAX_PATH);
+    return (GetTempPath(MAX_PATH, temp_dir)
+            && GetTempFileName(temp_dir, "qem", 0, filename)
+            ? 0 : -GetLastError());
+#else
+    int fd;
+    const char *tmpdir;
+    tmpdir = getenv("TMPDIR");
+    if (!tmpdir)
+        tmpdir = "/tmp";
+    if (snprintf(filename, size, "%s/vl.XXXXXX", tmpdir) >= size) {
+        return -EOVERFLOW;
+    }
+    fd = mkstemp(filename);
+    if (fd < 0) {
+        return -errno;
+    }
+    if (close(fd) != 0) {
+        unlink(filename);
+        return -errno;
+    }
+    return 0;
+#endif
+}
+
+/*
+ * Detect host devices. By convention, /dev/cdrom[N] is always
+ * recognized as a host CDROM.
+ */
+static BlockDriver *find_hdev_driver(const char *filename)
+{
+    int score_max = 0, score;
+    BlockDriver *drv = NULL, *d;
+
+    QLIST_FOREACH(d, &bdrv_drivers, list) {
+        if (d->bdrv_probe_device) {
+            score = d->bdrv_probe_device(filename);
+            if (score > score_max) {
+                score_max = score;
+                drv = d;
+            }
+        }
+    }
+
+    return drv;
+}
+
+BlockDriver *bdrv_find_protocol(const char *filename,
+                                bool allow_protocol_prefix)
+{
+    BlockDriver *drv1;
+    char protocol[128];
+    int len;
+    const char *p;
+
+    /* TODO Drivers without bdrv_file_open must be specified explicitly */
+
+    /*
+     * XXX(hch): we really should not let host device detection
+     * override an explicit protocol specification, but moving this
+     * later breaks access to device names with colons in them.
+     * Thanks to the brain-dead persistent naming schemes on udev-
+     * based Linux systems those actually are quite common.
+     */
+    drv1 = find_hdev_driver(filename);
+    if (drv1) {
+        return drv1;
+    }
+
+    if (!path_has_protocol(filename) || !allow_protocol_prefix) {
+        return bdrv_find_format("file");
+    }
+
+    p = strchr(filename, ':');
+    assert(p != NULL);
+    len = p - filename;
+    if (len > sizeof(protocol) - 1)
+        len = sizeof(protocol) - 1;
+    memcpy(protocol, filename, len);
+    protocol[len] = '\0';
+    QLIST_FOREACH(drv1, &bdrv_drivers, list) {
+        if (drv1->protocol_name &&
+            !strcmp(drv1->protocol_name, protocol)) {
+            return drv1;
+        }
+    }
+    return NULL;
+}
+
+static int find_image_format(BlockDriverState *bs, const char *filename,
+                             BlockDriver **pdrv)
+{
+    int score, score_max;
+    BlockDriver *drv1, *drv;
+    uint8_t buf[2048];
+    int ret = 0;
+
+    /* Return the raw BlockDriver * to scsi-generic devices or empty drives */
+    if (bs->sg || !bdrv_is_inserted(bs) || bdrv_getlength(bs) == 0) {
+        drv = bdrv_find_format("raw");
+        if (!drv) {
+            ret = -ENOENT;
+        }
+        *pdrv = drv;
+        return ret;
+    }
+
+    ret = bdrv_pread(bs, 0, buf, sizeof(buf));
+    if (ret < 0) {
+        *pdrv = NULL;
+        return ret;
+    }
+
+    score_max = 0;
+    drv = NULL;
+    QLIST_FOREACH(drv1, &bdrv_drivers, list) {
+        if (drv1->bdrv_probe) {
+            score = drv1->bdrv_probe(buf, ret, filename);
+            if (score > score_max) {
+                score_max = score;
+                drv = drv1;
+            }
+        }
+    }
+    if (!drv) {
+        ret = -ENOENT;
+    }
+    *pdrv = drv;
+    return ret;
+}
+
+/**
+ * Set the current 'total_sectors' value
+ */
+static int refresh_total_sectors(BlockDriverState *bs, int64_t hint)
+{
+    BlockDriver *drv = bs->drv;
+
+    /* Do not attempt drv->bdrv_getlength() on scsi-generic devices */
+    if (bs->sg)
+        return 0;
+
+    /* query actual device if possible, otherwise just trust the hint */
+    if (drv->bdrv_getlength) {
+        int64_t length = drv->bdrv_getlength(bs);
+        if (length < 0) {
+            return length;
+        }
+        hint = length >> BDRV_SECTOR_BITS;
+    }
+
+    bs->total_sectors = hint;
+    return 0;
+}
+
+/**
+ * Set open flags for a given discard mode
+ *
+ * Return 0 on success, -1 if the discard mode was invalid.
+ */
+int bdrv_parse_discard_flags(const char *mode, int *flags)
+{
+    *flags &= ~BDRV_O_UNMAP;
+
+    if (!strcmp(mode, "off") || !strcmp(mode, "ignore")) {
+        /* do nothing */
+    } else if (!strcmp(mode, "on") || !strcmp(mode, "unmap")) {
+        *flags |= BDRV_O_UNMAP;
+    } else {
+        return -1;
+    }
+
+    return 0;
+}
+
+/**
+ * Set open flags for a given cache mode
+ *
+ * Return 0 on success, -1 if the cache mode was invalid.
+ */
+int bdrv_parse_cache_flags(const char *mode, int *flags)
+{
+    *flags &= ~BDRV_O_CACHE_MASK;
+
+    if (!strcmp(mode, "off") || !strcmp(mode, "none")) {
+        *flags |= BDRV_O_NOCACHE | BDRV_O_CACHE_WB;
+    } else if (!strcmp(mode, "directsync")) {
+        *flags |= BDRV_O_NOCACHE;
+    } else if (!strcmp(mode, "writeback")) {
+        *flags |= BDRV_O_CACHE_WB;
+    } else if (!strcmp(mode, "unsafe")) {
+        *flags |= BDRV_O_CACHE_WB;
+        *flags |= BDRV_O_NO_FLUSH;
+    } else if (!strcmp(mode, "writethrough")) {
+        /* this is the default */
+    } else {
+        return -1;
+    }
+
+    return 0;
+}
+
+/**
+ * The copy-on-read flag is actually a reference count so multiple users may
+ * use the feature without worrying about clobbering its previous state.
+ * Copy-on-read stays enabled until all users have called to disable it.
+ */
+void bdrv_enable_copy_on_read(BlockDriverState *bs)
+{
+    bs->copy_on_read++;
+}
+
+void bdrv_disable_copy_on_read(BlockDriverState *bs)
+{
+    assert(bs->copy_on_read > 0);
+    bs->copy_on_read--;
+}
+
+static int bdrv_open_flags(BlockDriverState *bs, int flags)
+{
+    int open_flags = flags | BDRV_O_CACHE_WB;
+
+    /*
+     * Clear flags that are internal to the block layer before opening the
+     * image.
+     */
+    open_flags &= ~(BDRV_O_SNAPSHOT | BDRV_O_NO_BACKING);
+
+    /*
+     * Snapshots should be writable.
+     */
+    if (bs->is_temporary) {
+        open_flags |= BDRV_O_RDWR;
+    }
+
+    return open_flags;
+}
+
+/*
+ * Common part for opening disk images and files
+ *
+ * Removes all processed options from *options.
+ */
+static int bdrv_open_common(BlockDriverState *bs, BlockDriverState *file,
+    QDict *options, int flags, BlockDriver *drv)
+{
+    int ret, open_flags;
+    const char *filename;
+
+    assert(drv != NULL);
+    assert(bs->file == NULL);
+    assert(options != NULL && bs->options != options);
+
+    if (file != NULL) {
+        filename = file->filename;
+    } else {
+        filename = qdict_get_try_str(options, "filename");
+    }
+
+    trace_bdrv_open_common(bs, filename ?: "", flags, drv->format_name);
+
+    /* bdrv_open() with directly using a protocol as drv. This layer is already
+     * opened, so assign it to bs (while file becomes a closed BlockDriverState)
+     * and return immediately. */
+    if (file != NULL && drv->bdrv_file_open) {
+        bdrv_swap(file, bs);
+        return 0;
+    }
+
+    bs->open_flags = flags;
+    bs->buffer_alignment = 512;
+    open_flags = bdrv_open_flags(bs, flags);
+    bs->read_only = !(open_flags & BDRV_O_RDWR);
+
+    if (use_bdrv_whitelist && !bdrv_is_whitelisted(drv, bs->read_only)) {
+        return -ENOTSUP;
+    }
+
+    assert(bs->copy_on_read == 0); /* bdrv_new() and bdrv_close() make it so */
+    if (!bs->read_only && (flags & BDRV_O_COPY_ON_READ)) {
+        bdrv_enable_copy_on_read(bs);
+    }
+
+    if (filename != NULL) {
+        pstrcpy(bs->filename, sizeof(bs->filename), filename);
+    } else {
+        bs->filename[0] = '\0';
+    }
+
+    bs->drv = drv;
+    bs->opaque = g_malloc0(drv->instance_size);
+
+    bs->enable_write_cache = !!(flags & BDRV_O_CACHE_WB);
+
+    /* Open the image, either directly or using a protocol */
+    if (drv->bdrv_file_open) {
+        assert(file == NULL);
+        assert(drv->bdrv_parse_filename || filename != NULL);
+        ret = drv->bdrv_file_open(bs, options, open_flags);
+    } else {
+        if (file == NULL) {
+            qerror_report(ERROR_CLASS_GENERIC_ERROR, "Can't use '%s' as a "
+                          "block driver for the protocol level",
+                          drv->format_name);
+            ret = -EINVAL;
+            goto free_and_fail;
+        }
+        assert(file != NULL);
+        bs->file = file;
+        ret = drv->bdrv_open(bs, options, open_flags);
+    }
+
+    if (ret < 0) {
+        goto free_and_fail;
+    }
+
+    ret = refresh_total_sectors(bs, bs->total_sectors);
+    if (ret < 0) {
+        goto free_and_fail;
+    }
+
+#ifndef _WIN32
+    if (bs->is_temporary) {
+        assert(filename != NULL);
+        unlink(filename);
+    }
+#endif
+    return 0;
+
+free_and_fail:
+    bs->file = NULL;
+    g_free(bs->opaque);
+    bs->opaque = NULL;
+    bs->drv = NULL;
+    return ret;
+}
+
+/*
+ * Opens a file using a protocol (file, host_device, nbd, ...)
+ *
+ * options is a QDict of options to pass to the block drivers, or NULL for an
+ * empty set of options. The reference to the QDict belongs to the block layer
+ * after the call (even on failure), so if the caller intends to reuse the
+ * dictionary, it needs to use QINCREF() before calling bdrv_file_open.
+ */
+int bdrv_file_open(BlockDriverState **pbs, const char *filename,
+                   QDict *options, int flags)
+{
+    BlockDriverState *bs;
+    BlockDriver *drv;
+    const char *drvname;
+    bool allow_protocol_prefix = false;
+    int ret;
+
+    /* NULL means an empty set of options */
+    if (options == NULL) {
+        options = qdict_new();
+    }
+
+    bs = bdrv_new("");
+    bs->options = options;
+    options = qdict_clone_shallow(options);
+
+    /* Fetch the file name from the options QDict if necessary */
+    if (!filename) {
+        filename = qdict_get_try_str(options, "filename");
+    } else if (filename && !qdict_haskey(options, "filename")) {
+        qdict_put(options, "filename", qstring_from_str(filename));
+        allow_protocol_prefix = true;
+    } else {
+        qerror_report(ERROR_CLASS_GENERIC_ERROR, "Can't specify 'file' and "
+                      "'filename' options at the same time");
+        ret = -EINVAL;
+        goto fail;
+    }
+
+    /* Find the right block driver */
+    drvname = qdict_get_try_str(options, "driver");
+    if (drvname) {
+        drv = bdrv_find_whitelisted_format(drvname, !(flags & BDRV_O_RDWR));
+        qdict_del(options, "driver");
+    } else if (filename) {
+        drv = bdrv_find_protocol(filename, allow_protocol_prefix);
+        if (!drv) {
+            qerror_report(ERROR_CLASS_GENERIC_ERROR, "Unknown protocol");
+        }
+    } else {
+        qerror_report(ERROR_CLASS_GENERIC_ERROR,
+                      "Must specify either driver or file");
+        drv = NULL;
+    }
+
+    if (!drv) {
+        ret = -ENOENT;
+        goto fail;
+    }
+
+    /* Parse the filename and open it */
+    if (drv->bdrv_parse_filename && filename) {
+        Error *local_err = NULL;
+        drv->bdrv_parse_filename(filename, options, &local_err);
+        if (error_is_set(&local_err)) {
+            qerror_report_err(local_err);
+            error_free(local_err);
+            ret = -EINVAL;
+            goto fail;
+        }
+        qdict_del(options, "filename");
+    } else if (!drv->bdrv_parse_filename && !filename) {
+        qerror_report(ERROR_CLASS_GENERIC_ERROR,
+                      "The '%s' block driver requires a file name",
+                      drv->format_name);
+        ret = -EINVAL;
+        goto fail;
+    }
+
+    ret = bdrv_open_common(bs, NULL, options, flags, drv);
+    if (ret < 0) {
+        goto fail;
+    }
+
+    /* Check if any unknown options were used */
+    if (qdict_size(options) != 0) {
+        const QDictEntry *entry = qdict_first(options);
+        qerror_report(ERROR_CLASS_GENERIC_ERROR, "Block protocol '%s' doesn't "
+                      "support the option '%s'",
+                      drv->format_name, entry->key);
+        ret = -EINVAL;
+        goto fail;
+    }
+    QDECREF(options);
+
+    bs->growable = 1;
+    *pbs = bs;
+    return 0;
+
+fail:
+    QDECREF(options);
+    if (!bs->drv) {
+        QDECREF(bs->options);
+    }
+    bdrv_delete(bs);
+    return ret;
+}
+
+/*
+ * Opens the backing file for a BlockDriverState if not yet open
+ *
+ * options is a QDict of options to pass to the block drivers, or NULL for an
+ * empty set of options. The reference to the QDict is transferred to this
+ * function (even on failure), so if the caller intends to reuse the dictionary,
+ * it needs to use QINCREF() before calling bdrv_file_open.
+ */
+int bdrv_open_backing_file(BlockDriverState *bs, QDict *options)
+{
+    char backing_filename[PATH_MAX];
+    int back_flags, ret;
+    BlockDriver *back_drv = NULL;
+
+    if (bs->backing_hd != NULL) {
+        QDECREF(options);
+        return 0;
+    }
+
+    /* NULL means an empty set of options */
+    if (options == NULL) {
+        options = qdict_new();
+    }
+
+    bs->open_flags &= ~BDRV_O_NO_BACKING;
+    if (qdict_haskey(options, "file.filename")) {
+        backing_filename[0] = '\0';
+    } else if (bs->backing_file[0] == '\0' && qdict_size(options) == 0) {
+        QDECREF(options);
+        return 0;
+    }
+
+    bs->backing_hd = bdrv_new("");
+    bdrv_get_full_backing_filename(bs, backing_filename,
+                                   sizeof(backing_filename));
+
+    if (bs->backing_format[0] != '\0') {
+        back_drv = bdrv_find_format(bs->backing_format);
+    }
+
+    /* backing files always opened read-only */
+    back_flags = bs->open_flags & ~(BDRV_O_RDWR | BDRV_O_SNAPSHOT);
+
+    ret = bdrv_open(bs->backing_hd,
+                    *backing_filename ? backing_filename : NULL, options,
+                    back_flags, back_drv);
+    if (ret < 0) {
+        bdrv_delete(bs->backing_hd);
+        bs->backing_hd = NULL;
+        bs->open_flags |= BDRV_O_NO_BACKING;
+        return ret;
+    }
+    return 0;
+}
+
+static void extract_subqdict(QDict *src, QDict **dst, const char *start)
+{
+    const QDictEntry *entry, *next;
+    const char *p;
+
+    *dst = qdict_new();
+    entry = qdict_first(src);
+
+    while (entry != NULL) {
+        next = qdict_next(src, entry);
+        if (strstart(entry->key, start, &p)) {
+            qobject_incref(entry->value);
+            qdict_put_obj(*dst, p, entry->value);
+            qdict_del(src, entry->key);
+        }
+        entry = next;
+    }
+}
+
+/*
+ * Opens a disk image (raw, qcow2, vmdk, ...)
+ *
+ * options is a QDict of options to pass to the block drivers, or NULL for an
+ * empty set of options. The reference to the QDict belongs to the block layer
+ * after the call (even on failure), so if the caller intends to reuse the
+ * dictionary, it needs to use QINCREF() before calling bdrv_open.
+ */
+int bdrv_open(BlockDriverState *bs, const char *filename, QDict *options,
+              int flags, BlockDriver *drv)
+{
+    int ret;
+    /* TODO: extra byte is a hack to ensure MAX_PATH space on Windows. */
+    char tmp_filename[PATH_MAX + 1];
+    BlockDriverState *file = NULL;
+    QDict *file_options = NULL;
+
+    /* NULL means an empty set of options */
+    if (options == NULL) {
+        options = qdict_new();
+    }
+
+    bs->options = options;
+    options = qdict_clone_shallow(options);
+
+    /* For snapshot=on, create a temporary qcow2 overlay */
+    if (flags & BDRV_O_SNAPSHOT) {
+        BlockDriverState *bs1;
+        int64_t total_size;
+        BlockDriver *bdrv_qcow2;
+        QEMUOptionParameter *create_options;
+        char backing_filename[PATH_MAX];
+
+        if (qdict_size(options) != 0) {
+            error_report("Can't use snapshot=on with driver-specific options");
+            ret = -EINVAL;
+            goto fail;
+        }
+        assert(filename != NULL);
+
+        /* if snapshot, we create a temporary backing file and open it
+           instead of opening 'filename' directly */
+
+        /* if there is a backing file, use it */
+        bs1 = bdrv_new("");
+        ret = bdrv_open(bs1, filename, NULL, 0, drv);
+        if (ret < 0) {
+            bdrv_delete(bs1);
+            goto fail;
+        }
+        total_size = bdrv_getlength(bs1) & BDRV_SECTOR_MASK;
+
+        bdrv_delete(bs1);
+
+        ret = get_tmp_filename(tmp_filename, sizeof(tmp_filename));
+        if (ret < 0) {
+            goto fail;
+        }
+
+        /* Real path is meaningless for protocols */
+        if (path_has_protocol(filename)) {
+            snprintf(backing_filename, sizeof(backing_filename),
+                     "%s", filename);
+        } else if (!realpath(filename, backing_filename)) {
+            ret = -errno;
+            goto fail;
+        }
+
+        bdrv_qcow2 = bdrv_find_format("qcow2");
+        create_options = parse_option_parameters("", bdrv_qcow2->create_options,
+                                                 NULL);
+
+        set_option_parameter_int(create_options, BLOCK_OPT_SIZE, total_size);
+        set_option_parameter(create_options, BLOCK_OPT_BACKING_FILE,
+                             backing_filename);
+        if (drv) {
+            set_option_parameter(create_options, BLOCK_OPT_BACKING_FMT,
+                drv->format_name);
+        }
+
+        ret = bdrv_create(bdrv_qcow2, tmp_filename, create_options);
+        free_option_parameters(create_options);
+        if (ret < 0) {
+            goto fail;
+        }
+
+        filename = tmp_filename;
+        drv = bdrv_qcow2;
+        bs->is_temporary = 1;
+    }
+
+    /* Open image file without format layer */
+    if (flags & BDRV_O_RDWR) {
+        flags |= BDRV_O_ALLOW_RDWR;
+    }
+
+    extract_subqdict(options, &file_options, "file.");
+
+    ret = bdrv_file_open(&file, filename, file_options,
+                         bdrv_open_flags(bs, flags | BDRV_O_UNMAP));
+    if (ret < 0) {
+        goto fail;
+    }
+
+    /* Find the right image format driver */
+    if (!drv) {
+        ret = find_image_format(file, filename, &drv);
+    }
+
+    if (!drv) {
+        goto unlink_and_fail;
+    }
+
+    /* Open the image */
+    ret = bdrv_open_common(bs, file, options, flags, drv);
+    if (ret < 0) {
+        goto unlink_and_fail;
+    }
+
+    if (bs->file != file) {
+        bdrv_delete(file);
+        file = NULL;
+    }
+
+    /* If there is a backing file, use it */
+    if ((flags & BDRV_O_NO_BACKING) == 0) {
+        QDict *backing_options;
+
+        extract_subqdict(options, &backing_options, "backing.");
+        ret = bdrv_open_backing_file(bs, backing_options);
+        if (ret < 0) {
+            goto close_and_fail;
+        }
+    }
+
+    /* Check if any unknown options were used */
+    if (qdict_size(options) != 0) {
+        const QDictEntry *entry = qdict_first(options);
+        qerror_report(ERROR_CLASS_GENERIC_ERROR, "Block format '%s' used by "
+            "device '%s' doesn't support the option '%s'",
+            drv->format_name, bs->device_name, entry->key);
+
+        ret = -EINVAL;
+        goto close_and_fail;
+    }
+    QDECREF(options);
+
+    if (!bdrv_key_required(bs)) {
+        bdrv_dev_change_media_cb(bs, true);
+    }
+
+    /* throttling disk I/O limits */
+    if (bs->io_limits_enabled) {
+        bdrv_io_limits_enable(bs);
+    }
+
+    return 0;
+
+unlink_and_fail:
+    if (file != NULL) {
+        bdrv_delete(file);
+    }
+    if (bs->is_temporary) {
+        unlink(filename);
+    }
+fail:
+    QDECREF(bs->options);
+    QDECREF(options);
+    bs->options = NULL;
+    return ret;
+
+close_and_fail:
+    bdrv_close(bs);
+    QDECREF(options);
+    return ret;
+}
+
+typedef struct BlockReopenQueueEntry {
+     bool prepared;
+     BDRVReopenState state;
+     QSIMPLEQ_ENTRY(BlockReopenQueueEntry) entry;
+} BlockReopenQueueEntry;
+
+/*
+ * Adds a BlockDriverState to a simple queue for an atomic, transactional
+ * reopen of multiple devices.
+ *
+ * bs_queue can either be an existing BlockReopenQueue that has had QSIMPLE_INIT
+ * already performed, or alternatively may be NULL a new BlockReopenQueue will
+ * be created and initialized. This newly created BlockReopenQueue should be
+ * passed back in for subsequent calls that are intended to be of the same
+ * atomic 'set'.
+ *
+ * bs is the BlockDriverState to add to the reopen queue.
+ *
+ * flags contains the open flags for the associated bs
+ *
+ * returns a pointer to bs_queue, which is either the newly allocated
+ * bs_queue, or the existing bs_queue being used.
+ *
+ */
+BlockReopenQueue *bdrv_reopen_queue(BlockReopenQueue *bs_queue,
+                                    BlockDriverState *bs, int flags)
+{
+    assert(bs != NULL);
+
+    BlockReopenQueueEntry *bs_entry;
+    if (bs_queue == NULL) {
+        bs_queue = g_new0(BlockReopenQueue, 1);
+        QSIMPLEQ_INIT(bs_queue);
+    }
+
+    if (bs->file) {
+        bdrv_reopen_queue(bs_queue, bs->file, flags);
+    }
+
+    bs_entry = g_new0(BlockReopenQueueEntry, 1);
+    QSIMPLEQ_INSERT_TAIL(bs_queue, bs_entry, entry);
+
+    bs_entry->state.bs = bs;
+    bs_entry->state.flags = flags;
+
+    return bs_queue;
+}
+
+/*
+ * Reopen multiple BlockDriverStates atomically & transactionally.
+ *
+ * The queue passed in (bs_queue) must have been built up previous
+ * via bdrv_reopen_queue().
+ *
+ * Reopens all BDS specified in the queue, with the appropriate
+ * flags.  All devices are prepared for reopen, and failure of any
+ * device will cause all device changes to be abandonded, and intermediate
+ * data cleaned up.
+ *
+ * If all devices prepare successfully, then the changes are committed
+ * to all devices.
+ *
+ */
+int bdrv_reopen_multiple(BlockReopenQueue *bs_queue, Error **errp)
+{
+    int ret = -1;
+    BlockReopenQueueEntry *bs_entry, *next;
+    Error *local_err = NULL;
+
+    assert(bs_queue != NULL);
+
+    bdrv_drain_all();
+
+    QSIMPLEQ_FOREACH(bs_entry, bs_queue, entry) {
+        if (bdrv_reopen_prepare(&bs_entry->state, bs_queue, &local_err)) {
+            error_propagate(errp, local_err);
+            goto cleanup;
+        }
+        bs_entry->prepared = true;
+    }
+
+    /* If we reach this point, we have success and just need to apply the
+     * changes
+     */
+    QSIMPLEQ_FOREACH(bs_entry, bs_queue, entry) {
+        bdrv_reopen_commit(&bs_entry->state);
+    }
+
+    ret = 0;
+
+cleanup:
+    QSIMPLEQ_FOREACH_SAFE(bs_entry, bs_queue, entry, next) {
+        if (ret && bs_entry->prepared) {
+            bdrv_reopen_abort(&bs_entry->state);
+        }
+        g_free(bs_entry);
+    }
+    g_free(bs_queue);
+    return ret;
+}
+
+
+/* Reopen a single BlockDriverState with the specified flags. */
+int bdrv_reopen(BlockDriverState *bs, int bdrv_flags, Error **errp)
+{
+    int ret = -1;
+    Error *local_err = NULL;
+    BlockReopenQueue *queue = bdrv_reopen_queue(NULL, bs, bdrv_flags);
+
+    ret = bdrv_reopen_multiple(queue, &local_err);
+    if (local_err != NULL) {
+        error_propagate(errp, local_err);
+    }
+    return ret;
+}
+
+
+/*
+ * Prepares a BlockDriverState for reopen. All changes are staged in the
+ * 'opaque' field of the BDRVReopenState, which is used and allocated by
+ * the block driver layer .bdrv_reopen_prepare()
+ *
+ * bs is the BlockDriverState to reopen
+ * flags are the new open flags
+ * queue is the reopen queue
+ *
+ * Returns 0 on success, non-zero on error.  On error errp will be set
+ * as well.
+ *
+ * On failure, bdrv_reopen_abort() will be called to clean up any data.
+ * It is the responsibility of the caller to then call the abort() or
+ * commit() for any other BDS that have been left in a prepare() state
+ *
+ */
+int bdrv_reopen_prepare(BDRVReopenState *reopen_state, BlockReopenQueue *queue,
+                        Error **errp)
+{
+    int ret = -1;
+    Error *local_err = NULL;
+    BlockDriver *drv;
+
+    assert(reopen_state != NULL);
+    assert(reopen_state->bs->drv != NULL);
+    drv = reopen_state->bs->drv;
+
+    /* if we are to stay read-only, do not allow permission change
+     * to r/w */
+    if (!(reopen_state->bs->open_flags & BDRV_O_ALLOW_RDWR) &&
+        reopen_state->flags & BDRV_O_RDWR) {
+        error_set(errp, QERR_DEVICE_IS_READ_ONLY,
+                  reopen_state->bs->device_name);
+        goto error;
+    }
+
+
+    ret = bdrv_flush(reopen_state->bs);
+    if (ret) {
+        error_set(errp, ERROR_CLASS_GENERIC_ERROR, "Error (%s) flushing drive",
+                  strerror(-ret));
+        goto error;
+    }
+
+    if (drv->bdrv_reopen_prepare) {
+        ret = drv->bdrv_reopen_prepare(reopen_state, queue, &local_err);
+        if (ret) {
+            if (local_err != NULL) {
+                error_propagate(errp, local_err);
+            } else {
+                error_setg(errp, "failed while preparing to reopen image '%s'",
+                           reopen_state->bs->filename);
+            }
+            goto error;
+        }
+    } else {
+        /* It is currently mandatory to have a bdrv_reopen_prepare()
+         * handler for each supported drv. */
+        error_set(errp, QERR_BLOCK_FORMAT_FEATURE_NOT_SUPPORTED,
+                  drv->format_name, reopen_state->bs->device_name,
+                 "reopening of file");
+        ret = -1;
+        goto error;
+    }
+
+    ret = 0;
+
+error:
+    return ret;
+}
+
+/*
+ * Takes the staged changes for the reopen from bdrv_reopen_prepare(), and
+ * makes them final by swapping the staging BlockDriverState contents into
+ * the active BlockDriverState contents.
+ */
+void bdrv_reopen_commit(BDRVReopenState *reopen_state)
+{
+    BlockDriver *drv;
+
+    assert(reopen_state != NULL);
+    drv = reopen_state->bs->drv;
+    assert(drv != NULL);
+
+    /* If there are any driver level actions to take */
+    if (drv->bdrv_reopen_commit) {
+        drv->bdrv_reopen_commit(reopen_state);
+    }
+
+    /* set BDS specific flags now */
+    reopen_state->bs->open_flags         = reopen_state->flags;
+    reopen_state->bs->enable_write_cache = !!(reopen_state->flags &
+                                              BDRV_O_CACHE_WB);
+    reopen_state->bs->read_only = !(reopen_state->flags & BDRV_O_RDWR);
+}
+
+/*
+ * Abort the reopen, and delete and free the staged changes in
+ * reopen_state
+ */
+void bdrv_reopen_abort(BDRVReopenState *reopen_state)
+{
+    BlockDriver *drv;
+
+    assert(reopen_state != NULL);
+    drv = reopen_state->bs->drv;
+    assert(drv != NULL);
+
+    if (drv->bdrv_reopen_abort) {
+        drv->bdrv_reopen_abort(reopen_state);
+    }
+}
+
+
+void bdrv_close(BlockDriverState *bs)
+{
+    if (bs->job) {
+        block_job_cancel_sync(bs->job);
+    }
+    bdrv_drain_all(); /* complete I/O */
+    bdrv_flush(bs);
+    bdrv_drain_all(); /* in case flush left pending I/O */
+    notifier_list_notify(&bs->close_notifiers, bs);
+
+    if (bs->drv) {
+        if (bs->backing_hd) {
+            bdrv_delete(bs->backing_hd);
+            bs->backing_hd = NULL;
+        }
+        bs->drv->bdrv_close(bs);
+        g_free(bs->opaque);
+#ifdef _WIN32
+        if (bs->is_temporary) {
+            unlink(bs->filename);
+        }
+#endif
+        bs->opaque = NULL;
+        bs->drv = NULL;
+        bs->copy_on_read = 0;
+        bs->backing_file[0] = '\0';
+        bs->backing_format[0] = '\0';
+        bs->total_sectors = 0;
+        bs->encrypted = 0;
+        bs->valid_key = 0;
+        bs->sg = 0;
+        bs->growable = 0;
+        QDECREF(bs->options);
+        bs->options = NULL;
+
+        if (bs->file != NULL) {
+            bdrv_delete(bs->file);
+            bs->file = NULL;
+        }
+    }
+
+    bdrv_dev_change_media_cb(bs, false);
+
+    /*throttling disk I/O limits*/
+    if (bs->io_limits_enabled) {
+        bdrv_io_limits_disable(bs);
+    }
+}
+
+void bdrv_close_all(void)
+{
+    BlockDriverState *bs;
+
+    QTAILQ_FOREACH(bs, &bdrv_states, list) {
+        bdrv_close(bs);
+    }
+}
+
+/*
+ * Wait for pending requests to complete across all BlockDriverStates
+ *
+ * This function does not flush data to disk, use bdrv_flush_all() for that
+ * after calling this function.
+ *
+ * Note that completion of an asynchronous I/O operation can trigger any
+ * number of other I/O operations on other devices---for example a coroutine
+ * can be arbitrarily complex and a constant flow of I/O can come until the
+ * coroutine is complete.  Because of this, it is not possible to have a
+ * function to drain a single device's I/O queue.
+ */
+void bdrv_drain_all(void)
+{
+    BlockDriverState *bs;
+    bool busy;
+
+    do {
+        busy = qemu_aio_wait();
+
+        /* FIXME: We do not have timer support here, so this is effectively
+         * a busy wait.
+         */
+        QTAILQ_FOREACH(bs, &bdrv_states, list) {
+            if (!qemu_co_queue_empty(&bs->throttled_reqs)) {
+                qemu_co_queue_restart_all(&bs->throttled_reqs);
+                busy = true;
+            }
+        }
+    } while (busy);
+
+    /* If requests are still pending there is a bug somewhere */
+    QTAILQ_FOREACH(bs, &bdrv_states, list) {
+        assert(QLIST_EMPTY(&bs->tracked_requests));
+        assert(qemu_co_queue_empty(&bs->throttled_reqs));
+    }
+}
+
+/* make a BlockDriverState anonymous by removing from bdrv_state list.
+   Also, NULL terminate the device_name to prevent double remove */
+void bdrv_make_anon(BlockDriverState *bs)
+{
+    if (bs->device_name[0] != '\0') {
+        QTAILQ_REMOVE(&bdrv_states, bs, list);
+    }
+    bs->device_name[0] = '\0';
+}
+
+static void bdrv_rebind(BlockDriverState *bs)
+{
+    if (bs->drv && bs->drv->bdrv_rebind) {
+        bs->drv->bdrv_rebind(bs);
+    }
+}
+
+static void bdrv_move_feature_fields(BlockDriverState *bs_dest,
+                                     BlockDriverState *bs_src)
+{
+    /* move some fields that need to stay attached to the device */
+    bs_dest->open_flags         = bs_src->open_flags;
+
+    /* dev info */
+    bs_dest->dev_ops            = bs_src->dev_ops;
+    bs_dest->dev_opaque         = bs_src->dev_opaque;
+    bs_dest->dev                = bs_src->dev;
+    bs_dest->buffer_alignment   = bs_src->buffer_alignment;
+    bs_dest->copy_on_read       = bs_src->copy_on_read;
+
+    bs_dest->enable_write_cache = bs_src->enable_write_cache;
+
+    /* i/o timing parameters */
+    bs_dest->slice_start        = bs_src->slice_start;
+    bs_dest->slice_end          = bs_src->slice_end;
+    bs_dest->slice_submitted    = bs_src->slice_submitted;
+    bs_dest->io_limits          = bs_src->io_limits;
+    bs_dest->throttled_reqs     = bs_src->throttled_reqs;
+    bs_dest->block_timer        = bs_src->block_timer;
+    bs_dest->io_limits_enabled  = bs_src->io_limits_enabled;
+
+    /* r/w error */
+    bs_dest->on_read_error      = bs_src->on_read_error;
+    bs_dest->on_write_error     = bs_src->on_write_error;
+
+    /* i/o status */
+    bs_dest->iostatus_enabled   = bs_src->iostatus_enabled;
+    bs_dest->iostatus           = bs_src->iostatus;
+
+    /* dirty bitmap */
+    bs_dest->dirty_bitmap       = bs_src->dirty_bitmap;
+
+    /* job */
+    bs_dest->in_use             = bs_src->in_use;
+    bs_dest->job                = bs_src->job;
+
+    /* keep the same entry in bdrv_states */
+    pstrcpy(bs_dest->device_name, sizeof(bs_dest->device_name),
+            bs_src->device_name);
+    bs_dest->list = bs_src->list;
+}
+
+/*
+ * Swap bs contents for two image chains while they are live,
+ * while keeping required fields on the BlockDriverState that is
+ * actually attached to a device.
+ *
+ * This will modify the BlockDriverState fields, and swap contents
+ * between bs_new and bs_old. Both bs_new and bs_old are modified.
+ *
+ * bs_new is required to be anonymous.
+ *
+ * This function does not create any image files.
+ */
+void bdrv_swap(BlockDriverState *bs_new, BlockDriverState *bs_old)
+{
+    BlockDriverState tmp;
+
+    /* bs_new must be anonymous and shouldn't have anything fancy enabled */
+    assert(bs_new->device_name[0] == '\0');
+    assert(bs_new->dirty_bitmap == NULL);
+    assert(bs_new->job == NULL);
+    assert(bs_new->dev == NULL);
+    assert(bs_new->in_use == 0);
+    assert(bs_new->io_limits_enabled == false);
+    assert(bs_new->block_timer == NULL);
+
+    tmp = *bs_new;
+    *bs_new = *bs_old;
+    *bs_old = tmp;
+
+    /* there are some fields that should not be swapped, move them back */
+    bdrv_move_feature_fields(&tmp, bs_old);
+    bdrv_move_feature_fields(bs_old, bs_new);
+    bdrv_move_feature_fields(bs_new, &tmp);
+
+    /* bs_new shouldn't be in bdrv_states even after the swap!  */
+    assert(bs_new->device_name[0] == '\0');
+
+    /* Check a few fields that should remain attached to the device */
+    assert(bs_new->dev == NULL);
+    assert(bs_new->job == NULL);
+    assert(bs_new->in_use == 0);
+    assert(bs_new->io_limits_enabled == false);
+    assert(bs_new->block_timer == NULL);
+
+    bdrv_rebind(bs_new);
+    bdrv_rebind(bs_old);
+}
+
+/*
+ * Add new bs contents at the top of an image chain while the chain is
+ * live, while keeping required fields on the top layer.
+ *
+ * This will modify the BlockDriverState fields, and swap contents
+ * between bs_new and bs_top. Both bs_new and bs_top are modified.
+ *
+ * bs_new is required to be anonymous.
+ *
+ * This function does not create any image files.
+ */
+void bdrv_append(BlockDriverState *bs_new, BlockDriverState *bs_top)
+{
+    bdrv_swap(bs_new, bs_top);
+
+    /* The contents of 'tmp' will become bs_top, as we are
+     * swapping bs_new and bs_top contents. */
+    bs_top->backing_hd = bs_new;
+    bs_top->open_flags &= ~BDRV_O_NO_BACKING;
+    pstrcpy(bs_top->backing_file, sizeof(bs_top->backing_file),
+            bs_new->filename);
+    pstrcpy(bs_top->backing_format, sizeof(bs_top->backing_format),
+            bs_new->drv ? bs_new->drv->format_name : "");
+}
+
+void bdrv_delete(BlockDriverState *bs)
+{
+    assert(!bs->dev);
+    assert(!bs->job);
+    assert(!bs->in_use);
+
+    /* remove from list, if necessary */
+    bdrv_make_anon(bs);
+
+    bdrv_close(bs);
+
+    g_free(bs);
+}
+
+int bdrv_attach_dev(BlockDriverState *bs, void *dev)
+/* TODO change to DeviceState *dev when all users are qdevified */
+{
+    if (bs->dev) {
+        return -EBUSY;
+    }
+    bs->dev = dev;
+    bdrv_iostatus_reset(bs);
+    return 0;
+}
+
+/* TODO qdevified devices don't use this, remove when devices are qdevified */
+void bdrv_attach_dev_nofail(BlockDriverState *bs, void *dev)
+{
+    if (bdrv_attach_dev(bs, dev) < 0) {
+        abort();
+    }
+}
+
+void bdrv_detach_dev(BlockDriverState *bs, void *dev)
+/* TODO change to DeviceState *dev when all users are qdevified */
+{
+    assert(bs->dev == dev);
+    bs->dev = NULL;
+    bs->dev_ops = NULL;
+    bs->dev_opaque = NULL;
+    bs->buffer_alignment = 512;
+}
+
+/* TODO change to return DeviceState * when all users are qdevified */
+void *bdrv_get_attached_dev(BlockDriverState *bs)
+{
+    return bs->dev;
+}
+
+void bdrv_set_dev_ops(BlockDriverState *bs, const BlockDevOps *ops,
+                      void *opaque)
+{
+    bs->dev_ops = ops;
+    bs->dev_opaque = opaque;
+}
+
+void bdrv_emit_qmp_error_event(const BlockDriverState *bdrv,
+                               enum MonitorEvent ev,
+                               BlockErrorAction action, bool is_read)
+{
+    QObject *data;
+    const char *action_str;
+
+    switch (action) {
+    case BDRV_ACTION_REPORT:
+        action_str = "report";
+        break;
+    case BDRV_ACTION_IGNORE:
+        action_str = "ignore";
+        break;
+    case BDRV_ACTION_STOP:
+        action_str = "stop";
+        break;
+    default:
+        abort();
+    }
+
+    data = qobject_from_jsonf("{ 'device': %s, 'action': %s, 'operation': %s }",
+                              bdrv->device_name,
+                              action_str,
+                              is_read ? "read" : "write");
+    monitor_protocol_event(ev, data);
+
+    qobject_decref(data);
+}
+
+static void bdrv_emit_qmp_eject_event(BlockDriverState *bs, bool ejected)
+{
+    QObject *data;
+
+    data = qobject_from_jsonf("{ 'device': %s, 'tray-open': %i }",
+                              bdrv_get_device_name(bs), ejected);
+    monitor_protocol_event(QEVENT_DEVICE_TRAY_MOVED, data);
+
+    qobject_decref(data);
+}
+
+static void bdrv_dev_change_media_cb(BlockDriverState *bs, bool load)
+{
+    if (bs->dev_ops && bs->dev_ops->change_media_cb) {
+        bool tray_was_closed = !bdrv_dev_is_tray_open(bs);
+        bs->dev_ops->change_media_cb(bs->dev_opaque, load);
+        if (tray_was_closed) {
+            /* tray open */
+            bdrv_emit_qmp_eject_event(bs, true);
+        }
+        if (load) {
+            /* tray close */
+            bdrv_emit_qmp_eject_event(bs, false);
+        }
+    }
+}
+
+bool bdrv_dev_has_removable_media(BlockDriverState *bs)
+{
+    return !bs->dev || (bs->dev_ops && bs->dev_ops->change_media_cb);
+}
+
+void bdrv_dev_eject_request(BlockDriverState *bs, bool force)
+{
+    if (bs->dev_ops && bs->dev_ops->eject_request_cb) {
+        bs->dev_ops->eject_request_cb(bs->dev_opaque, force);
+    }
+}
+
+bool bdrv_dev_is_tray_open(BlockDriverState *bs)
+{
+    if (bs->dev_ops && bs->dev_ops->is_tray_open) {
+        return bs->dev_ops->is_tray_open(bs->dev_opaque);
+    }
+    return false;
+}
+
+static void bdrv_dev_resize_cb(BlockDriverState *bs)
+{
+    if (bs->dev_ops && bs->dev_ops->resize_cb) {
+        bs->dev_ops->resize_cb(bs->dev_opaque);
+    }
+}
+
+bool bdrv_dev_is_medium_locked(BlockDriverState *bs)
+{
+    if (bs->dev_ops && bs->dev_ops->is_medium_locked) {
+        return bs->dev_ops->is_medium_locked(bs->dev_opaque);
+    }
+    return false;
+}
+
+/*
+ * Run consistency checks on an image
+ *
+ * Returns 0 if the check could be completed (it doesn't mean that the image is
+ * free of errors) or -errno when an internal error occurred. The results of the
+ * check are stored in res.
+ */
+int bdrv_check(BlockDriverState *bs, BdrvCheckResult *res, BdrvCheckMode fix)
+{
+    if (bs->drv->bdrv_check == NULL) {
+        return -ENOTSUP;
+    }
+
+    memset(res, 0, sizeof(*res));
+    return bs->drv->bdrv_check(bs, res, fix);
+}
+
+#define COMMIT_BUF_SECTORS 2048
+
+/* commit COW file into the raw image */
+int bdrv_commit(BlockDriverState *bs)
+{
+    BlockDriver *drv = bs->drv;
+    int64_t sector, total_sectors;
+    int n, ro, open_flags;
+    int ret = 0;
+    uint8_t *buf;
+    char filename[PATH_MAX];
+
+    if (!drv)
+        return -ENOMEDIUM;
+    
+    if (!bs->backing_hd) {
+        return -ENOTSUP;
+    }
+
+    if (bdrv_in_use(bs) || bdrv_in_use(bs->backing_hd)) {
+        return -EBUSY;
+    }
+
+    ro = bs->backing_hd->read_only;
+    /* Use pstrcpy (not strncpy): filename must be NUL-terminated. */
+    pstrcpy(filename, sizeof(filename), bs->backing_hd->filename);
+    open_flags =  bs->backing_hd->open_flags;
+
+    if (ro) {
+        if (bdrv_reopen(bs->backing_hd, open_flags | BDRV_O_RDWR, NULL)) {
+            return -EACCES;
+        }
+    }
+
+    total_sectors = bdrv_getlength(bs) >> BDRV_SECTOR_BITS;
+    buf = g_malloc(COMMIT_BUF_SECTORS * BDRV_SECTOR_SIZE);
+
+    for (sector = 0; sector < total_sectors; sector += n) {
+        if (bdrv_is_allocated(bs, sector, COMMIT_BUF_SECTORS, &n)) {
+
+            if (bdrv_read(bs, sector, buf, n) != 0) {
+                ret = -EIO;
+                goto ro_cleanup;
+            }
+
+            if (bdrv_write(bs->backing_hd, sector, buf, n) != 0) {
+                ret = -EIO;
+                goto ro_cleanup;
+            }
+        }
+    }
+
+    if (drv->bdrv_make_empty) {
+        ret = drv->bdrv_make_empty(bs);
+        bdrv_flush(bs);
+    }
+
+    /*
+     * Make sure all data we wrote to the backing device is actually
+     * stable on disk.
+     */
+    if (bs->backing_hd)
+        bdrv_flush(bs->backing_hd);
+
+ro_cleanup:
+    g_free(buf);
+
+    if (ro) {
+        /* ignoring error return here */
+        bdrv_reopen(bs->backing_hd, open_flags & ~BDRV_O_RDWR, NULL);
+    }
+
+    return ret;
+}
+
+int bdrv_commit_all(void)
+{
+    BlockDriverState *bs;
+
+    QTAILQ_FOREACH(bs, &bdrv_states, list) {
+        if (bs->drv && bs->backing_hd) {
+            int ret = bdrv_commit(bs);
+            if (ret < 0) {
+                return ret;
+            }
+        }
+    }
+    return 0;
+}
+
+/**
+ * Remove an active request from the tracked requests list
+ *
+ * This function should be called when a tracked request is completing.
+ */
+static void tracked_request_end(BdrvTrackedRequest *req)
+{
+    QLIST_REMOVE(req, list);
+    qemu_co_queue_restart_all(&req->wait_queue);
+}
+
+/**
+ * Add an active request to the tracked requests list
+ */
+static void tracked_request_begin(BdrvTrackedRequest *req,
+                                  BlockDriverState *bs,
+                                  int64_t sector_num,
+                                  int nb_sectors, bool is_write)
+{
+    *req = (BdrvTrackedRequest){
+        .bs = bs,
+        .sector_num = sector_num,
+        .nb_sectors = nb_sectors,
+        .is_write = is_write,
+        .co = qemu_coroutine_self(),
+    };
+
+    qemu_co_queue_init(&req->wait_queue);
+
+    QLIST_INSERT_HEAD(&bs->tracked_requests, req, list);
+}
+
+/**
+ * Round a region to cluster boundaries
+ */
+void bdrv_round_to_clusters(BlockDriverState *bs,
+                            int64_t sector_num, int nb_sectors,
+                            int64_t *cluster_sector_num,
+                            int *cluster_nb_sectors)
+{
+    BlockDriverInfo bdi;
+
+    if (bdrv_get_info(bs, &bdi) < 0 || bdi.cluster_size == 0) {
+        *cluster_sector_num = sector_num;
+        *cluster_nb_sectors = nb_sectors;
+    } else {
+        int64_t c = bdi.cluster_size / BDRV_SECTOR_SIZE;
+        *cluster_sector_num = QEMU_ALIGN_DOWN(sector_num, c);
+        *cluster_nb_sectors = QEMU_ALIGN_UP(sector_num - *cluster_sector_num +
+                                            nb_sectors, c);
+    }
+}
+
+static bool tracked_request_overlaps(BdrvTrackedRequest *req,
+                                     int64_t sector_num, int nb_sectors) {
+    /*        aaaa   bbbb */
+    if (sector_num >= req->sector_num + req->nb_sectors) {
+        return false;
+    }
+    /* bbbb   aaaa        */
+    if (req->sector_num >= sector_num + nb_sectors) {
+        return false;
+    }
+    return true;
+}
+
+static void coroutine_fn wait_for_overlapping_requests(BlockDriverState *bs,
+        int64_t sector_num, int nb_sectors)
+{
+    BdrvTrackedRequest *req;
+    int64_t cluster_sector_num;
+    int cluster_nb_sectors;
+    bool retry;
+
+    /* If we touch the same cluster it counts as an overlap.  This guarantees
+     * that allocating writes will be serialized and not race with each other
+     * for the same cluster.  For example, in copy-on-read it ensures that the
+     * CoR read and write operations are atomic and guest writes cannot
+     * interleave between them.
+     */
+    bdrv_round_to_clusters(bs, sector_num, nb_sectors,
+                           &cluster_sector_num, &cluster_nb_sectors);
+
+    do {
+        retry = false;
+        QLIST_FOREACH(req, &bs->tracked_requests, list) {
+            if (tracked_request_overlaps(req, cluster_sector_num,
+                                         cluster_nb_sectors)) {
+                /* Hitting this means there was a reentrant request, for
+                 * example, a block driver issuing nested requests.  This must
+                 * never happen since it means deadlock.
+                 */
+                assert(qemu_coroutine_self() != req->co);
+
+                qemu_co_queue_wait(&req->wait_queue);
+                retry = true;
+                break;
+            }
+        }
+    } while (retry);
+}
+
+/*
+ * Return values:
+ * 0        - success
+ * -EINVAL  - backing format specified, but no file
+ * -ENOSPC  - can't update the backing file because no space is left in the
+ *            image file header
+ * -ENOTSUP - format driver doesn't support changing the backing file
+ */
+int bdrv_change_backing_file(BlockDriverState *bs,
+    const char *backing_file, const char *backing_fmt)
+{
+    BlockDriver *drv = bs->drv;
+    int ret;
+
+    /* Backing file format doesn't make sense without a backing file */
+    if (backing_fmt && !backing_file) {
+        return -EINVAL;
+    }
+
+    if (drv->bdrv_change_backing_file != NULL) {
+        ret = drv->bdrv_change_backing_file(bs, backing_file, backing_fmt);
+    } else {
+        ret = -ENOTSUP;
+    }
+
+    if (ret == 0) {
+        pstrcpy(bs->backing_file, sizeof(bs->backing_file), backing_file ?: "");
+        pstrcpy(bs->backing_format, sizeof(bs->backing_format), backing_fmt ?: "");
+    }
+    return ret;
+}
+
+/*
+ * Finds the image layer in the chain that has 'bs' as its backing file.
+ *
+ * active is the current topmost image.
+ *
+ * Returns NULL if bs is not found in active's image chain,
+ * or if active == bs.
+ */
+BlockDriverState *bdrv_find_overlay(BlockDriverState *active,
+                                    BlockDriverState *bs)
+{
+    BlockDriverState *overlay = NULL;
+    BlockDriverState *intermediate;
+
+    assert(active != NULL);
+    assert(bs != NULL);
+
+    /* if bs is the same as active, then by definition it has no overlay
+     */
+    if (active == bs) {
+        return NULL;
+    }
+
+    intermediate = active;
+    while (intermediate->backing_hd) {
+        if (intermediate->backing_hd == bs) {
+            overlay = intermediate;
+            break;
+        }
+        intermediate = intermediate->backing_hd;
+    }
+
+    return overlay;
+}
+
+typedef struct BlkIntermediateStates {
+    BlockDriverState *bs;
+    QSIMPLEQ_ENTRY(BlkIntermediateStates) entry;
+} BlkIntermediateStates;
+
+
+/*
+ * Drops images above 'base' up to and including 'top', and sets the image
+ * above 'top' to have base as its backing file.
+ *
+ * Requires that the overlay to 'top' is opened r/w, so that the backing file
+ * information in 'bs' can be properly updated.
+ *
+ * E.g., this will convert the following chain:
+ * bottom <- base <- intermediate <- top <- active
+ *
+ * to
+ *
+ * bottom <- base <- active
+ *
+ * It is allowed for bottom==base, in which case it converts:
+ *
+ * base <- intermediate <- top <- active
+ *
+ * to
+ *
+ * base <- active
+ *
+ * Error conditions:
+ *  if active == top, that is considered an error
+ *
+ */
+int bdrv_drop_intermediate(BlockDriverState *active, BlockDriverState *top,
+                           BlockDriverState *base)
+{
+    BlockDriverState *intermediate;
+    BlockDriverState *base_bs = NULL;
+    BlockDriverState *new_top_bs = NULL;
+    BlkIntermediateStates *intermediate_state, *next;
+    int ret = -EIO;
+
+    QSIMPLEQ_HEAD(states_to_delete, BlkIntermediateStates) states_to_delete;
+    QSIMPLEQ_INIT(&states_to_delete);
+
+    if (!top->drv || !base->drv) {
+        goto exit;
+    }
+
+    new_top_bs = bdrv_find_overlay(active, top);
+
+    if (new_top_bs == NULL) {
+        /* we could not find the image above 'top', this is an error */
+        goto exit;
+    }
+
+    /* special case of new_top_bs->backing_hd already pointing to base - nothing
+     * to do, no intermediate images */
+    if (new_top_bs->backing_hd == base) {
+        ret = 0;
+        goto exit;
+    }
+
+    intermediate = top;
+
+    /* now we will go down through the list, and add each BDS we find
+     * into our deletion queue, until we hit the 'base'
+     */
+    while (intermediate) {
+        intermediate_state = g_malloc0(sizeof(BlkIntermediateStates));
+        intermediate_state->bs = intermediate;
+        QSIMPLEQ_INSERT_TAIL(&states_to_delete, intermediate_state, entry);
+
+        if (intermediate->backing_hd == base) {
+            base_bs = intermediate->backing_hd;
+            break;
+        }
+        intermediate = intermediate->backing_hd;
+    }
+    if (base_bs == NULL) {
+        /* something went wrong, we did not end at the base. safely
+         * unravel everything, and exit with error */
+        goto exit;
+    }

path: root/rpc/xdr/src/xdr-nfs3.c

/*
  Copyright (c) 2010-2011 Gluster, Inc. <http://www.gluster.com>
  This file is part of GlusterFS.

  GlusterFS is free software; you can redistribute it and/or modify
  it under the terms of the GNU General Public License as published
  by the Free Software Foundation; either version 3 of the License,
  or (at your option) any later version.

  GlusterFS is distributed in the hope that it will be useful, but
  WITHOUT ANY WARRANTY; without even the implied warranty of
  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
  General Public License for more details.

  You should have received a copy of the GNU General Public License
  along with this program.  If not, see
  <http://www.gnu.org/licenses/>.
*/

#include "xdr-nfs3.h"
#include "mem-pool.h"
#include "xdr-common.h"

#if GF_DARWIN_HOST_OS
#define xdr_u_quad_t xdr_u_int64_t
#define xdr_quad_t   xdr_int64_t
#define xdr_uint32_t xdr_u_int32_t
#define xdr_uint64_t xdr_u_int64_t
#endif

bool_t
xdr_uint64 (XDR *xdrs, uint64 *objp)
{
	 if (!xdr_uint64_t (xdrs, objp))
		 return FALSE;
	return TRUE;
}

bool_t
xdr_int64 (XDR *xdrs, int64 *objp)
{
	 if (!xdr_int64_t (xdrs, objp))
		 return FALSE;
	return TRUE;
}

bool_t
xdr_uint32 (XDR *xdrs, uint32 *objp)
{
	 if (!xdr_uint32_t (xdrs, objp))
		 return FALSE;
	return TRUE;
}

bool_t
xdr_int32 (XDR *xdrs, int32 *objp)
{
	 if (!xdr_int32_t (xdrs, objp))
		 return FALSE;
	return TRUE;
}

bool_t
xdr_filename3 (XDR *xdrs, filename3 *objp)
{
	 if (!xdr_string (xdrs, objp, ~0))
		 return FALSE;
	return TRUE;
}

bool_t
xdr_nfspath3 (XDR *xdrs, nfspath3 *objp)
{
	 if (!xdr_string (xdrs, objp, ~0))
		 return FALSE;
	return TRUE;
}

bool_t
xdr_fileid3 (XDR *xdrs, fileid3 *objp)
{
	 if (!xdr_uint64 (xdrs, objp))
		 return FALSE;
	return TRUE;
}

bool_t
xdr_cookie3 (XDR *xdrs, cookie3 *objp)
{
	 if (!xdr_uint64 (xdrs, objp))
		 return FALSE;
	return TRUE;
}

bool_t
xdr_cookieverf3 (XDR *xdrs, cookieverf3 objp)
{
	 if (!xdr_opaque (xdrs, objp, NFS3_COOKIEVERFSIZE))
		 return FALSE;
	return TRUE;
}

bool_t
xdr_createverf3 (XDR *xdrs, createverf3 objp)
{
	 if (!xdr_opaque (xdrs, objp, NFS3_CREATEVERFSIZE))
		 return FALSE;
	return TRUE;
}

bool_t
xdr_writeverf3 (XDR *xdrs, writeverf3 objp)
{
	 if (!xdr_opaque (xdrs, objp, NFS3_WRITEVERFSIZE))
		 return FALSE;
	return TRUE;
}

bool_t
xdr_uid3 (XDR *xdrs, uid3 *objp)
{
	 if (!xdr_uint32 (xdrs, objp))
		 return FALSE;
	return TRUE;
}

bool_t
xdr_gid3 (XDR *xdrs, gid3 *objp)
{
	 if (!xdr_uint32 (xdrs, objp))
		 return FALSE;
	return TRUE;
}

bool_t
xdr_size3 (XDR *xdrs, size3 *objp)
{
	 if (!xdr_uint64 (xdrs, objp))
		 return FALSE;
	return TRUE;
}

bool_t
xdr_offset3 (XDR *xdrs, offset3 *objp)
{
	 if (!xdr_uint64 (xdrs, objp))
		 return FALSE;
	return TRUE;
}

bool_t
xdr_mode3 (XDR *xdrs, mode3 *objp)
{
	 if (!xdr_uint32 (xdrs, objp))
		 return FALSE;
	return TRUE;
}

bool_t
xdr_count3 (XDR *xdrs, count3 *objp)
{
	 if (!xdr_uint32 (xdrs, objp))
		 return FALSE;
	return TRUE;
}

bool_t
xdr_nfsstat3 (XDR *xdrs, nfsstat3 *objp)
{
	 if (!xdr_enum (xdrs, (enum_t *) objp))
		 return FALSE;
	return TRUE;
}

bool_t
xdr_ftype3 (XDR *xdrs, ftype3 *objp)
{
	 if (!xdr_enum (xdrs, (enum_t *) objp))
		 return FALSE;
	return TRUE;
}

bool_t
xdr_specdata3 (XDR *xdrs, specdata3 *objp)
{
	 if (!xdr_uint32 (xdrs, &objp->specdata1))
		 return FALSE;
	 if (!xdr_uint32 (xdrs, &objp->specdata2))
		 return FALSE;
	return TRUE;
}

bool_t
xdr_nfs_fh3 (XDR *xdrs, nfs_fh3 *objp)
{
	 if (!xdr_bytes (xdrs, (char **)&objp->data.data_val, (u_int *) &objp->data.data_len, NFS3_FHSIZE))
		 return FALSE;
	return TRUE;
}

bool_t
xdr_nfstime3 (XDR *xdrs, nfstime3 *objp)
{
	 if (!xdr_uint32 (xdrs, &objp->seconds))
		 return FALSE;
	 if (!xdr_uint32 (xdrs, &objp->nseconds))
		 return FALSE;
	return TRUE;
}

bool_t
xdr_fattr3 (XDR *xdrs, fattr3 *objp)
{
	 if (!xdr_ftype3 (xdrs, &objp->type))
		 return FALSE;
	 if (!xdr_mode3 (xdrs, &objp->mode))
		 return FALSE;
	 if (!xdr_uint32 (xdrs, &objp->nlink))
		 return FALSE;
	 if (!xdr_uid3 (xdrs, &objp->uid))
		 return FALSE;
	 if (!xdr_gid3 (xdrs, &objp->gid))
		 return FALSE;
	 if (!xdr_size3 (xdrs, &objp->size))
		 return FALSE;
	 if (!xdr_size3 (xdrs, &objp->used))
		 return FALSE;
	 if (!xdr_specdata3 (xdrs, &objp->rdev))
		 return FALSE;
	 if (!xdr_uint64 (xdrs, &objp->fsid))
		 return FALSE;
	 if (!xdr_fileid3 (xdrs, &objp->fileid))
		 return FALSE;
	 if (!xdr_nfstime3 (xdrs, &objp->atime))
		 return FALSE;
	 if (!xdr_nfstime3 (xdrs, &objp->mtime))
		 return FALSE;
	 if (!xdr_nfstime3 (xdrs, &objp->ctime))
		 return FALSE;
	return TRUE;
}

bool_t
xdr_post_op_attr (XDR *xdrs, post_op_attr *objp)
{
	 if (!xdr_bool (xdrs, &objp->attributes_follow))
		 return FALSE;
	switch (objp->attributes_follow) {
	case TRUE:
		 if (!xdr_fattr3 (xdrs, &objp->post_op_attr_u.attributes))
			 return FALSE;
		break;
	case FALSE:
		break;
	default:
		return FALSE;
	}
	return TRUE;
}

bool_t
xdr_wcc_attr (XDR *xdrs, wcc_attr *objp)
{
	 if (!xdr_size3 (xdrs, &objp->size))
		 return FALSE;
	 if (!xdr_nfstime3 (xdrs, &objp->mtime))
		 return FALSE;
	 if (!xdr_nfstime3 (xdrs, &objp->ctime))
		 return FALSE;
	return TRUE;
}

bool_t
xdr_pre_op_attr (XDR *xdrs, pre_op_attr *objp)
{
	 if (!xdr_bool (xdrs, &objp->attributes_follow))
		 return FALSE;
	switch (objp->attributes_follow) {
	case TRUE:
		 if (!xdr_wcc_attr (xdrs, &objp->pre_op_attr_u.attributes))
			 return FALSE;
		break;
	case FALSE:
		break;
	default:
		return FALSE;
	}
	return TRUE;
}

bool_t
xdr_wcc_data (XDR *xdrs, wcc_data *objp)
{
	 if (!xdr_pre_op_attr (xdrs, &objp->before))
		 return FALSE;
	 if (!xdr_post_op_attr (xdrs, &objp->after))
		 return FALSE;
	return TRUE;
}

bool_t
xdr_post_op_fh3 (XDR *xdrs, post_op_fh3 *objp)
{
	 if (!xdr_bool (xdrs, &objp->handle_follows))
		 return FALSE;
	switch (objp->handle_follows) {
	case TRUE:
		 if (!xdr_nfs_fh3 (xdrs, &objp->post_op_fh3_u.handle))
			 return FALSE;
		break;
	case FALSE:
		break;
	default:
		return FALSE;
	}
	return TRUE;
}

bool_t
xdr_time_how (XDR *xdrs, time_how *objp)
{
	 if (!xdr_enum (xdrs, (enum_t *) objp))
		 return FALSE;
	return TRUE;
}

bool_t
xdr_set_mode3 (XDR *xdrs, set_mode3 *objp)
{
	 if (!xdr_bool (xdrs, &objp->set_it))
		 return FALSE;
	switch (objp->set_it) {
	case TRUE:
		 if (!xdr_mode3 (xdrs, &objp->set_mode3_u.mode))
			 return FALSE;
		break;
	default:
		break;
	}
	return TRUE;
}

bool_t
xdr_set_uid3 (XDR *xdrs, set_uid3 *objp)
{
	 if (!xdr_bool (xdrs, &objp->set_it))
		 return FALSE;
	switch (objp->set_it) {
	case TRUE:
		 if (!xdr_uid3 (xdrs, &objp->set_uid3_u.uid))
			 return FALSE;
		break;
	default:
		break;
	}
	return TRUE;
}

bool_t
xdr_set_gid3 (XDR *xdrs, set_gid3 *objp)
{
	 if (!xdr_bool (xdrs, &objp->set_it))
		 return FALSE;
	switch (objp->set_it) {
	case TRUE:
		 if (!xdr_gid3 (xdrs, &objp->set_gid3_u.gid))
			 return FALSE;
		break;
	default:
		break;
	}
	return TRUE;
}

bool_t
xdr_set_size3 (XDR *xdrs, set_size3 *objp)
{
	 if (!xdr_bool (xdrs, &objp->set_it))
		 return FALSE;
	switch (objp->set_it) {
	case TRUE:
		 if (!xdr_size3 (xdrs, &objp->set_size3_u.size))
			 return FALSE;
		break;
	default:
		break;
	}
	return TRUE;
}

bool_t
xdr_set_atime (XDR *xdrs, set_atime *objp)
{
	 if (!xdr_time_how (xdrs, &objp->set_it))
		 return FALSE;
	switch (objp->set_it) {
	case SET_TO_CLIENT_TIME:
		 if (!xdr_nfstime3 (xdrs, &objp->set_atime_u.atime))
			 return FALSE;
		break;
	default:
		break;
	}
	return TRUE;
}

bool_t
xdr_set_mtime (XDR *xdrs, set_mtime *objp)
{
	 if (!xdr_time_how (xdrs, &objp->set_it))
		 return FALSE;
	switch (objp->set_it) {
	case SET_TO_CLIENT_TIME:
		 if (!xdr_nfstime3 (xdrs, &objp->set_mtime_u.mtime))
			 return FALSE;
		break;
	default:
		break;
	}
	return TRUE;
}

bool_t
xdr_sattr3 (XDR *xdrs, sattr3 *objp)
{
	 if (!xdr_set_mode3 (xdrs, &objp->mode))
		 return FALSE;
	 if (!xdr_set_uid3 (xdrs, &objp->uid))
		 return FALSE;
	 if (!xdr_set_gid3 (xdrs, &objp->gid))
		 return FALSE;
	 if (!xdr_set_size3 (xdrs, &objp->size))
		 return FALSE;
	 if (!xdr_set_atime (xdrs, &objp->atime))
		 return FALSE;
	 if (!xdr_set_mtime (xdrs, &objp->mtime))
		 return FALSE;
	return TRUE;
}

bool_t
xdr_diropargs3 (XDR *xdrs, diropargs3 *objp)
{
	 if (!xdr_nfs_fh3 (xdrs, &objp->dir))
		 return FALSE;
	 if (!xdr_filename3 (xdrs, &objp->name))
		 return FALSE;
	return TRUE;
}

bool_t
xdr_getattr3args (XDR *xdrs, getattr3args *objp)
{
	 if (!xdr_nfs_fh3 (xdrs, &objp->object))
		 return FALSE;
	return TRUE;
}

bool_t
xdr_getattr3resok (XDR *xdrs, getattr3resok *objp)
{
	 if (!xdr_fattr3 (xdrs, &objp->obj_attributes))
		 return FALSE;
	return TRUE;
}

bool_t
xdr_getattr3res (XDR *xdrs, getattr3res *objp)
{
	 if (!xdr_nfsstat3 (xdrs, &objp->status))
		 return FALSE;
	switch (objp->status) {
	case NFS3_OK:
		 if (!xdr_getattr3resok (xdrs, &objp->getattr3res_u.resok))
			 return FALSE;
		break;
	default:
		break;
	}
	return TRUE;
}

bool_t
xdr_sattrguard3 (XDR *xdrs, sattrguard3 *objp)
{
	 if (!xdr_bool (xdrs, &objp->check))
		 return FALSE;
	switch (objp->check) {
	case TRUE:
		 if (!xdr_nfstime3 (xdrs, &objp->sattrguard3_u.obj_ctime))
			 return FALSE;
		break;
	case FALSE:
		break;
	default:
		return FALSE;
	}
	return TRUE;
}

bool_t
xdr_setattr3args (XDR *xdrs, setattr3args *objp)
{
	 if (!xdr_nfs_fh3 (xdrs, &objp->object))
		 return FALSE;
	 if (!xdr_sattr3 (xdrs, &objp->new_attributes))
		 return FALSE;
	 if (!xdr_sattrguard3 (xdrs, &objp->guard))
		 return FALSE;
	return TRUE;
}

bool_t
xdr_setattr3resok (XDR *xdrs, setattr3resok *objp)
{
	 if (!xdr_wcc_data (xdrs, &objp->obj_wcc))
		 return FALSE;
	return TRUE;
}

bool_t
xdr_setattr3resfail (XDR *xdrs, setattr3resfail *objp)
{
	 if (!xdr_wcc_data (xdrs, &objp->obj_wcc))
		 return FALSE;
	return TRUE;
}

bool_t
xdr_setattr3res (XDR *xdrs, setattr3res *objp)
{
	 if (!xdr_nfsstat3 (xdrs, &objp->status))
		 return FALSE;
	switch (objp->status) {
	case NFS3_OK:
		 if (!xdr_setattr3resok (xdrs, &objp->setattr3res_u.resok))
			 return FALSE;
		break;
	default:
		 if (!xdr_setattr3resfail (xdrs, &objp->setattr3res_u.resfail))
			 return FALSE;
		break;
	}
	return TRUE;
}

bool_t
xdr_lookup3args (XDR *xdrs, lookup3args *objp)
{
	 if (!xdr_diropargs3 (xdrs, &objp->what))
		 return FALSE;
	return TRUE;
}

bool_t
xdr_lookup3resok (XDR *xdrs, lookup3resok *objp)
{
	 if (!xdr_nfs_fh3 (xdrs, &objp->object))
		 return FALSE;
	 if (!xdr_post_op_attr (xdrs, &objp->obj_attributes))
		 return FALSE;
	 if (!xdr_post_op_attr (xdrs, &objp->dir_attributes))
		 return FALSE;
	return TRUE;
}

bool_t
xdr_lookup3resfail (XDR *xdrs, lookup3resfail *objp)
{
	 if (!xdr_post_op_attr (xdrs, &objp->dir_attributes))
		 return FALSE;
	return TRUE;
}

bool_t
xdr_lookup3res (XDR *xdrs, lookup3res *objp)
{
	 if (!xdr_nfsstat3 (xdrs, &objp->status))
		 return FALSE;
	switch (objp->status) {
	case NFS3_OK:
		 if (!xdr_lookup3resok (xdrs, &objp->lookup3res_u.resok))
			 return FALSE;
		break;
	default:
		 if (!xdr_lookup3resfail (xdrs, &objp->lookup3res_u.resfail))
			 return FALSE;
		break;
	}
	return TRUE;
}

bool_t
xdr_access3args (XDR *xdrs, access3args *objp)
{
	 if (!xdr_nfs_fh3 (xdrs, &objp->object))
		 return FALSE;
	 if (!xdr_uint32 (xdrs, &objp->access))
		 return FALSE;
	return TRUE;
}

bool_t
xdr_access3resok (XDR *xdrs, access3resok *objp)
{
	 if (!xdr_post_op_attr (xdrs, &objp->obj_attributes))
		 return FALSE;
	 if (!xdr_uint32 (xdrs, &objp->access))
		 return FALSE;
	return TRUE;
}

bool_t
xdr_access3resfail (XDR *xdrs, access3resfail *objp)
{
	 if (!xdr_post_op_attr (xdrs, &objp->obj_attributes))
		 return FALSE;
	return TRUE;
}

bool_t
xdr_access3res (XDR *xdrs, access3res *objp)
{
	 if (!xdr_nfsstat3 (xdrs, &objp->status))
		 return FALSE;
	switch (objp->status) {
	case NFS3_OK:
		 if (!xdr_access3resok (xdrs, &objp->access3res_u.resok))
			 return FALSE;
		break;
	default:
		 if (!xdr_access3resfail (xdrs, &objp->access3res_u.resfail))
			 return FALSE;
		break;
	}
	return TRUE;
}

bool_t
xdr_readlink3args (XDR *xdrs, readlink3args *objp)
{
	 if (!xdr_nfs_fh3 (xdrs, &objp->symlink))
		 return FALSE;
	return TRUE;
}

bool_t
xdr_readlink3resok (XDR *xdrs, readlink3resok *objp)
{
	 if (!xdr_post_op_attr (xdrs, &objp->symlink_attributes))
		 return FALSE;
	 if (!xdr_nfspath3 (xdrs, &objp->data))
		 return FALSE;
	return TRUE;
}

bool_t
xdr_readlink3resfail (XDR *xdrs, readlink3resfail *objp)
{
	 if (!xdr_post_op_attr (xdrs, &objp->symlink_attributes))
		 return FALSE;
	return TRUE;
}

bool_t
xdr_readlink3res (XDR *xdrs, readlink3res *objp)
{
	 if (!xdr_nfsstat3 (xdrs, &objp->status))
		 return FALSE;
	switch (objp->status) {
	case NFS3_OK:
		 if (!xdr_readlink3resok (xdrs, &objp->readlink3res_u.resok))
			 return FALSE;
		break;
	default:
		 if (!xdr_readlink3resfail (xdrs, &objp->readlink3res_u.resfail))
			 return FALSE;
		break;
	}
	return TRUE;
}

bool_t
xdr_read3args (XDR *xdrs, read3args *objp)
{
	 if (!xdr_nfs_fh3 (xdrs, &objp->file))
		 return FALSE;
	 if (!xdr_offset3 (xdrs, &objp->offset))
		 return FALSE;
	 if (!xdr_count3 (xdrs, &objp->count))
		 return FALSE;
	return TRUE;
}

bool_t
xdr_read3resok_nocopy (XDR *xdrs, read3resok *objp)
{
	 if (!xdr_post_op_attr (xdrs, &objp->file_attributes))
		 return FALSE;
	 if (!xdr_count3 (xdrs, &objp->count))
		 return FALSE;
	 if (!xdr_bool (xdrs, &objp->eof))
		 return FALSE;
         if (!xdr_u_int (xdrs, (u_int *) &objp->data.data_len))
                 return FALSE;
	return TRUE;
}


bool_t
xdr_read3resok (XDR *xdrs, read3resok *objp)
{
	 if (!xdr_post_op_attr (xdrs, &objp->file_attributes))
		 return FALSE;
	 if (!xdr_count3 (xdrs, &objp->count))
		 return FALSE;
	 if (!xdr_bool (xdrs, &objp->eof))
		 return FALSE;
	 if (!xdr_bytes (xdrs, (char **)&objp->data.data_val, (u_int *) &objp->data.data_len, ~0))
		 return FALSE;
	return TRUE;
}

bool_t
xdr_read3resfail (XDR *xdrs, read3resfail *objp)
{
	 if (!xdr_post_op_attr (xdrs, &objp->file_attributes))
		 return FALSE;
	return TRUE;
}


bool_t
xdr_read3res_nocopy (XDR *xdrs, read3res *objp)
{
	 if (!xdr_nfsstat3 (xdrs, &objp->status))
		 return FALSE;
	switch (objp->status) {
	case NFS3_OK:
		 if (!xdr_read3resok_nocopy (xdrs, &objp->read3res_u.resok))
			 return FALSE;
		break;
	default:
		 if (!xdr_read3resfail (xdrs, &objp->read3res_u.resfail))
			 return FALSE;
		break;
	}
	return TRUE;
}


bool_t
xdr_read3res (XDR *xdrs, read3res *objp)
{
	 if (!xdr_nfsstat3 (xdrs, &objp->status))
		 return FALSE;
	switch (objp->status) {
	case NFS3_OK:
		 if (!xdr_read3resok (xdrs, &objp->read3res_u.resok))
			 return FALSE;
		break;
	default:
		 if (!xdr_read3resfail (xdrs, &objp->read3res_u.resfail))
			 return FALSE;
		break;
	}
	return TRUE;
}

bool_t
xdr_stable_how (XDR *xdrs, stable_how *objp)
{
	 if (!xdr_enum (xdrs, (enum_t *) objp))
		 return FALSE;
	return TRUE;
}

bool_t
xdr_write3args (XDR *xdrs, write3args *objp)
{
	 if (!xdr_nfs_fh3 (xdrs, &objp->file))
		 return FALSE;
	 if (!xdr_offset3 (xdrs, &objp->offset))
		 return FALSE;
	 if (!xdr_count3 (xdrs, &objp->count))
		 return FALSE;
	 if (!xdr_stable_how (xdrs, &objp->stable))
		 return FALSE;

         /* Added specifically to avoid copies from the xdr buffer into
          * the write3args structure, which will also require an already
          * allocated buffer. That is not optimal.
          */
         if (!xdr_u_int (xdrs, (u_int *) &objp->data.data_len))
                 return FALSE;

         /* The remaining bytes in the xdr buffer are the bytes that need to be
          * written. See how these bytes are extracted in the xdr_to_write3args
          * code path. Be careful, while using the write3args structure, since
          * only the data.data_len has been filled. The actual data is
          * extracted in xdr_to_write3args path.
          */

         /*	 if (!xdr_bytes (xdrs, (char **)&objp->data.data_val, (u_int *) &objp->data.data_len, ~0))
		 return FALSE;
                 */
	return TRUE;
}

bool_t
xdr_write3resok (XDR *xdrs, write3resok *objp)
{
	 if (!xdr_wcc_data (xdrs, &objp->file_wcc))
		 return FALSE;
	 if (!xdr_count3 (xdrs, &objp->count))
		 return FALSE;
	 if (!xdr_stable_how (xdrs, &objp->committed))
		 return FALSE;
	 if (!xdr_writeverf3 (xdrs, objp->verf))
		 return FALSE;
	return TRUE;
}

bool_t
xdr_write3resfail (XDR *xdrs, write3resfail *objp)
{
	 if (!xdr_wcc_data (xdrs, &objp->file_wcc))
		 return FALSE;
	return TRUE;
}

bool_t
xdr_write3res (XDR *xdrs, write3res *objp)
{
	 if (!xdr_nfsstat3 (xdrs, &objp->status))
		 return FALSE;
	switch (objp->status) {
	case NFS3_OK:
		 if (!xdr_write3resok (xdrs, &objp->write3res_u.resok))
			 return FALSE;
		break;
	default:
		 if (!xdr_write3resfail (xdrs, &objp->write3res_u.resfail))
			 return FALSE;
		break;
	}
	return TRUE;
}

bool_t
xdr_createmode3 (XDR *xdrs, createmode3 *objp)
{
	 if (!xdr_enum (xdrs, (enum_t *) objp))
		 return FALSE;
	return TRUE;
}

bool_t
xdr_createhow3 (XDR *xdrs, createhow3 *objp)
{
	 if (!xdr_createmode3 (xdrs, &objp->mode))
		 return FALSE;
	switch (objp->mode) {
	case UNCHECKED:
	case GUARDED:
		 if (!xdr_sattr3 (xdrs, &objp->createhow3_u.obj_attributes))
			 return FALSE;
		break;
	case EXCLUSIVE:
		 if (!xdr_createverf3 (xdrs, objp->createhow3_u.verf))
			 return FALSE;
		break;
	default:
		return FALSE;
	}
	return TRUE;
}

bool_t
xdr_create3args (XDR *xdrs, create3args *objp)
{
	 if (!xdr_diropargs3 (xdrs, &objp->where))
		 return FALSE;
	 if (!xdr_createhow3 (xdrs, &objp->how))
		 return FALSE;
	return TRUE;
}

bool_t
xdr_create3resok (XDR *xdrs, create3resok *objp)
{
	 if (!xdr_post_op_fh3 (xdrs, &objp->obj))
		 return FALSE;
	 if (!xdr_post_op_attr (xdrs, &objp->obj_attributes))
		 return FALSE;
	 if (!xdr_wcc_data (xdrs, &objp->dir_wcc))
		 return FALSE;
	return TRUE;
}

bool_t
xdr_create3resfail (XDR *xdrs, create3resfail *objp)
{
	 if (!xdr_wcc_data (xdrs, &objp->dir_wcc))
		 return FALSE;
	return TRUE;
}

bool_t
xdr_create3res (XDR *xdrs, create3res *objp)
{
	 if (!xdr_nfsstat3 (xdrs, &objp->status))
		 return FALSE;
	switch (objp->status) {
	case NFS3_OK:
		 if (!xdr_create3resok (xdrs, &objp->create3res_u.resok))
			 return FALSE;
		break;
	default:
		 if (!xdr_create3resfail (xdrs, &objp->create3res_u.resfail))
			 return FALSE;
		break;
	}
	return TRUE;
}

bool_t
xdr_mkdir3args (XDR *xdrs, mkdir3args *objp)
{
	 if (!xdr_diropargs3 (xdrs, &objp->where))
		 return FALSE;
	 if (!xdr_sattr3 (xdrs, &objp->attributes))
		 return FALSE;
	return TRUE;
}

bool_t
xdr_mkdir3resok (XDR *xdrs, mkdir3resok *objp)
{
	 if (!xdr_post_op_fh3 (xdrs, &objp->obj))
		 return FALSE;
	 if (!xdr_post_op_attr (xdrs, &objp->obj_attributes))
		 return FALSE;
	 if (!xdr_wcc_data (xdrs, &objp->dir_wcc))
		 return FALSE;
	return TRUE;
}

bool_t
xdr_mkdir3resfail (XDR *xdrs, mkdir3resfail *objp)
{
	 if (!xdr_wcc_data (xdrs, &objp->dir_wcc))
		 return FALSE;
	return TRUE;
}

bool_t
xdr_mkdir3res (XDR *xdrs, mkdir3res *objp)
{
	 if (!xdr_nfsstat3 (xdrs, &objp->status))
		 return FALSE;
	switch (objp->status) {
	case NFS3_OK:
		 if (!xdr_mkdir3resok (xdrs, &objp->mkdir3res_u.resok))
			 return FALSE;
		break;
	default:
		 if (!xdr_mkdir3resfail (xdrs, &objp->mkdir3res_u.resfail))
			 return FALSE;
		break;
	}
	return TRUE;
}

bool_t
xdr_symlinkdata3 (XDR *xdrs, symlinkdata3 *objp)
{
	 if (!xdr_sattr3 (xdrs, &objp->symlink_attributes))
		 return FALSE;
	 if (!xdr_nfspath3 (xdrs, &objp->symlink_data))
		 return FALSE;
	return TRUE;
}

bool_t
xdr_symlink3args (XDR *xdrs, symlink3args *objp)
{
	 if (!xdr_diropargs3 (xdrs, &objp->where))
		 return FALSE;
	 if (!xdr_symlinkdata3 (xdrs, &objp->symlink))
		 return FALSE;
	return TRUE;
}

bool_t
xdr_symlink3resok (XDR *xdrs, symlink3resok *objp)
{
	 if (!xdr_post_op_fh3 (xdrs, &objp->obj))
		 return FALSE;
	 if (!xdr_post_op_attr (xdrs, &objp->obj_attributes))
		 return FALSE;
	 if (!xdr_wcc_data (xdrs, &objp->dir_wcc))
		 return FALSE;
	return TRUE;
}

bool_t
xdr_symlink3resfail (XDR *xdrs, symlink3resfail *objp)
{
	 if (!xdr_wcc_data (xdrs, &objp->dir_wcc))
		 return FALSE;
	return TRUE;
}

bool_t
xdr_symlink3res (XDR *xdrs, symlink3res *objp)
{
	 if (!xdr_nfsstat3 (xdrs, &objp->status))
		 return FALSE;
	switch (objp->status) {
	case NFS3_OK:
		 if (!xdr_symlink3resok (xdrs, &objp->symlink3res_u.resok))
			 return FALSE;
		break;
	default:
		 if (!xdr_symlink3resfail (xdrs, &objp->symlink3res_u.resfail))
			 return FALSE;
		break;
	}
	return TRUE;
}

bool_t
xdr_devicedata3 (XDR *xdrs, devicedata3 *objp)
{
	 if (!xdr_sattr3 (xdrs, &objp->dev_attributes))
		 return FALSE;
	 if (!xdr_specdata3 (xdrs, &objp->spec))
		 return FALSE;
	return TRUE;
}

bool_t
xdr_mknoddata3 (XDR *xdrs, mknoddata3 *objp)
{
	 if (!xdr_ftype3 (xdrs, &objp->type))
		 return FALSE;
	switch (objp->type) {
	case NF3CHR:
	case NF3BLK:
		 if (!xdr_devicedata3 (xdrs, &objp->mknoddata3_u.device))
			 return FALSE;
		break;
	case NF3SOCK:
	case NF3FIFO:
		 if (!xdr_sattr3 (xdrs, &objp->mknoddata3_u.pipe_attributes))
			 return FALSE;
		break;
	default:
		break;
	}
	return TRUE;
}

bool_t
xdr_mknod3args (XDR *xdrs, mknod3args *objp)
{
	 if (!xdr_diropargs3 (xdrs, &objp->where))
		 return FALSE;
	 if (!xdr_mknoddata3 (xdrs, &objp->what))
		 return FALSE;
	return TRUE;
}

bool_t
xdr_mknod3resok (XDR *xdrs, mknod3resok *objp)
{
	 if (!xdr_post_op_fh3 (xdrs, &objp->obj))
		 return FALSE;
	 if (!xdr_post_op_attr (xdrs, &objp->obj_attributes))
		 return FALSE;
	 if (!xdr_wcc_data (xdrs, &objp->dir_wcc))
		 return FALSE;
	return TRUE;
}

bool_t
xdr_mknod3resfail (XDR *xdrs, mknod3resfail *objp)
{
	 if (!xdr_wcc_data (xdrs, &objp->dir_wcc))
		 return FALSE;
	return TRUE;
}

bool_t
xdr_mknod3res (XDR *xdrs, mknod3res *objp)
{
	 if (!xdr_nfsstat3 (xdrs, &objp->status))
		 return FALSE;
	switch (objp->status) {
	case NFS3_OK:
		 if (!xdr_mknod3resok (xdrs, &objp->mknod3res_u.resok))
			 return FALSE;
		break;
	default:
		 if (!xdr_mknod3resfail (xdrs, &objp->mknod3res_u.resfail))
			 return FALSE;
		break;
	}
	return TRUE;
}

bool_t
xdr_remove3args (XDR *xdrs, remove3args *objp)
{
	 if (!xdr_diropargs3 (xdrs, &objp->object))
		 return FALSE;
	return TRUE;
}

bool_t
xdr_remove3resok (XDR *xdrs, remove3resok *objp)
{
	 if (!xdr_wcc_data (xdrs, &objp->dir_wcc))
		 return FALSE;
	return TRUE;
}

bool_t
xdr_remove3resfail (XDR *xdrs, remove3resfail *objp)
{
	 if (!xdr_wcc_data (xdrs, &objp->dir_wcc))
		 return FALSE;
	return TRUE;
}

bool_t
xdr_remove3res (XDR *xdrs, remove3res *objp)
{
	 if (!xdr_nfsstat3 (xdrs, &objp->status))
		 return FALSE;
	switch (objp->status) {
	case NFS3_OK:
		 if (!xdr_remove3resok (xdrs, &objp->remove3res_u.resok))
			 return FALSE;
		break;
	default:
		 if (!xdr_remove3resfail (xdrs, &objp->remove3res_u.resfail))
			 return FALSE;
		break;
	}
	return TRUE;
}

bool_t
xdr_rmdir3args (XDR *xdrs, rmdir3args *objp)
{
	 if (!xdr_diropargs3 (xdrs, &objp->object))
		 return FALSE;
	return TRUE;
}

bool_t
xdr_rmdir3resok (XDR *xdrs, rmdir3resok *objp)
{
	 if (!xdr_wcc_data (xdrs, &objp->dir_wcc))
		 return FALSE;
	return TRUE;
}

bool_t
xdr_rmdir3resfail (XDR *xdrs, rmdir3resfail *objp)
{
	 if (!xdr_wcc_data (xdrs, &objp->dir_wcc))
		 return FALSE;
	return TRUE;
}

bool_t
xdr_rmdir3res (XDR *xdrs, rmdir3res *objp)
{
	 if (!xdr_nfsstat3 (xdrs, &objp->status))
		 return FALSE;
	switch (objp->status) {
	case NFS3_OK:
		 if (!xdr_rmdir3resok (xdrs, &objp->rmdir3res_u.resok))
			 return FALSE;
		break;
	default:
		 if (!xdr_rmdir3resfail (xdrs, &objp->rmdir3res_u.resfail))
			 return FALSE;
		break;
	}
	return TRUE;
}

bool_t
xdr_rename3args (XDR *xdrs, rename3args *objp)
{
	 if (!xdr_diropargs3 (xdrs, &objp->from))
		 return FALSE;
	 if (!xdr_diropargs3 (xdrs, &objp->to))
		 return FALSE;
	return TRUE;
}

bool_t
xdr_rename3resok (XDR *xdrs, rename3resok *objp)
{
	 if (!xdr_wcc_data (xdrs, &objp->fromdir_wcc))
		 return FALSE;
	 if (!xdr_wcc_data (xdrs, &objp->todir_wcc))
		 return FALSE;
	return TRUE;
}

bool_t
xdr_rename3resfail (XDR *xdrs, rename3resfail *objp)
{
	 if (!xdr_wcc_data (xdrs, &objp->fromdir_wcc))
		 return FALSE;
	 if (!xdr_wcc_data (xdrs, &objp->todir_wcc))
		 return FALSE;
	return TRUE;
}

bool_t
xdr_rename3res (XDR *xdrs, rename3res *objp)
{
	 if (!xdr_nfsstat3 (xdrs, &objp->status))
		 return FALSE;
	switch (objp->status) {
	case NFS3_OK:
		 if (!xdr_rename3resok (xdrs, &objp->rename3res_u.resok))
			 return FALSE;
		break;
	default:
		 if (!xdr_rename3resfail (xdrs, &objp->rename3res_u.resfail))
			 return FALSE;
		break;
	}
	return TRUE;
}

bool_t
xdr_link3args (XDR *xdrs, link3args *objp)
{
	 if (!xdr_nfs_fh3 (xdrs, &objp->file))
		 return FALSE;
	 if (!xdr_diropargs3 (xdrs, &objp->link))
		 return FALSE;
	return TRUE;
}

bool_t
xdr_link3resok (XDR *xdrs, link3resok *objp)
{
	 if (!xdr_post_op_attr (xdrs, &objp->file_attributes))
		 return FALSE;
	 if (!xdr_wcc_data (xdrs, &objp->linkdir_wcc))
		 return FALSE;
	return TRUE;
}

bool_t
xdr_link3resfail (XDR *xdrs, link3resfail *objp)
{
	 if (!xdr_post_op_attr (xdrs, &objp->file_attributes))
		 return FALSE;
	 if (!xdr_wcc_data (xdrs, &objp->linkdir_wcc))
		 return FALSE;
	return TRUE;
}

bool_t
xdr_link3res (XDR *xdrs, link3res *objp)
{
	 if (!xdr_nfsstat3 (xdrs, &objp->status))
		 return FALSE;
	switch (objp->status) {
	case NFS3_OK:
		 if (!xdr_link3resok (xdrs, &objp->link3res_u.resok))
			 return FALSE;
		break;
	default:
		 if (!xdr_link3resfail (xdrs, &objp->link3res_u.resfail))
			 return FALSE;
		break;
	}
	return TRUE;
}

bool_t
xdr_readdir3args (XDR *xdrs, readdir3args *objp)
{
	 if (!xdr_nfs_fh3 (xdrs, &objp->dir))
		 return FALSE;
	 if (!xdr_cookie3 (xdrs, &objp->cookie))
		 return FALSE;
	 if (!xdr_cookieverf3 (xdrs, objp->cookieverf))
		 return FALSE;
	 if (!xdr_count3 (xdrs, &objp->count))
		 return FALSE;
	return TRUE;
}

bool_t
xdr_entry3 (XDR *xdrs, entry3 *objp)
{
	 if (!xdr_fileid3 (xdrs, &objp->fileid))
		 return FALSE;
	 if (!xdr_filename3 (xdrs, &objp->name))
		 return FALSE;
	 if (!xdr_cookie3 (xdrs, &objp->cookie))
		 return FALSE;
	 if (!xdr_pointer (xdrs, (char **)&objp->nextentry, sizeof (entry3), (xdrproc_t) xdr_entry3))
		 return FALSE;
	return TRUE;
}

bool_t
xdr_dirlist3 (XDR *xdrs, dirlist3 *objp)
{
	 if (!xdr_pointer (xdrs, (char **)&objp->entries, sizeof (entry3), (xdrproc_t) xdr_entry3))
		 return FALSE;
	 if (!xdr_bool (xdrs, &objp->eof))
		 return FALSE;
	return TRUE;
}

bool_t
xdr_readdir3resok (XDR *xdrs, readdir3resok *objp)
{
	 if (!xdr_post_op_attr (xdrs, &objp->dir_attributes))
		 return FALSE;
	 if (!xdr_cookieverf3 (xdrs, objp->cookieverf))
		 return FALSE;
	 if (!xdr_dirlist3 (xdrs, &objp->reply))
		 return FALSE;
	return TRUE;
}

bool_t
xdr_readdir3resfail (XDR *xdrs, readdir3resfail *objp)
{
	 if (!xdr_post_op_attr (xdrs, &objp->dir_attributes))
		 return FALSE;
	return TRUE;
}

bool_t
xdr_readdir3res (XDR *xdrs, readdir3res *objp)
{
	 if (!xdr_nfsstat3 (xdrs, &objp->status))
		 return FALSE;
	switch (objp->status) {
	case NFS3_OK:
		 if (!xdr_readdir3resok (xdrs, &objp->readdir3res_u.resok))
			 return FALSE;
		break;
	default:
		 if (!xdr_readdir3resfail (xdrs, &objp->readdir3res_u.resfail))
			 return FALSE;
		break;
	}
	return TRUE;
}

bool_t
xdr_readdirp3args (XDR *xdrs, readdirp3args *objp)
{
	 if (!xdr_nfs_fh3 (xdrs, &objp->dir))
		 return FALSE;
	 if (!xdr_cookie3 (xdrs, &objp->cookie))
		 return FALSE;
	 if (!xdr_cookieverf3 (xdrs, objp->cookieverf))
		 return FALSE;
	 if (!xdr_count3 (xdrs, &objp->dircount))
		 return FALSE;
	 if (!xdr_count3 (xdrs, &objp->maxcount))
		 return FALSE;
	return TRUE;
}

bool_t
xdr_entryp3 (XDR *xdrs, entryp3 *objp)
{
	 if (!xdr_fileid3 (xdrs, &objp->fileid))
		 return FALSE;
	 if (!xdr_filename3 (xdrs, &objp->name))
		 return FALSE;
	 if (!xdr_cookie3 (xdrs, &objp->cookie))
		 return FALSE;
	 if (!xdr_post_op_attr (xdrs, &objp->name_attributes))
		 return FALSE;
	 if (!xdr_post_op_fh3 (xdrs, &objp->name_handle))
		 return FALSE;
	 if (!xdr_pointer (xdrs, (char **)&objp->nextentry, sizeof (entryp3), (xdrproc_t) xdr_entryp3))
		 return FALSE;
	return TRUE;
}

bool_t
xdr_dirlistp3 (XDR *xdrs, dirlistp3 *objp)
{
	 if (!xdr_pointer (xdrs, (char **)&objp->entries, sizeof (entryp3), (xdrproc_t) xdr_entryp3))
		 return FALSE;
	 if (!xdr_bool (xdrs, &objp->eof))
		 return FALSE;
	return TRUE;
}

bool_t
xdr_readdirp3resok (XDR *xdrs, readdirp3resok *objp)
{
	 if (!xdr_post_op_attr (xdrs, &objp->dir_attributes))
		 return FALSE;
	 if (!xdr_cookieverf3 (xdrs, objp->cookieverf))
		 return FALSE;
	 if (!xdr_dirlistp3 (xdrs, &objp->reply))
		 return FALSE;
	return TRUE;
}

bool_t
xdr_readdirp3resfail (XDR *xdrs, readdirp3resfail *objp)
{
	 if (!xdr_post_op_attr (xdrs, &objp->dir_attributes))
		 return FALSE;
	return TRUE;
}

bool_t
xdr_readdirp3res (XDR *xdrs, readdirp3res *objp)
{
	 if (!xdr_nfsstat3 (xdrs, &objp->status))
		 return FALSE;
	switch (objp->status) {
	case NFS3_OK:
		 if (!xdr_readdirp3resok (xdrs, &objp->readdirp3res_u.resok))
			 return FALSE;
		break;
	default:
		 if (!xdr_readdirp3resfail (xdrs, &objp->readdirp3res_u.resfail))
			 return FALSE;
		break;
	}
	return TRUE;
}

bool_t
xdr_fsstat3args (XDR *xdrs, fsstat3args *objp)
{
	 if (!xdr_nfs_fh3 (xdrs, &objp->fsroot))
		 return FALSE;
	return TRUE;
}

bool_t
xdr_fsstat3resok (XDR *xdrs, fsstat3resok *objp)
{
	 if (!xdr_post_op_attr (xdrs, &objp->obj_attributes))
		 return FALSE;
	 if (!xdr_size3 (xdrs, &objp->tbytes))
		 return FALSE;
	 if (!xdr_size3 (xdrs, &objp->fbytes))
		 return FALSE;
	 if (!xdr_size3 (xdrs, &objp->abytes))
		 return FALSE;
	 if (!xdr_size3 (xdrs, &objp->tfiles))
		 return FALSE;
	 if (!xdr_size3 (xdrs, &objp->ffiles))
		 return FALSE;
	 if (!xdr_size3 (xdrs, &objp->afiles))
		 return FALSE;
	 if (!xdr_uint32 (xdrs, &objp->invarsec))
		 return FALSE;
	return TRUE;
}

bool_t
xdr_fsstat3resfail (XDR *xdrs, fsstat3resfail *objp)
{
	 if (!xdr_post_op_attr (xdrs, &objp->obj_attributes))
		 return FALSE;
	return TRUE;
}

bool_t
xdr_fsstat3res (XDR *xdrs, fsstat3res *objp)
{
	 if (!xdr_nfsstat3 (xdrs, &objp->status))
		 return FALSE;
	switch (objp->status) {
	case NFS3_OK:
		 if (!xdr_fsstat3resok (xdrs, &objp->fsstat3res_u.resok))
			 return FALSE;
		break;
	default:
		 if (!xdr_fsstat3resfail (xdrs, &objp->fsstat3res_u.resfail))
			 return FALSE;
		break;
	}
	return TRUE;
}

bool_t
xdr_fsinfo3args (XDR *xdrs, fsinfo3args *objp)
{
	 if (!xdr_nfs_fh3 (xdrs, &objp->fsroot))
		 return FALSE;
	return TRUE;
}

bool_t
xdr_fsinfo3resok (XDR *xdrs, fsinfo3resok *objp)
{
	 if (!xdr_post_op_attr (xdrs, &objp->obj_attributes))
		 return FALSE;
	 if (!xdr_uint32 (xdrs, &objp->rtmax))
		 return FALSE;
	 if (!xdr_uint32 (xdrs, &objp->rtpref))
		 return FALSE;
	 if (!xdr_uint32 (xdrs, &objp->rtmult))
		 return FALSE;
	 if (!xdr_uint32 (xdrs, &objp->wtmax))
		 return FALSE;
	 if (!xdr_uint32 (xdrs, &objp->wtpref))
		 return FALSE;
	 if (!xdr_uint32 (xdrs, &objp->wtmult))
		 return FALSE;
	 if (!xdr_uint32 (xdrs, &objp->dtpref))
		 return FALSE;
	 if (!xdr_size3 (xdrs, &objp->maxfilesize))
		 return FALSE;
	 if (!xdr_nfstime3 (xdrs, &objp->time_delta))
		 return FALSE;
	 if (!xdr_uint32 (xdrs, &objp->properties))
		 return FALSE;
	return TRUE;
}

bool_t
xdr_fsinfo3resfail (XDR *xdrs, fsinfo3resfail *objp)
{
	 if (!xdr_post_op_attr (xdrs, &objp->obj_attributes))
		 return FALSE;
	return TRUE;
}

bool_t
xdr_fsinfo3res (XDR *xdrs, fsinfo3res *objp)
{
	 if (!xdr_nfsstat3 (xdrs, &objp->status))
		 return FALSE;
	switch (objp->status) {
	case NFS3_OK:
		 if (!xdr_fsinfo3resok (xdrs, &objp->fsinfo3res_u.resok))
			 return FALSE;
		break;
	default:
		 if (!xdr_fsinfo3resfail (xdrs, &objp->fsinfo3res_u.resfail))
			 return FALSE;
		break;
	}
	return TRUE;
}

bool_t
xdr_pathconf3args (XDR *xdrs, pathconf3args *objp)
{
	 if (!xdr_nfs_fh3 (xdrs, &objp->object))
		 return FALSE;
	return TRUE;
}

bool_t
xdr_pathconf3resok (XDR *xdrs, pathconf3resok *objp)
{
	register int32_t *buf;


	if (xdrs->x_op == XDR_ENCODE) {
		 if (!xdr_post_op_attr (xdrs, &objp->obj_attributes))
			 return FALSE;
		 if (!xdr_uint32 (xdrs, &objp->linkmax))
			 return FALSE;
		 if (!xdr_uint32 (xdrs, &objp->name_max))
			 return FALSE;
		buf = XDR_INLINE (xdrs, 4 * BYTES_PER_XDR_UNIT);
		if (buf == NULL) {
			 if (!xdr_bool (xdrs, &objp->no_trunc))
				 return FALSE;
			 if (!xdr_bool (xdrs, &objp->chown_restricted))
				 return FALSE;
			 if (!xdr_bool (xdrs, &objp->case_insensitive))
				 return FALSE;
			 if (!xdr_bool (xdrs, &objp->case_preserving))
				 return FALSE;
		} else {
			IXDR_PUT_BOOL(buf, objp->no_trunc);
			IXDR_PUT_BOOL(buf, objp->chown_restricted);
			IXDR_PUT_BOOL(buf, objp->case_insensitive);
			IXDR_PUT_BOOL(buf, objp->case_preserving);
		}
		return TRUE;
	} else if (xdrs->x_op == XDR_DECODE) {
		 if (!xdr_post_op_attr (xdrs, &objp->obj_attributes))
			 return FALSE;
		 if (!xdr_uint32 (xdrs, &objp->linkmax))
			 return FALSE;
		 if (!xdr_uint32 (xdrs, &objp->name_max))
			 return FALSE;
		buf = XDR_INLINE (xdrs, 4 * BYTES_PER_XDR_UNIT);
		if (buf == NULL) {
			 if (!xdr_bool (xdrs, &objp->no_trunc))
				 return FALSE;
			 if (!xdr_bool (xdrs, &objp->chown_restricted))
				 return FALSE;
			 if (!xdr_bool (xdrs, &objp->case_insensitive))
				 return FALSE;
			 if (!xdr_bool (xdrs, &objp->case_preserving))
				 return FALSE;
		} else {
			objp->no_trunc = IXDR_GET_BOOL(buf);
			objp->chown_restricted = IXDR_GET_BOOL(buf);
			objp->case_insensitive = IXDR_GET_BOOL(buf);
			objp->case_preserving = IXDR_GET_BOOL(buf);
		}
	 return TRUE;
	}

	 if (!xdr_post_op_attr (xdrs, &objp->obj_attributes))
		 return FALSE;
	 if (!xdr_uint32 (xdrs, &objp->linkmax))
		 return FALSE;
	 if (!xdr_uint32 (xdrs, &objp->name_max))
		 return FALSE;
	 if (!xdr_bool (xdrs, &objp->no_trunc))
		 return FALSE;
	 if (!xdr_bool (xdrs, &objp->chown_restricted))
		 return FALSE;
	 if (!xdr_bool (xdrs, &objp->case_insensitive))
		 return FALSE;
	 if (!xdr_bool (xdrs, &objp->case_preserving))
		 return FALSE;
	return TRUE;
}

bool_t
xdr_pathconf3resfail (XDR *xdrs, pathconf3resfail *objp)
{
	 if (!xdr_post_op_attr (xdrs, &objp->obj_attributes))
		 return FALSE;
	return TRUE;
}

bool_t
xdr_pathconf3res (XDR *xdrs, pathconf3res *objp)
{
	 if (!xdr_nfsstat3 (xdrs, &objp->status))
		 return FALSE;
	switch (objp->status) {
	case NFS3_OK:
		 if (!xdr_pathconf3resok (xdrs, &objp->pathconf3res_u.resok))
			 return FALSE;
		break;
	default:
		 if (!xdr_pathconf3resfail (xdrs, &objp->pathconf3res_u.resfail))
			 return FALSE;
		break;
	}
	return TRUE;
}

bool_t
xdr_commit3args (XDR *xdrs, commit3args *objp)
{
	 if (!xdr_nfs_fh3 (xdrs, &objp->file))
		 return FALSE;
	 if (!xdr_offset3 (xdrs, &objp->offset))
		 return FALSE;
	 if (!xdr_count3 (xdrs, &objp->count))
		 return FALSE;
	return TRUE;
}

bool_t
xdr_commit3resok (XDR *xdrs, commit3resok *objp)
{
	 if (!xdr_wcc_data (xdrs, &objp->file_wcc))
		 return FALSE;
	 if (!xdr_writeverf3 (xdrs, objp->verf))
		 return FALSE;
	return TRUE;
}

bool_t
xdr_commit3resfail (XDR *xdrs, commit3resfail *objp)
{
	 if (!xdr_wcc_data (xdrs, &objp->file_wcc))
		 return FALSE;
	return TRUE;
}

bool_t
xdr_commit3res (XDR *xdrs, commit3res *objp)
{
	 if (!xdr_nfsstat3 (xdrs, &objp->status))
		 return FALSE;
	switch (objp->status) {
	case NFS3_OK:
		 if (!xdr_commit3resok (xdrs, &objp->commit3res_u.resok))
			 return FALSE;
		break;
	default:
		 if (!xdr_commit3resfail (xdrs, &objp->commit3res_u.resfail))
			 return FALSE;
		break;
	}
	return TRUE;
}

bool_t
xdr_fhandle3 (XDR *xdrs, fhandle3 *objp)
{
	 if (!xdr_bytes (xdrs, (char **)&objp->fhandle3_val, (u_int *) &objp->fhandle3_len, FHSIZE3))
		 return FALSE;
	return TRUE;
}

bool_t
xdr_dirpath (XDR *xdrs, dirpath *objp)
{
	 if (!xdr_string (xdrs, objp, MNTPATHLEN))
		 return FALSE;
	return TRUE;
}

bool_t
xdr_name (XDR *xdrs, name *objp)
{
	 if (!xdr_string (xdrs, objp, MNTNAMLEN))
		 return FALSE;
	return TRUE;
}

bool_t
xdr_mountstat3 (XDR *xdrs, mountstat3 *objp)
{
	 if (!xdr_enum (xdrs, (enum_t *) objp))
		 return FALSE;
	return TRUE;
}

bool_t
xdr_mountres3_ok (XDR *xdrs, mountres3_ok *objp)
{
	 if (!xdr_fhandle3 (xdrs, &objp->fhandle))
		 return FALSE;
	 if (!xdr_array (xdrs, (char **)&objp->auth_flavors.auth_flavors_val, (u_int *) &objp->auth_flavors.auth_flavors_len, ~0,
		sizeof (int), (xdrproc_t) xdr_int))
		 return FALSE;
	return TRUE;
}

bool_t
xdr_mountres3 (XDR *xdrs, mountres3 *objp)
{
	 if (!xdr_mountstat3 (xdrs, &objp->fhs_status))
		 return FALSE;
	switch (objp->fhs_status) {
	case MNT3_OK:
		 if (!xdr_mountres3_ok (xdrs, &objp->mountres3_u.mountinfo))
			 return FALSE;
		break;
	default:
		break;
	}
	return TRUE;
}

bool_t
xdr_mountlist (XDR *xdrs, mountlist *objp)
{
	 if (!xdr_pointer (xdrs, (char **)objp, sizeof (struct mountbody), (xdrproc_t) xdr_mountbody))
		 return FALSE;
	return TRUE;
}

bool_t
xdr_mountbody (XDR *xdrs, mountbody *objp)
{
	 if (!xdr_name (xdrs, &objp->ml_hostname))
		 return FALSE;
	 if (!xdr_dirpath (xdrs, &objp->ml_directory))
		 return FALSE;
	 if (!xdr_mountlist (xdrs, &objp->ml_next))
		 return FALSE;
	return TRUE;
}

bool_t
xdr_groups (XDR *xdrs, groups *objp)
{
	 if (!xdr_pointer (xdrs, (char **)objp, sizeof (struct groupnode), (xdrproc_t) xdr_groupnode))
		 return FALSE;
	return TRUE;
}

bool_t
xdr_groupnode (XDR *xdrs, groupnode *objp)
{
	 if (!xdr_name (xdrs, &objp->gr_name))
		 return FALSE;
	 if (!xdr_groups (xdrs, &objp->gr_next))
		 return FALSE;
	return TRUE;
}

bool_t
xdr_exports (XDR *xdrs, exports *objp)
{
	 if (!xdr_pointer (xdrs, (char **)objp, sizeof (struct exportnode), (xdrproc_t) xdr_exportnode))
		 return FALSE;
	return TRUE;
}

bool_t
xdr_exportnode (XDR *xdrs, exportnode *objp)
{
	 if (!xdr_dirpath (xdrs, &objp->ex_dir))
		 return FALSE;
	 if (!xdr_groups (xdrs, &objp->ex_groups))
		 return FALSE;
	 if (!xdr_exports (xdrs, &objp->ex_next))
		 return FALSE;
	return TRUE;
}

void
xdr_free_exports_list (struct exportnode *first)
{
        struct exportnode       *elist = NULL;

        if (!first)
                return;

        while (first) {
                elist = first->ex_next;
                if (first->ex_dir)
                        GF_FREE (first->ex_dir);

                if (first->ex_groups) {
                        if (first->ex_groups->gr_name)
                                GF_FREE (first->ex_groups->gr_name);
                        GF_FREE (first->ex_groups);
                }

                GF_FREE (first);
                first = elist;
        }

}


void
xdr_free_mountlist (mountlist ml)
{
        struct mountbody        *next = NULL;

        if (!ml)
                return;

        while (ml) {
                GF_FREE (ml->ml_hostname);
                GF_FREE (ml->ml_directory);
                next = ml->ml_next;
                GF_FREE (ml);
                ml = next;
        }

        return;
}


/* Free statements are based on the way sunrpc xdr decoding
 * code performs memory allocations.
 */
void
xdr_free_write3args_nocopy (write3args *wa)
{
        if (!wa)
                return;

        FREE (wa->file.data.data_val);
}