2388 files changed, 561204 insertions, 502280 deletions

diff --git a/.clang-format b/.clang-format
new file mode 100644
index 00000000000..84c2efe3fad
--- /dev/null
+++ b/.clang-format
@@ -0,0 +1,107 @@
+---
+Language:        Cpp
+# BasedOnStyle:  Chromium
+AccessModifierOffset: -1
+AlignAfterOpenBracket: Align
+AlignConsecutiveAssignments: false
+AlignConsecutiveDeclarations: false
+AlignEscapedNewlines: Right
+AlignOperands:   true
+AlignTrailingComments: true
+AllowAllParametersOfDeclarationOnNextLine: false
+AllowShortBlocksOnASingleLine: false
+AllowShortCaseLabelsOnASingleLine: false
+AllowShortFunctionsOnASingleLine: Inline
+AllowShortIfStatementsOnASingleLine: false
+AllowShortLoopsOnASingleLine: false
+AlwaysBreakAfterDefinitionReturnType: All
+AlwaysBreakAfterReturnType: All
+AlwaysBreakBeforeMultilineStrings: true
+AlwaysBreakTemplateDeclarations: true
+BinPackArguments: true
+BinPackParameters: true
+BraceWrapping:
+  AfterClass:      false
+  AfterControlStatement: false
+  AfterEnum:       false
+  AfterFunction:   true
+  AfterNamespace:  false
+  AfterObjCDeclaration: false
+  AfterStruct:     false
+  AfterUnion:      false
+  BeforeCatch:     false
+  BeforeElse:      false
+  IndentBraces:    false
+  SplitEmptyFunction: true
+  SplitEmptyRecord: true
+  SplitEmptyNamespace: true
+BreakBeforeBinaryOperators: None
+BreakBeforeBraces: Linux
+BreakBeforeInheritanceComma: false
+BreakBeforeTernaryOperators: true
+BreakConstructorInitializersBeforeComma: false
+BreakConstructorInitializers: BeforeColon
+BreakAfterJavaFieldAnnotations: false
+BreakStringLiterals: true
+ColumnLimit:     80
+CommentPragmas:  '^ IWYU pragma:'
+CompactNamespaces: false
+ConstructorInitializerAllOnOneLineOrOnePerLine: true
+ConstructorInitializerIndentWidth: 4
+ContinuationIndentWidth: 4
+Cpp11BracedListStyle: true
+DerivePointerAlignment: false
+DisableFormat:   false
+ExperimentalAutoDetectBinPacking: false
+FixNamespaceComments: true
+ForEachMacros:
+  - foreach
+  - Q_FOREACH
+  - BOOST_FOREACH
+IncludeCategories:
+  - Regex:           '^<.*\.h>'
+    Priority:        1
+  - Regex:           '^<.*'
+    Priority:        2
+  - Regex:           '.*'
+    Priority:        3
+IncludeIsMainRegex: '([-_](test|unittest))?$'
+IndentCaseLabels: true
+IndentWidth:     4
+IndentWrappedFunctionNames: false
+JavaScriptQuotes: Leave
+JavaScriptWrapImports: true
+KeepEmptyLinesAtTheStartOfBlocks: false
+MacroBlockBegin: ''
+MacroBlockEnd:   ''
+MaxEmptyLinesToKeep: 1
+NamespaceIndentation: None
+ObjCBlockIndentWidth: 2
+ObjCSpaceAfterProperty: false
+ObjCSpaceBeforeProtocolList: false
+PenaltyBreakAssignment: 200
+PenaltyBreakBeforeFirstCallParameter: 1
+PenaltyBreakComment: 300
+PenaltyBreakFirstLessLess: 120
+PenaltyBreakString: 1000
+PenaltyExcessCharacter: 1000000
+PenaltyReturnTypeOnItsOwnLine: 200
+PointerAlignment: Right
+ReflowComments:  true
+SortIncludes:    false
+SortUsingDeclarations: true
+SpaceAfterCStyleCast: false
+SpaceAfterTemplateKeyword: true
+SpaceBeforeAssignmentOperators: true
+SpaceBeforeParens: ControlStatements
+SpaceInEmptyParentheses: false
+SpacesBeforeTrailingComments: 2
+SpacesInAngles:  false
+SpacesInContainerLiterals: true
+SpacesInCStyleCastParentheses: false
+SpacesInParentheses: false
+SpacesInSquareBrackets: false
+Standard:        Auto
+TabWidth:        8
+UseTab:          Never
+...
diff --git a/.github/ISSUE_TEMPLATE b/.github/ISSUE_TEMPLATE
new file mode 100644
index 00000000000..386ed2d8dd5
--- /dev/null
+++ b/.github/ISSUE_TEMPLATE
@@ -0,0 +1,30 @@
+<!-- Please use this template while reporting an issue, providing as much information as possible. Failure to do so may result in a delayed response. Thank you! -->
+
+**Description of problem:**
+
+
+**The exact command to reproduce the issue**:
+
+
+**The full output of the command that failed**:
+<details>
+
+
+
+</details>
+
+**Expected results:**
+
+
+**Additional info:**
+
+
+**- The output of the `gluster volume info` command**:
+<details>
+
+
+
+</details>
+
+**- The operating system / glusterfs version**:
+
diff --git a/.github/PULL_REQUEST_TEMPLATE b/.github/PULL_REQUEST_TEMPLATE
new file mode 100644
index 00000000000..e69de29bb2d
--- /dev/null
+++ b/.github/PULL_REQUEST_TEMPLATE
diff --git a/.github/RELEASE_TRACKER_TEMPLATE b/.github/RELEASE_TRACKER_TEMPLATE
new file mode 100644
index 00000000000..502bbd5556c
--- /dev/null
+++ b/.github/RELEASE_TRACKER_TEMPLATE
@@ -0,0 +1,12 @@
+<!-- Please use this template while creating a tracker issue -->
+
+**Description of problem:**
+A tracker issue to track the issues that will be fixed as a part of this release
+
+
+**Major or minor release**:
+
+
+**Release version**:
+
+
diff --git a/.github/stale.yml b/.github/stale.yml
new file mode 100644
index 00000000000..460e327c6ea
--- /dev/null
+++ b/.github/stale.yml
@@ -0,0 +1,25 @@
+# Number of days of inactivity before an issue becomes stale
+daysUntilStale: 210
+# Number of days of inactivity before a stale issue is closed
+daysUntilClose: 15
+# Issues with these labels will never be considered stale
+exemptLabels:
+  - pinned
+  - security
+# Label to use when marking an issue as stale
+staleLabel: wontfix
+
+# Comment to post when marking an issue as stale. Set to `false` to disable
+markComment: >
+  Thank you for your contributions.
+
+  Noticed that this issue is not having any activity in last ~6 months! We
+  are marking this issue as stale because it has not had recent activity.
+
+  It will be closed in 2 weeks if no one responds with a comment here.
+  
+
+# Comment to post when closing a stale issue. Set to `false` to disable
+closeComment: >
+  Closing this issue as there was no update since my last update on issue.
+  If this is an issue which is still valid, feel free to open it.
diff --git a/.gitignore b/.gitignore
index cb7833ce1b1..fc5ba586f8e 100644
--- a/.gitignore
+++ b/.gitignore
@@ -4,13 +4,16 @@ build
 config.*
 configure
 cscope.*
+tags
 depcomp
+INSTALL
 install-sh
 ltmain.sh
 Makefile
 Makefile.in
 missing
 stamp-h1
+stamp-h2
 test-driver
 *compile
 *.gcda
@@ -26,10 +29,12 @@ test-driver
 *.patch
 .libs
 .deps
+.dirstamp
 
 # Softlinks to test and log
 log
 *.vol
+.clang-format
 
 # cmocka unit tests
 *.log
@@ -37,15 +42,16 @@ log
 *_unittest
 
 # Generated files
+site.h
 *.py[co]
 api/examples/__init__.py
 api/examples/setup.py
 api/src/gfapi.map
 cli/src/gluster
-contrib/argp-standalone/libargp.a
 contrib/fuse-util/fusermount-glusterfs
-contrib/uuid/uuid_types.h
 extras/geo-rep/gsync-sync-gfid
+extras/geo-rep/schedule_georep.py
+extras/snap_scheduler/conf.py
 extras/init.d/glusterd-Debian
 extras/init.d/glusterd-FreeBSD
 extras/init.d/glusterd-Redhat
@@ -55,14 +61,19 @@ extras/ocf/glusterd
 extras/ocf/volume
 extras/run-gluster.tmpfiles
 extras/systemd/glusterd.service
+extras/systemd/gluster-ta-volume.service
+extras/systemd/glusterfssharedstorage.service
 extras/who-wrote-glusterfs/gitdm
 geo-replication/.tox
+geo-replication/gsyncd.conf
 geo-replication/src/gsyncd
 geo-replication/src/peer_gsec_create
 geo-replication/src/peer_mountbroker
+geo-replication/src/peer_mountbroker.py
 geo-replication/src/set_geo_rep_pem_keys.sh
+geo-replication/src/peer_georep-sshkey.py
 geo-replication/syncdaemon.egg-info
-geo-replication/syncdaemon/configinterface.py
+geo-replication/syncdaemon/conf.py
 geo-replication/tests/unit/.coverage
 geo-replication/tests/unit/cover
 geo-replication/tests/unit/coverage.xml
@@ -73,15 +84,20 @@ glusterfs.spec
 glusterfsd/src/glusterd
 glusterfsd/src/glusterfs
 glusterfsd/src/glusterfsd
+glusterfsd/src/gf_attach
 heal/src/glfsheal
 libgfchangelog.pc
-libgfdb.pc
 libglusterfs/src/graph.lex.c
 libglusterfs/src/y.tab.c
 libglusterfs/src/y.tab.h
 libglusterfs/src/defaults.c
+libglusterfs/src/cli1-xdr.h
+libglusterfs/src/protocol-common.h
 libtool
+# copied XDR for cyclic libglusterfs <-> rpc-header dependency
 run-tests.sh
+!tests/basic/fuse/Makefile
+!tests/basic/gfapi/Makefile
 tests/env.rc
 tests/utils/arequal-checksum
 xlators/features/glupy/src/__init__.py
@@ -92,3 +108,18 @@ extras/peer_add_secret_pub
 tools/gfind_missing_files/gcrawler
 tools/glusterfind/glusterfind
 tools/glusterfind/src/tool.conf
+# Eventing
+events/src/eventsapiconf.py
+extras/systemd/glustereventsd.service
+events/src/eventtypes.py
+libglusterfs/src/eventtypes.h
+extras/init.d/glustereventsd-Debian
+extras/init.d/glustereventsd-FreeBSD
+extras/init.d/glustereventsd-Redhat
+tools/setgfid2path/src/gluster-setgfid2path
+xlators/features/cloudsync/src/cloudsync-autogen-fops.c
+xlators/features/cloudsync/src/cloudsync-autogen-fops.h
+xlators/features/utime/src/utime-autogen-fops.c
+xlators/features/utime/src/utime-autogen-fops.h
+tests/basic/metadisp/ftruncate
+xlators/features/metadisp/src/fops.c
diff --git a/.mailmap b/.mailmap
index d4a13b40065..141a1667ffa 100644
--- a/.mailmap
+++ b/.mailmap
@@ -10,15 +10,20 @@ Amar Tumballi <amarts@redhat.com> <amar@gluster.com> <amar@del.gluster.com>
 Anand Avati <avati@redhat.com> <avati@gluster.com> <avati@dev.gluster.com> <avati@amp.gluster.com> <avati@blackhole.gluster.com>
 Anush Shetty <ashetty@redhat.com> <anush@gluster.com>
 Csaba Henk <csaba@redhat.com> <csaba@gluster.com> <csaba@lowlife.hu> <csaba@zresearch.com>
+Günther Deschner <gd@redhat.com> <gd@samba.org>
 Harshavardhana <fharshav@redhat.com> <harsha@gluster.com> <harsha@zresearch.com> <harsha@dev.gluster.com> <harsha@harshavardhana.net>
+Ji-Hyeon Gim <potatogim@gluesys.com> <potatogim@potatogim.net>
 Justin Clift <justin@gluster.org> <jclift@redhat.com>
-Kaleb S. KEITHLEY <kkeithle@redhat.com> <kkeithle@f16node1.kkeithle.usersys.redhat.com>
+Kaleb S. KEITHLEY <kkeithle@redhat.com> <kkeithle@f16node1.kkeithle.usersys.redhat.com> <kkeithle@linux.keithley.org>
 Kaushal M <kaushal@redhat.com> <kaushal@gluster.com>
 Kaushik BV <kbudiger@redhat.com> <kaushikbv@gluster.com>
 Krishna Srinivas <ksriniva@redhat.com> <krishna@gluster.com> <krishna@zresearch.com> <krishna@guest-laptop>
 Krishnan Parthasarathi <kparthas@redhat.com> <kp@gluster.com>
 Louis Zuckerman <louiszuckerman@gmail.com> <me@louiszuckerman.com>
 M S Vishwanath Bhat <vbhat@redhat.com> <msvbhat@gmail.com> <vishwanath@gluster.com>
+Michael Adam <madam@redhat.com> <obnox@samba.org>
+Oleksandr Natalenko <oleksandr@natalenko.name> <o.natalenko@lanet.ua>
+Patrick Uiterwijk <puiterwijk@fedoraproject.org> <patrick@puiterwijk.org>
 Pavan Sondur <pavan@gluster.com> <pavan@dev.gluster.com>
 Pete Zaitcev <zaitcev@kotori.zaitcev.us> <zaitcev@yahoo.com>
 Pranith Kumar K <pkarampu@redhat.com> <pranithk@gluster.com>
@@ -27,9 +32,12 @@ Raghavendra Bhat <raghavendra@redhat.com> <raghavendrabhat@gluster.com>
 Raghavendra G <rgowdapp@redhat.com> <raghavendra@gluster.com> <raghavendra@zresearch.com>
 Rahul C S <rahulcs@redhat.com> <rahulcssjce@gmail.com>
 Rajesh Amaravathi <rajesh@redhat.com> <rajesh@gluster.com> <rajesh.amaravathi@gmail.com>
+Ravishankar N <ravishankar@redhat.com> <root@ravi2.(none)>
+Sakshi Bansal <sabansal@redhat.com> <sabansal@localhost.localdomain>
 Shehjar Tikoo <shehjart@gluster.com> <shehjart@zresearch.com>
 Venky Shankar <vshankar@redhat.com> <venky@gluster.com>
 Vijay Bellur <vbellur@redhat.com> <vijay@gluster.com> <vijay@dev.gluster.com>
 Vijaykumar Koppad <vkoppad@redhat.com> <vijaykumar.koppad@gmail.com>
+Vijaikumar Mallikarjuna <vmallika@redhat.com>
 Vikas Gorur <vikas@gluster.com> <vikas@zresearch.com>
 shishir gowda <sgowda@redhat.com> <shishirng@gluster.com>
diff --git a/.testignore b/.testignore
new file mode 100644
index 00000000000..fe8f838bf2b
--- /dev/null
+++ b/.testignore
@@ -0,0 +1,64 @@
+.github/ISSUE_TEMPLATE
+.github/PULL_REQUEST_TEMPLATE
+.github/stale.yml
+.gitignore
+.mailmap
+.testignore
+.clang-format
+rfc.sh
+submit-for-review.sh
+AUTHORS
+CONTRIBUTING.md
+COPYING-GPLV2
+COPYING-LGPLV3
+ChangeLog
+INSTALL
+MAINTAINERS
+NEWS
+README.md
+THANKS
+COMMITMENT
+api/examples/README
+api/examples/getvolfile.py
+api/src/README.Symbol_Versions
+build-aux/checkpatch.pl
+contrib/fuse-lib/COPYING.LIB
+contrib/fuse-util/COPYING
+contrib/macfuse/COPYING.txt
+doc/*
+extras/FreeBSD/README.FreeBSD
+extras/Solaris/README.solaris
+extras/Ubuntu/README.Ubuntu
+extras/benchmarking/README
+extras/cliutils/README.md
+extras/command-completion/README
+extras/create_new_xlator/README.md
+extras/glusterfs.vim
+extras/glusterfs-logrotate
+extras/glusterfs-georep-logrotate
+extras/init.d/glusterd-Debian.in
+extras/init.d/glusterd-FreeBSD.in
+extras/init.d/glusterd-Redhat.in
+extras/init.d/glusterd-SuSE.in
+extras/init.d/glusterd.plist.in
+extras/init.d/glustereventsd-Debian.in
+extras/init.d/glustereventsd-FreeBSD.in
+extras/init.d/glustereventsd-Redhat.in
+extras/init.d/rhel5-load-fuse.modules
+extras/logger.conf.example
+extras/snap_scheduler/README.md
+extras/test/ld-preload-test/README
+extras/who-wrote-glusterfs/*
+extras/distributed-testing/*
+geo-replication/syncdaemon/README.md
+geo-replication/test-requirements.txt
+rpc/xdr/src/.gitignore
+tests/README.md
+xlators/experimental/README.md
+xlators/experimental/dht2/README.md
+xlators/experimental/dht2/TODO.md
+xlators/experimental/posix2/README.md
+xlators/experimental/posix2/TODO.md
+xlators/features/glupy/doc/README.md
+xlators/features/glupy/doc/TESTING
+xlators/features/glupy/doc/test.vol
diff --git a/COMMITMENT b/COMMITMENT
new file mode 100644
index 00000000000..16b75efcf29
--- /dev/null
+++ b/COMMITMENT
@@ -0,0 +1,46 @@
+Common Cure Rights Commitment
+Version 1.0
+
+Before filing or continuing to prosecute any legal proceeding or claim
+(other than a Defensive Action) arising from termination of a Covered
+License, we commit to extend to the person or entity ('you') accused
+of violating the Covered License the following provisions regarding
+cure and reinstatement, taken from GPL version 3. As used here, the
+term 'this License' refers to the specific Covered License being
+enforced.
+
+    However, if you cease all violation of this License, then your
+    license from a particular copyright holder is reinstated (a)
+    provisionally, unless and until the copyright holder explicitly
+    and finally terminates your license, and (b) permanently, if the
+    copyright holder fails to notify you of the violation by some
+    reasonable means prior to 60 days after the cessation.
+
+    Moreover, your license from a particular copyright holder is
+    reinstated permanently if the copyright holder notifies you of the
+    violation by some reasonable means, this is the first time you
+    have received notice of violation of this License (for any work)
+    from that copyright holder, and you cure the violation prior to 30
+    days after your receipt of the notice.
+
+We intend this Commitment to be irrevocable, and binding and
+enforceable against us and assignees of or successors to our
+copyrights.
+
+Definitions
+
+'Covered License' means the GNU General Public License, version 2
+(GPLv2), the GNU Lesser General Public License, version 2.1
+(LGPLv2.1), or the GNU Library General Public License, version 2
+(LGPLv2), all as published by the Free Software Foundation.
+
+'Defensive Action' means a legal proceeding or claim that We bring
+against you in response to a prior proceeding or claim initiated by
+you or your affiliate.
+
+'We' means each contributor to this repository as of the date of
+inclusion of this file, including subsidiaries of a corporate
+contributor.
+
+This work is available under a Creative Commons Attribution-ShareAlike
+4.0 International license (https://creativecommons.org/licenses/by-sa/4.0/).
diff --git a/CONTRIBUTING b/CONTRIBUTING
deleted file mode 100644
index 7bccd88d7e5..00000000000
--- a/CONTRIBUTING
+++ /dev/null
@@ -1,25 +0,0 @@
-        Developer's Certificate of Origin 1.1
-
-        By making a contribution to this project, I certify that:
-
-        (a) The contribution was created in whole or in part by me and I
-            have the right to submit it under the open source license
-            indicated in the file; or
-
-        (b) The contribution is based upon previous work that, to the best
-            of my knowledge, is covered under an appropriate open source
-            license and I have the right under that license to submit that
-            work with modifications, whether created in whole or in part
-            by me, under the same open source license (unless I am
-            permitted to submit under a different license), as indicated
-            in the file; or
-
-        (c) The contribution was provided directly to me by some other
-            person who certified (a), (b) or (c) and I have not modified
-            it.
-
-        (d) I understand and agree that this project and the contribution
-            are public and that a record of the contribution (including all
-            personal information I submit with it, including my sign-off) is
-            maintained indefinitely and may be redistributed consistent with
-            this project or the open source license(s) involved.
diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md
new file mode 100644
index 00000000000..65fc3497104
--- /dev/null
+++ b/CONTRIBUTING.md
@@ -0,0 +1,114 @@
+# GlusterFS project Contribution guidelines
+
+## Development Workflow
+
+We follow most of the details as per the [document here](https://help.github.com/en/github/collaborating-with-issues-and-pull-requests). If you are not aware of the github workflow, it is recommended to go through them before continuing here.
+
+
+#### Get the Repository setup
+
+0. Fork Repository
+   - Fork [GlusterFS repository](https://github.com/gluster/glusterfs/fork).
+
+1. Clone Repository
+   - Clone the glusterfs repo freshly from github using below steps.
+
+```
+   git clone git@github.com:${username}/glusterfs.git
+   cd glusterfs/
+   git remote add upstream git@github.com:gluster/glusterfs.git
+```
+
+About two tasks are one time for the life time. You can continue to use the same repository for all the work in future.
+
+#### Development & Other flows
+
+0. Issue:
+   - Make sure there is an issue filed for the task you are working on.
+   - If it is not filed, open the issue with all the description.
+   - If it is a bug fix, add label "Type:Bug".
+   - If it is an RFC, provide all the documentation, and request for "DocApproved", and "SpecApproved" label.
+
+1. Code:
+   - Start coding
+   - Build and test locally
+   - Make sure clang-format is installed and is run on the patch.
+
+2. Keep up-to-date
+   - GlusterFS is a large project with many developers, so there would be one or the other patch everyday.
+   - It is critical for developer to be up-to-date with `devel` repo to be Conflict-Free when PR is opened.
+   - Git provides many options to keep up-to-date, below is one of them
+```
+   git fetch upstream
+   git rebase upstream/devel
+```
+   - It is recommended you keep pushing to your repo every day, so you don't loose any work.
+     - It can be done by `./rfc.sh` (or `git push origin HEAD:issueNNN`)
+
+2. Commit Message / PR description:
+   - The name of the branch on your personal fork can start with issueNNNN, followed by anything of your choice.
+   - PRs continue to have the title of format "component: \<title\>", like it is practiced now.
+   - When you open a PR, having a reference Issue for the commit is mandatory in GlusterFS.
+   - Commit message can have, either `Fixes: #NNNN` or `Updates: #NNNN` in a separate line in the commit message.
+     - Here, NNNN is the Issue ID in glusterfs repository.
+   - Each commit needs the author to have the "Signed-off-by: Name \<email\>" line.
+     - Can do this by `-s` option for `git commit`.
+   - If the PR is not ready for review, apply the label `work-in-progress`.
+     - Check the availability of "Draft PR" is present for you, if yes, use that instead.
+
+3. Tests:
+   - All the required smoke tests would be auto-triggered.
+     - Developers get a chance to retrigger the smoke tests using **"/recheck smoke"** as comment.
+   - The "regression" tests would be triggered by a comment **"/run regression"** from developers in the [@gluster-maintainers](https://github.com/orgs/gluster/teams/gluster-maintainers) group.
+     - Ask for help as comment in PR if you have any questions about the process!
+
+4. Review Process:
+   - `+2` : is equivalent to "Approve" from the people in the maintainer's group.
+   - `+1` : can be given by a maintainer/reviewer by explicitly stating that in the comment.
+   - `-1` : provide details on required changes and pick "Request Changes" while submitting your review.
+   - `-2` : done by adding the `DO-NOT-MERGE` label.
+
+   - Any further discussions can happen as comments in the PR.
+
+5. Making changes:
+   - There are 2 approaches to submit changes done after addressing review comments.
+     - Commit changes as a new commit on top of the original commits in the branch, and push the changes to same branch (issueNNNN)
+     - Commit changes into the same commit with `--amend` option, and do a push to the same branch with `--force` option.
+
+6. Merging:
+   - GlusterFS project follows 'Squash and Merge' method
+     - This is mainly to preserve the historic Gerrit method of one patch in `git log` for one URL link.
+     - This also makes every merge a complete patch, which has passed all tests.
+   - The merging of the patch is expected to be done by the maintainers.
+     - It can be done when all the tests (smoke and regression) pass.
+     - When the PR has 'Approved' flag from corresponding maintainer.
+   - If you feel there is delay, feel free to add a comment, discuss the same in Slack channel, or send email.
+
+## By contributing to this project, the contributor would need to agree to below.
+
+### Developer's Certificate of Origin 1.1
+
+By making a contribution to this project, I certify that:
+
+(a) The contribution was created in whole or in part by me and I
+    have the right to submit it under the open source license
+    indicated in the file; or
+
+(b) The contribution is based upon previous work that, to the best
+    of my knowledge, is covered under an appropriate open source
+    license and I have the right under that license to submit that
+    work with modifications, whether created in whole or in part
+    by me, under the same open source license (unless I am
+    permitted to submit under a different license), as indicated
+    in the file; or
+
+(c) The contribution was provided directly to me by some other
+    person who certified (a), (b) or (c) and I have not modified
+    it.
+
+(d) I understand and agree that this project and the contribution
+    are public and that a record of the contribution (including all
+    personal information I submit with it, including my sign-off) is
+    maintained indefinitely and may be redistributed consistent with
+    this project or the open source license(s) involved.
+
diff --git a/INSTALL b/INSTALL
index 76beb978775..a56390e54fb 100644
--- a/INSTALL
+++ b/INSTALL
@@ -44,4 +44,4 @@ Installation completed :-)
 Make sure your version is the latest from the release, and the one you 
 just installed :-)
 
-For more information on GlusterFS installation refer# http://gluster.org/community/documentation/index.php/Building_GlusterFS
+For more information on GlusterFS installation refer# http://docs.gluster.org/en/latest/Developer-guide/Building-GlusterFS/
diff --git a/MAINTAINERS b/MAINTAINERS
index 057fb62f062..953e8755fd9 100644
--- a/MAINTAINERS
+++ b/MAINTAINERS
@@ -11,7 +11,7 @@ consult gluster-devel@gluster.org and not any specific individual privately.
 
 Descriptions of section entries:
 
-	M: Mail patches to: FullName <address@domain>
+	M: Main contact that knows and takes care of this area
 	L: Mailing list that is relevant to this area
 	W: Web-page with status/info
 	Q: Patchwork web based patch tracking system site
@@ -46,253 +46,409 @@ Descriptions of section entries:
 	      matches patches or files that contain one or more of the words
 	      printk, pr_info or pr_err
 	   One regex pattern per line.  Multiple K: lines acceptable.
+        P: Peer for a component
 
 
 General Project Architects
 --------------------------
-M: Anand Avati <avati@redhat.com>
-M: Jeff Darcy <jdarcy@redhat.com>
-M: Kaleb S. Keithley <kkeithle@redhat.com>
-M: Vijay Bellur <vbellur@redhat.com>
-
+M: Amar Tumballi <amarts@gmail.com>
+M: Xavier Hernandez  <xhernandez@redhat.com>
+P: Pranith Karampuri <pranith.karampuri@phonepe.com>
+P: Atin Mukherjee <amukherj@redhat.com>
 
 xlators:
 --------
+Access Control List (ACL)
+M: Raghavendra Talur <rtalur@redhat.com>
+P: Jiffin Tony Thottan <jthottan@redhat.com>
+S: Maintained
+F: xlators/system/posix-acl/
+
+Arbiter
+M: Ravishankar N <ravishankar@redhat.com>
+P: Pranith Karampuri <pranith.karampuri@phonepe.com>
+S: Maintained
+F: xlators/features/arbiter/
+
 Automatic File Replication (AFR)
-M: Pranith Karampuri <pkarampu@redhat.com>
+M: Pranith Karampuri <pranith.karampuri@phonepe.com>
+M: Ravishankar N <ravishankar@redhat.com>
+P: Karthik US <ksubrahm@redhat.com>
 S: Maintained
 F: xlators/cluster/afr/
 
-Block Device
-S: Orphan
-F: xlators/storage/bd/
+Barrier
+M: Raghavendra Bhat <rabhat@redhat.com>
+P: Atin Mukherjee <amukherj@redhat.com>
+S: Maintained
+F: xlators/features/barrier
 
 BitRot
-M: Venky Shankar <vshankar@redhat.com>
-M: Raghavendra Bhat (a.k.a. "Johnny") <rabhat@redhat.com>
+M: Kotresh HR <khiremat@redhat.com>
+P: Raghavendra Bhat <rabhat@redhat.com>
 S: Maintained
 F: xlators/features/bit-rot/
 
 Changelog
-M: Venky Shankar <vshankar@redhat.com>
+M: Aravinda V K <avishwan@redhat.com>
+P: Kotresh HR <khiremat@redhat.com>
 S: Maintained
 F: xlators/features/changelog/
 
-Changetimerecorder
-M: Dan Lambright <dlambrig@redhat.com>
-S: Maintained
-F: xlators/features/changetimerecorder/
-
 Distributed Hashing Table (DHT)
-M: Raghavendra Gowdappa <rgowdapp@redhat.com>
-M: Shyamsundar Ranganathan <srangana@redhat.com>
+P: Susant Palai <spalai@redhat.com>
 S: Maintained
 F: xlators/cluster/dht/
 
 Erasure Coding
-M: Pranith Karampuri <pkarampu@redhat.com>
-M: Xavier Hernandez  <xhernandez@datalab.es>
+M: Pranith Karampuri <pranith.karampuri@phonepe.com>
+M: Xavier Hernandez  <xhernandez@redhat.com>
+P: Ashish Pandey <aspandey@redhat.com>
 S: Maintained
 F: xlators/cluster/ec/
 
+Error-gen
+M: Raghavendra Talur <rtalur@redhat.com>
+S: Maintained
+F: xlators/debug/error-gen/
+
 FUSE Bridge
-M: Anand Avati <avati@redhat.com>
-M: Niels de Vos <ndevos@redhat.com>
-M: Raghavendra Bhat <rabhat@redhat.com>
+M: Csaba Henk <chenk@redhat.com>
+P: Niels de Vos <ndevos@redhat.com>
 S: Maintained
 F: xlators/mount/
 
 Index
-M: Pranith Karampuri <pkarampu@redhat.com>
+M: Pranith Karampuri <pranith.karampuri@phonepe.com>
+P: Ravishankar N <ravishankar@redhat.com>
 S: Maintained
 F: xlators/features/index/
 
+IO Cache
+P: Mohammed Rafi KC <rafi.kavungal@iternity.com>
+S: Maintained
+F: xlators/performance/io-cache/
+
+IO Statistics
+M: Krutika Dhananjay <kdhananj@redhat.com>
+M: Shyam Ranganathan <srangana@redhat.com>
+S: Maintained
+F: xlators/debug/io-stats/
+
 IO threads
-M: Pranith Karampuri <pkarampu@redhat.com>
+M: Pranith Karampuri <pranith.karampuri@phonepe.com>
+P: Ravishankar N <ravishankar@redhat.com>
 S: Maintained
 F: xlators/performance/io-threads/
 
+Leases
+M: Poornima G <pgurusid@redhat.com>
+P: Niels de Vos <ndevos@redhat.com>
+P: Soumya Koduri <skoduri@redhat.com>
+S: Maintained
+F: xlators/features/leases/
+
 Locks
-M: Pranith Karampuri <pkarampu@redhat.com>
+M: Krutika Dhananjay <kdhananj@redhat.com>
+P: Xavier Hernandez  <xhernandez@redhat.com>
 S: Maintained
 F: xlators/features/locks/
 
 Marker
-M: Raghavendra Gowdappa <rgowdapp@redhat.com>
+M: Kotresh HR <khiremat@redhat.com>
 S: Maintained
 F: xlators/features/marker/
 
-NFS
-M: Niels de Vos <ndevos@redhat.com>
+Meta
+M: Mohammed Rafi KC <rafi.kavungal@iternity.com>
+S: Maintained
+F: xlators/features/meta/
+
+Metadata-cache
+M: Poornima G <pgurusid@redhat.com>
+P: Soumya Koduri <skoduri@redhat.com>
 S: Maintained
+F: xlators/performance/md-cache/
+
+Negative-lookup Cache
+M: Poornima G <pgurusid@redhat.com>
+P: Pranith Karampuri <pranith.karampuri@phonepe.com>
+S: Maintained
+F: xlators/performance/nl-cache/
+
+gNFS
+M: Jiffin Tony Thottan <jthottan@redhat.com>
+P: Xie Changlong <xiechanglong@cmss.chinamobile.com>
+P: Amar Tumballi <amarts@gmail.com>
+S: Odd Fixes
 F: xlators/nfs/server/
 
-Performance
-M: Raghavendra Gowdappa <rgowdapp@redhat.com>
+Open-behind
 S: Maintained
-F: xlators/performance/
+F: xlators/performance/open-behind/
 
 Posix:
-M: Pranith Karampuri <pkarampu@redhat.com>
 M: Raghavendra Bhat <raghavendra@redhat.com>
+P: Kotresh HR <khiremat@redhat.com>
+P: Krutika Dhananjay <kdhananj@redhat.com>
 S: Maintained
 F: xlators/storage/posix/
 
+Quick-read
+S: Maintained
+F: xlators/performance/quick-read/
+
 Quota
-M: Krishnan Parthasarathi <kparthas@redhat.com>
-M: Raghavendra Gowdappa <rgowdapp@redhat.com>
+M: Shyamsundar Ranganathan <srangana@redhat.com>
+P: Hari Gowtham <hgowtham@redhat.com>
 S: Maintained
 F: xlators/features/quota/
 
-Tiering
-M: Dan Lambright <dlambrig@redhat.com>
-M: Joseph Fernandes <josferna@redhat.com>
+Read-ahead
+P: Csaba Henk <chenk@redhat.com>
+S: Maintained
+F: xlators/performance/read-ahead/
+
+Readdir-ahead
 S: Maintained
-F: xlators/cluster/dht/src/tier.c
-F: xlators/features/changetimerecorder
-F: libglusterfs/src/gfdb
-W: http://www.gluster.org/community/documentation/index.php/Features/data-classification
+F: xlators/performance/readdir-ahead/
+
+Sharding
+M: Krutika Dhananjay <kdhananj@redhat.com>
+P: Xavier Hernandez  <xhernandez@redhat.com>
+S: Maintained
+F: xlators/features/shard/
+
+Trash
+M: Anoop C S <anoopcs@redhat.com>
+M: Jiffin Tony Thottan <jthottan@redhat.com>
+S: Maintained
+F: xlators/features/trash/
 
 Upcall
-M: Niels de Vos <ndevos@redhat.com>
+M: Poornima G <pgurusid@redhat.com>
+M: Soumya Koduri <skoduri@redhat.com>
+P: Niels de Vos <ndevos@redhat.com>
 S: Maintained
 F: xlators/features/upcall/
 
+Write-behind
+P: Csaba Henk <chenk@redhat.com>
+S: Maintained
+F: xlators/performance/write-behind/
+
+Write Once Read Many
+P: Karthik US <ksubrahm@redhat.com>
+S: Maintained
+F: xlators/features/read-only/
+
+Cloudsync
+M: Susant Kumar Palai <spalai@redhat.com>
+S: Maintained
+F: xlators/features/cloudsync/
+
 Other bits of code:
 -------------------
 
 Doc
 M: Humble Chirammal <hchiramm@redhat.com>
 M: Raghavendra Talur <rtalur@redhat.com>
-M: Prashanth Pai <ppai@redhat.com>
-M: Shravan Chandrashekar <shravantc99@gmail.com>
 S: Maintained
 F: doc/
 
 Geo Replication
-M: Venky Shankar <vshankar@redhat.com>
+M: Aravinda V K <avishwan@redhat.com>
+M: Kotresh HR <khiremat@redhat.com>
+M: Sunny Kumar <sunkumar@redhat.com>
 S: Maintained
 F: geo-replication/
 
-
-Glupy
-M: Justin Clift <justin@gluster.org>
+Glusterfind
+M: Aravinda VK <avishwan@redhat.com>
 S: Maintained
-F: xlators/features/glupy/
+F: tools/glusterfind/
 
 libgfapi
-M: Anand Avati <avati@redhat.com>
 M: Niels de Vos <ndevos@redhat.com>
-M: Shyamsundar Ranganathan <srangana@redhat.com>
+P: Poornima G <pgurusid@redhat.com>
+P: Shyamsundar Ranganathan <srangana@redhat.com>
+P: Soumya Koduri <skoduri@redhat.com>
 S: Maintained
 F: api/
 
-libgfdb
-M: Dan Lambright <dlambrig@redhat.com>
-S: Maintained
-F: libglusterfs/src/gfdb/
-
 libglusterfs
-M: Niels de Vos <ndevos@redhat.com>
-M: Pranith Karampuri <pkarampu@redhat.com>
+M: Amar Tumballi <amarts@gmail.com>
+M: Xavier Hernandez  <xhernandez@redhat.com>
+M: Jeff Darcy <jeff@pl.atyp.us>
+P: Kaleb Keithley <kkeithle@redhat.com>
+P: Niels de Vos <ndevos@redhat.com>
+P: Pranith Karampuri <pranith.karampuri@phonepe.com>
+P: Shyamsundar Ranganathan <srangana@redhat.com>
 S: Maintained
 F: libglusterfs/
 
-Management Daemon
-M: Krishnan Parthasarathi <kparthas@redhat.com>
-M: Kaushal Madappa <kmadapp@redhat.com>
+xxhash
+M: Aravinda VK <avishwan@redhat.com>
+M: Kotresh HR <khiremat@redhat.com>
+P: Yaniv Kaul <ykaul@redhat.com>
+S: Maintained
+F: contrib/xxhash/
+T: https://github.com/Cyan4973/xxHash.git
+
+Management Daemon - glusterd
 M: Atin Mukherjee <amukherj@redhat.com>
+M: Mohit Agrawal <moagrawa@redhat.com>
+M: Sanju Rakonde <srakonde@redhat.com>
 S: Maintained
 F: cli/
-F: xlators/mgmt/
+F: xlators/mgmt/glusterd/
+
+Protocol
+M: Niels de Vos <ndevos@redhat.com>
+P: Mohammed Rafi KC <rafi.kavungal@iternity.com>
+S: Maintained
+F: xlators/protocol/
 
 Remote Procedure Call subsystem
-M: Raghavendra Gowdappa <rgowdapp@redhat.com>
-M: Anand Avati <avati@redhat.com>
+P: Mohit Agrawal <moagrawa@redhat.com>
 S: Maintained
-F: rpc/
+F: rpc/rpc-lib/
+F: rpc/xdr/
 
 Snapshot
-M: Rajesh Joesh <rjoseph@redhat.com>
+M: Raghavendra Bhat <raghavendra@redhat.com>
+P: Mohammed Rafi KC <rafi.kavungal@iternity.com>
+P: Sunny Kumar <sunkumar@redhat.com>
 S: Maintained
 F: xlators/mgmt/glusterd/src/glusterd-snap*
 F: extras/snap-scheduler.py
 
-Distribution Specific:
-----------------------
-Build:
-M: Kaleb Keithley <kkeithle@redhat.com>
-M: Niels de Vos <ndevos@redhat.com>
+Socket subsystem
+P: Krutika Dhananjay <kdhananj@redhat.com>
+P: Milind Changire <mchangir@redhat.com>
+P: Mohammed Rafi KC <rafi.kavungal@iternity.com>
+P: Mohit Agrawal <moagrawa@redhat.com>
 S: Maintained
+F: rpc/rpc-transport/socket/
 
-Debian Packaging
-M: Patrick Matthäi <pmatthaei@debian.org>
-M: Louis Zuckerman <me@louiszuckerman.com>
+Testing - .t framework
+M: Raghavendra Talur <rtalur@redhat.com>
 S: Maintained
-W: http://packages.qa.debian.org/g/glusterfs.html
+F: tests/
 
-Fedora Packaging
-M: glusterfs-owner@fedoraproject.org
-M: Humble Chirammal <hchiramm@redhat.com>
-M: Kaleb Keithley <kkeithle@redhat.com>
-M: Lalatendu Mohanty <lmohanty@redhat.com>
+Utilities
+M: Aravinda VK <avishwan@redhat.com>
+P: Niels de Vos <ndevos@redhat.com>
+P: Raghavendra Talur <rtalur@redhat.com>
+P: Sachidanda Urs <surs@redhat.com>
+S: Maintained
+F: extras/
+
+Events APIs
+M: Aravinda VK <avishwan@redhat.com>
+S: Maintained
+F: events/
+F: libglusterfs/src/events*
+F: libglusterfs/src/eventtypes*
+F: extras/systemd/glustereventsd*
+
+Distribution Specific:
+----------------------
+Build:
 M: Niels de Vos <ndevos@redhat.com>
+M: Hari Gowtham <hgowtham@redhat.com>
+P: Anoop C S <anoopcs@redhat.com>
+P: Raghavendra Talur <rtalur@redhat.com>
+P: Rinku Kothiya <rkothiya@redhat.com>
 S: Maintained
-W: https://apps.fedoraproject.org/packages/glusterfs
-T: http://pkgs.fedoraproject.org/git/glusterfs.git
 
-FreeBSD port
-M: Harshavardhana <harsha@harshavardhana.net>
+Debian packages on download.gluster.org
+M: packaging@gluster.org
+M: Kaleb Keithley <kkeithle@redhat.com>
+P: Sheetal Pamecha <spamecha@redhat.com>
+P: Shwetha Acharya <sacharya@redhat.com>
 S: Maintained
+W: http://download.gluster.org/pub/gluster/glusterfs/LATEST/Debian/Debian.README
+T: https://github.com/gluster/glusterfs-debian.git
 
-MacOS X port
-M: Dennis Schafroth <dennis@schafroth.com>
-M: Harshavardhana <harsha@harshavardhana.net>
+OpenSuSE
+M: packaging@gluster.org
+M: Kaleb Keithley <kkeithle@redhat.com>
+P: Sheetal Pamecha <spamecha@redhat.com>
+P: Shwetha Acharya <sacharya@redhat.com>
 S: Maintained
+W: https://build.opensuse.org/repositories/home:glusterfs
+W: https://download.gluster.org/pub/gluster/glusterfs/LATEST/SuSE/SuSE.README
+T: https://github.com/gluster/glusterfs-suse.git
 
-NetBSD port
-M: Emmanuel Dreyfus <manu@netbsd.org>
+Packages for the CentOS Storage SIG
+M: centos-devel@centos.org
+M: Niels de Vos <ndevos@redhat.com>
+P: Kaleb Keithley <kkeithle@redhat.com>
 S: Maintained
-W: http://pkgsrc.se/filesystems/glusterfs
+W: https://wiki.centos.org/SpecialInterestGroup/Storage/Gluster
+T: https://github.com/CentOS-Storage-SIG/glusterfs.git
 
-Ubuntu Packaging
-M: Louis Zuckerman <me@louiszuckerman.com>
+Ubuntu PPA
+M: packaging@gluster.org
+M: Kaleb Keithley <kkeithle@redhat.com>
+P: Sheetal Pamecha <spamecha@redhat.com>
+P: Shwetha Acharya <sacharya@redhat.com>
 S: Maintained
+W: https://launchpad.net/~gluster
 W: http://download.gluster.org/pub/gluster/glusterfs/LATEST/Ubuntu/Ubuntu.README
-
+T: https://github.com/gluster/glusterfs-debian.git
 
 Related projects
 ----------------
-Gluster Openstack Swift
-M: Luis Pabon <lpabon@redhat.com>
+Gluster Block
+M: Prasanna Kumar Kalever <prasanna.kalever@redhat.com>
+M: Xiubo Li <xiubli@redhat.com>
 S: Maintained
-T: https://github.com/gluster/gluster-swift.git
+T: https://github.com/gluster/gluster-block.git
 
-GlusterFS Hadoop HCFS plugin
-M: Jay Vyas <jvyas@redhat.com>
+GlusterFS core-utils
+M: Anoop C S <anoopcs@redhat.com>
 S: Maintained
-T: https://github.com/gluster/glusterfs-hadoop.git
+T: https://github.com/gluster/glusterfs-coreutils.git
 
 NFS-Ganesha FSAL plugin
-M: Anand Subramanian <ansubram@redhat.com>
+M: Jiffin Tony Thottan <jthottan@redhat.com>
+M: Kaleb Keithley <kkeithle@redhat.com>
+M: Soumya Koduri <skoduri@redhat.com>
 S: Maintained
 T: git://github.com/nfs-ganesha/nfs-ganesha.git
 F: src/nfs-ganesha~/src/FSAL/FSAL_GLUSTER/
 
 QEMU integration
-M: Bharata B Rao <bharata@linux.vnet.ibm.com>
+M: Niels de Vos <ndevos@redhat.com>
+M: Prasanna Kumar Kalever <prasanna.kalever@redhat.com>
 S: Maintained
 T: git://git.qemu.org/qemu.git
 F: block/gluster.c
 
 Samba VFS plugin
+M: Anoop C S <anoopcs@redhat.com>
 M: Raghavendra Talur <rtalur@redhat.com>
-M: Jose Rivera <jrivera@redhat.com>
-M: Ira Cooper <icooper@redhat.com>
+M: Michael Adam <madam@redhat.com>
+M: Poornima G <pgurusid@redhat.com>
 S: Maintained
 T: git://git.samba.org/samba.git
 F: source3/modules/vfs_glusterfs.c
 
+Storhaug
+M: Jose A. Rivera <jarrpa@redhat.com>
+P: Kaleb Keithley <kkeithle@redhat.com>
+S: Maintained
+T: https://github.com/linux-ha-storage/storhaug.git
+
+Testing - Glusto-Tests
+M: Jonathan Holloway <jholloway@redhat.com>
+M: Vijay Bhaskar Reddy Avuthu <vavuthu@redhat.com>
+M: Akarsha Rai <akrai@redhat.com>
+S: Maintained
+T: https://github.com/gluster/glusto-tests.git
+
 Wireshark dissectors
 M: Niels de Vos <ndevos@redhat.com>
 S: Maintained
@@ -300,14 +456,55 @@ W: https://forge.gluster.org/wireshark
 T: http://code.wireshark.org/git/wireshark
 F: epan/dissectors/packet-gluster*
 
+Infrastructure
+--------------
+
+Platform
+M: Michael Scherer <misc@redhat.com>
+P: Shyamsundar Ranganathan <srangana@redhat.com>
+P: Amar Tumballi <amarts@gmail.com>
+
+Continuous Integration
+M: Michael Scherer <misc@redhat.com>
+M: Deepshikha Khandelwal <dkhandel@redhat.com>
+P: Niels de Vos <ndevos@redhat.com>
+
 Special Thanks
 --------------
 
 GlusterFS would not be possible without the contributions of:
 
-M: Amar Tumballi <amarts@gmail.com>
-M: Chris Hertel <chertel@redhat.com>
+
+M: Vijay Bellur <vbellur@redhat.com>
+M: Jeff Darcy <jeff@pl.atyp.us>
+M: Shreyas Siravara <sshreyas@fb.com>
+M: Kaushal M <kaushal@redhat.com>
+M: Nigel Babu
+M: Prashanth Pai
+P: Sanoj Unnikrishnan
+P: Milind Changire <mchangir@redhat.com>
+P: Sunil Kumar Acharya <sheggodu@redhat.com>
+M: Samikshan Bairagya <samikshan@gmail.com>
+M: Chris Hertel
 M: M. Mohan Kumar <mohan@in.ibm.com>
 M: Shishir Gowda <gowda.shishir@gmail.com>
 M: Brian Foster <bfoster@redhat.com>
-M: Csaba Henk <chenk@redhat.com>
+M: Anand Avati <avati@cs.stanford.edu>
+M: Dennis Schafroth <dennis@schafroth.com>
+M: Harshavardhana <harsha@harshavardhana.net>
+M: Krishnan Parthasarathi
+M: Justin Clift <justin@gluster.org>
+M: Venky Shankar <vshankar@redhat.com>
+M: Shravan Chandrashekar <shravantc99@gmail.com>
+M: Joseph Fernandes
+M: Vijaikumar Mallikarjuna
+M: Anand Subramanian
+M: Bharata B Rao <bharata@linux.vnet.ibm.com>
+M: Rajesh Joseph
+M: Dan Lambright
+M: Jay Vyas
+M: Luis Pabon
+M: Ira Cooper
+M: Shwetha Panduranga
+M: Nithya Balachandran
+M: Raghavendra Gowdappa
diff --git a/Makefile.am b/Makefile.am
index 1201ca2319b..98ea5c1038d 100644
--- a/Makefile.am
+++ b/Makefile.am
@@ -1,56 +1,52 @@
+SOURCES = site.h
+
 EXTRA_DIST = autogen.sh \
-	COPYING-GPLV2 COPYING-LGPLV3 \
+	COPYING-GPLV2 COPYING-LGPLV3 COMMITMENT \
 	INSTALL README.md AUTHORS THANKS NEWS \
-	glusterfs.spec glusterfs-api.pc.in libgfchangelog.pc.in libgfdb.pc.in \
+	glusterfs.spec glusterfs-api.pc.in libgfchangelog.pc.in \
 	run-tests.sh \
 	build-aux/pkg-version \
-	build-aux/xdrgen \
-        contrib/argp-standalone \
-        contrib/umountd \
-        contrib/uuid \
+	contrib/umountd \
 	$(shell find $(top_srcdir)/tests -type f -print)
 
-SUBDIRS = $(ARGP_STANDALONE_DIR) libglusterfs rpc api xlators glusterfsd \
-	$(FUSERMOUNT_SUBDIR) doc extras cli heal @SYNCDAEMON_SUBDIR@ \
-	@UMOUNTD_SUBDIR@ tools
+
+SUBDIRS = $(ARGP_STANDALONE_DIR) libglusterfs rpc libglusterd api \
+	glusterfsd xlators $(FUSERMOUNT_SUBDIR) doc extras cli heal \
+	@SYNCDAEMON_SUBDIR@ @UMOUNTD_SUBDIR@ tools events
 
 pkgconfigdir = @pkgconfigdir@
 pkgconfig_DATA = glusterfs-api.pc libgfchangelog.pc
-if USE_GFDB
-pkgconfig_DATA += libgfdb.pc
-endif
 
-CLEANFILES = glusterfs-api.pc libgfchangelog.pc libgfdb.pc \
-	tests/env.rc contrib/umountd/Makefile{,.in}
-CONFIG_CLEAN_FILES = $(CONTRIB_BUILDDIR)/uuid/uuid_types.h
+CLEANFILES = glusterfs-api.pc libgfchangelog.pc contrib/umountd/Makefile
+
+clean-local:
+	find . -name '*.o' -o -name '*.lo' -o -name '.Po' | xargs rm -f
 
 gitclean: distclean
 	find . -name Makefile.in -exec rm -f {} \;
 	find . -name mount.glusterfs -exec rm -f {} \;
+	find . -name .deps -o -name .libs | xargs rm -rf
 	rm -fr autom4te.cache
 	rm -f missing aclocal.m4 config.h.in config.guess config.sub ltmain.sh install-sh configure depcomp
-	-rm -fr $(CONTRIBDIR)/argp-standalone/autom4te.cache
-	-rm -f $(CONTRIBDIR)/argp-standalone/aclocal.m4
-	-rm -f $(CONTRIBDIR)/argp-standalone/config.h.in
-	-rm -f $(CONTRIBDIR)/argp-standalone/configure
-	-rm -f $(CONTRIBDIR)/argp-standalone/config.status
-	-rm -f $(CONTRIBDIR)/argp-standalone/config.log
-	-rm -f $(CONTRIBDIR)/argp-standalone/depcomp
-	-rm -fr $(CONTRIBDIR)/argp-standalone/.deps
-	-rm -f $(CONTRIBDIR)/argp-standalone/install-sh
-	-rm -f $(CONTRIBDIR)/argp-standalone/missing
 
+# dist-hook gets executed with 'make dist', this is the only target getting
+# executed, a dist-hook in other Makefile.am files seem to get ignored.
 dist-hook: gen-VERSION gen-ChangeLog
-	-rm -fr $(distdir)/contrib/argp-standalone/autom4te.cache
-	-rm -fr $(distdir)/contrib/argp-standalone/.deps
 	-rm -fr $(distdir)/contrib/umountd/.deps
-	-rm -fr $(distdir)/config.{guess,sub}
+	-rm -f $(distdir)/events/src/eventtypes.py
+	-rm -f $(distdir)/tests/env.rc
+	-cp -f $(top_srcdir)/build-aux/config.sub.dist $(distdir)/config.sub
+	-cp -f $(top_srcdir)/build-aux/config.guess.dist $(distdir)/config.guess
+
+.PHONY: gen-VERSION gen-ChangeLog clang-check
 
-.PHONY: gen-VERSION gen-ChangeLog
+clang-check:
+	@$(top_srcdir)/extras/clang-checker.sh
 
 gen-ChangeLog:
 	(cd $(srcdir) && git diff && echo ===== git log ==== && git log) > $(distdir)/ChangeLog
 
+.PHONY : gen-VERSION
 gen-VERSION:
 	if test -d $(top_srcdir)/.git; then             \
 	  cd $(top_srcdir);                             \
diff --git a/README.md b/README.md
index 0e2c2b7d2b4..9d68e033782 100644
--- a/README.md
+++ b/README.md
@@ -1,10 +1,46 @@
-For information about contributing to GlusterFS, please follow the below link :
-[Contributing to GlusterFS community](http://www.gluster.org/community/documentation/index.php/Main_Page#Contributing_to_the_Gluster_Community)
+# Gluster
+  Gluster is a software defined distributed storage that can scale to several
+  petabytes. It provides interfaces for object, block and file storage.
 
-*GlusterFS does not follow the [GitHub: Fork & pull](https://help.github.com/articles/using-pull-requests/) workflow but use [Gerrit](http://review.gluster.org) for code review.*
+## Development
+  The development workflow is documented in [Contributors guide](CONTRIBUTING.md)
 
-The development guidelines are detailed in [Development Workflow.](http://www.gluster.org/community/documentation/index.php/Simplified_dev_workflow)
+## Documentation
+  The Gluster documentation can be found at [Gluster Docs](http://docs.gluster.org).
 
-The GlusterFS documentation can be found at [Documentation](http://gluster.readthedocs.org/en/latest)
+## Deployment
+  Quick instructions to build and install can be found in [INSTALL](INSTALL) file.
 
-For more info, please visit http://www.gluster.org/.
+## Testing
+
+  GlusterFS source contains some functional tests under `tests/` directory. All
+  these tests are run against every patch submitted for review. If you want your
+  patch to be tested, please add a `.t` test file as part of your patch submission.
+  You can also submit a patch to only add a `.t` file for the test case you are
+  aware of.
+
+  To run these tests, on your test-machine, just run `./run-tests.sh`. Don't run
+  this on a machine where you have 'production' glusterfs is running, as it would
+  blindly kill all gluster processes in each runs.
+
+  If you are sending a patch, and want to validate one or few specific tests, then
+  run a single test by running the below command.
+
+```
+  bash# /bin/bash ${path_to_gluster}/tests/basic/rpc-coverage.t
+```
+
+  You can also use `prove` tool if available in your machine, as follows.
+
+```
+  bash# prove -vmfe '/bin/bash' ${path_to_gluster}/tests/basic/rpc-coverage.t
+```
+
+
+## Maintainers
+  The list of Gluster maintainers is available in [MAINTAINERS](MAINTAINERS) file.
+
+## License
+  Gluster is dual licensed under [GPLV2](COPYING-GPLV2) and [LGPLV3+](COPYING-LGPLV3).
+
+  Please visit the [Gluster Home Page](http://www.gluster.org/) to find out more about Gluster.
diff --git a/api/examples/getvolfile.py b/api/examples/getvolfile.py
index 184586c632d..3b2c8ab5a15 100755
--- a/api/examples/getvolfile.py
+++ b/api/examples/getvolfile.py
@@ -1,43 +1,45 @@
-#!/usr/bin/python
+#!/usr/bin/python3
 
+from __future__ import print_function
 import ctypes
 import ctypes.util
 
-api = ctypes.CDLL(ctypes.util.find_library("gfapi"))
+api = ctypes.CDLL("libgfapi.so")
 api.glfs_get_volfile.argtypes = [ctypes.c_void_p,
                                  ctypes.c_void_p,
                                  ctypes.c_ulong]
-api.glfs_get_volfile.restype = ctypes.c_long;
+api.glfs_get_volfile.restype = ctypes.c_long
 
-def get_volfile (host, volume):
-	# This is set to a large value to exercise the "buffer not big enough"
-	# path.  More realistically, you'd just start with a huge buffer.
-	BUF_LEN = 0
-	fs = api.glfs_new(volume)
-	#api.glfs_set_logging(fs,"/dev/stderr",7)
-	api.glfs_set_volfile_server(fs,"tcp",host,24007)
-	api.glfs_init(fs)
-	vbuf = ctypes.create_string_buffer(BUF_LEN)
-	vlen = api.glfs_get_volfile(fs,vbuf,BUF_LEN)
-	if vlen < 0:
-		vlen = BUF_LEN - vlen
-		vbuf = ctypes.create_string_buffer(vlen)
-		vlen = api.glfs_get_volfile(fs,vbuf,vlen)
-	api.glfs_fini(fs)
-	if vlen <= 0:
-		return vlen
-	return vbuf.value[:vlen]
+
+def get_volfile(host, volume):
+    # This is set to a large value to exercise the "buffer not big enough"
+    # path.  More realistically, you'd just start with a huge buffer.
+    BUF_LEN = 0
+    fs = api.glfs_new(volume)
+    # api.glfs_set_logging(fs,"/dev/stderr",7)
+    api.glfs_set_volfile_server(fs, "tcp", host, 24007)
+    api.glfs_init(fs)
+    vbuf = ctypes.create_string_buffer(BUF_LEN)
+    vlen = api.glfs_get_volfile(fs, vbuf, BUF_LEN)
+    if vlen < 0:
+        vlen = BUF_LEN - vlen
+        vbuf = ctypes.create_string_buffer(vlen)
+        vlen = api.glfs_get_volfile(fs, vbuf, vlen)
+    api.glfs_fini(fs)
+    if vlen <= 0:
+        return vlen
+    return vbuf.value[:vlen]
 
 if __name__ == "__main__":
-	import sys
+    import sys
 
-	try:
-		res = apply(get_volfile,sys.argv[1:3])
-	except:
-		print "fetching volfile failed (volume not started?)"
+    try:
+        res = get_volfile(*sys.argv[1:3])
+    except:
+        print("fetching volfile failed (volume not started?)")
 
-	try:
-		for line in res.split('\n'):
-			print line
-	except:
-		print "bad return value %s" % res
+    try:
+        for line in res.split('\n'):
+            print(line)
+    except:
+        print("bad return value %s" % res)
diff --git a/api/examples/glfsxmp.c b/api/examples/glfsxmp.c
index 7ff3f0eb7ee..a55616ef739 100644
--- a/api/examples/glfsxmp.c
+++ b/api/examples/glfsxmp.c
@@ -1,1598 +1,1811 @@
 #include <stdio.h>
 #include <stdlib.h>
 #include <errno.h>
-#include "api/glfs.h"
-#include "api/glfs-handles.h"
+#include <glusterfs/api/glfs.h>
+#include <glusterfs/api/glfs-handles.h>
 #include <string.h>
 #include <time.h>
 
+#define TEST_STR_LEN 2048
 
 int
-test_dirops (glfs_t *fs)
+test_dirops(glfs_t *fs)
 {
-        glfs_fd_t *fd = NULL;
-        char buf[512];
-        struct dirent *entry = NULL;
-
-        fd = glfs_opendir (fs, "/");
-        if (!fd) {
-                fprintf (stderr, "/: %s\n", strerror (errno));
-                return -1;
-        }
-
-        fprintf (stderr, "Entries:\n");
-        while (glfs_readdir_r (fd, (struct dirent *)buf, &entry), entry) {
-                fprintf (stderr, "%s: %lu\n", entry->d_name, glfs_telldir (fd));
-        }
-
-        glfs_closedir (fd);
-        return 0;
+    glfs_fd_t *fd = NULL;
+    char buf[512];
+    struct dirent *entry = NULL;
+
+    fd = glfs_opendir(fs, "/");
+    if (!fd) {
+        fprintf(stderr, "/: %s\n", strerror(errno));
+        return -1;
+    }
+
+    fprintf(stderr, "Entries:\n");
+    while (glfs_readdir_r(fd, (struct dirent *)buf, &entry), entry) {
+        fprintf(stderr, "%s: %lu\n", entry->d_name, glfs_telldir(fd));
+    }
+
+    glfs_closedir(fd);
+    return 0;
 }
 
-
 int
-test_xattr (glfs_t *fs)
+test_xattr(glfs_t *fs)
 {
-        char *filename = "/filename2";
-        char buf[512];
-        char *ptr;
-        int ret;
-
-        ret = glfs_setxattr (fs, filename, "user.testkey", "testval", 8, 0);
-        fprintf (stderr, "setxattr(%s): %d (%s)\n", filename, ret,
-                 strerror (errno));
-
-        ret = glfs_setxattr (fs, filename, "user.testkey2", "testval", 8, 0);
-        fprintf (stderr, "setxattr(%s): %d (%s)\n", filename, ret,
-                 strerror (errno));
-
-        ret = glfs_listxattr (fs, filename, buf, 512);
-        fprintf (stderr, "listxattr(%s): %d (%s)\n", filename, ret,
-                 strerror (errno));
-        if (ret < 0)
-                return -1;
-
-        for (ptr = buf; ptr < buf + ret; ptr++) {
-                printf ("key=%s\n", ptr);
-                ptr += strlen (ptr);
-        }
-
-        return 0;
+    char *filename = "/filename2";
+    char *linkfile = "/linkfile";
+    glfs_fd_t *fd = NULL;
+    char buf[512];
+    char *ptr;
+    int ret;
+
+    ret = glfs_setxattr(fs, filename, "user.testkey", "testval", 8, 0);
+    fprintf(stderr, "setxattr(%s): %d (%s)\n", filename, ret, strerror(errno));
+
+    ret = glfs_setxattr(fs, filename, "user.testkey2", "testval", 8, 0);
+    fprintf(stderr, "setxattr(%s): %d (%s)\n", filename, ret, strerror(errno));
+
+    ret = glfs_getxattr(fs, filename, "user.testkey", buf, 512);
+    fprintf(stderr, "getxattr(%s): %d (%s)\n", filename, ret, strerror(errno));
+    if (ret < 0)
+        return -1;
+
+    ret = glfs_listxattr(fs, filename, buf, 512);
+    fprintf(stderr, "listxattr(%s): %d (%s)\n", filename, ret, strerror(errno));
+    if (ret < 0)
+        return -1;
+
+    ret = glfs_symlink(fs, "filename", linkfile);
+    fprintf(stderr, "symlink(%s %s): %s\n", filename, linkfile,
+            strerror(errno));
+    if (ret < 0)
+        return -1;
+
+    ret = glfs_readlink(fs, linkfile, buf, 512);
+    fprintf(stderr, "readlink(%s) : %d (%s)\n", filename, ret, strerror(errno));
+    if (ret < 0)
+        return -1;
+
+    ret = glfs_lsetxattr(fs, filename, "user.testkey3", "testval", 8, 0);
+    fprintf(stderr, "lsetxattr(%s) : %d (%s)\n", linkfile, ret,
+            strerror(errno));
+    if (ret < 0)
+        return -1;
+
+    ret = glfs_llistxattr(fs, linkfile, buf, 512);
+    fprintf(stderr, "llistxattr(%s): %d (%s)\n", filename, ret,
+            strerror(errno));
+    if (ret < 0)
+        return -1;
+
+    ret = glfs_lgetxattr(fs, filename, "user.testkey3", buf, 512);
+    fprintf(stderr, "lgetxattr(%s): %d (%s)\n", linkfile, ret, strerror(errno));
+    if (ret < 0)
+        return -1;
+
+    for (ptr = buf; ptr < buf + ret; ptr++) {
+        printf("key=%s\n", ptr);
+        ptr += strlen(ptr);
+    }
+
+    ret = glfs_removexattr(fs, filename, "user.testkey2");
+    fprintf(stderr, "removexattr(%s): %d (%s)\n", filename, ret,
+            strerror(errno));
+
+    fd = glfs_open(fs, filename, O_RDWR);
+    fprintf(stderr, "open(%s): (%p) %s\n", filename, fd, strerror(errno));
+
+    ret = glfs_fsetxattr(fd, "user.testkey2", "testval", 8, 0);
+    fprintf(stderr, "fsetxattr(%s): %d (%s)\n", filename, ret, strerror(errno));
+
+    ret = glfs_fgetxattr(fd, "user.testkey2", buf, 512);
+    fprintf(stderr, "fgetxattr(%s): %d (%s)\n", filename, ret, strerror(errno));
+
+    ret = glfs_flistxattr(fd, buf, 512);
+    fprintf(stderr, "flistxattr(%s): %d (%s)\n", filename, ret,
+            strerror(errno));
+    if (ret < 0)
+        return -1;
+
+    for (ptr = buf; ptr < buf + ret; ptr++) {
+        printf("key=%s\n", ptr);
+        ptr += strlen(ptr);
+    }
+
+    ret = glfs_fremovexattr(fd, "user.testkey2");
+    fprintf(stderr, "fremovexattr(%s): %d (%s)\n", filename, ret,
+            strerror(errno));
+
+    glfs_close(fd);
+
+    return 0;
 }
 
-
 int
-test_chdir (glfs_t *fs)
+test_chdir(glfs_t *fs)
 {
-        int ret = -1;
-        char *topdir = "/topdir";
-        char *linkdir = "/linkdir";
-        char *subdir = "./subdir";
-        char *respath = NULL;
-        char pathbuf[4096];
-
-        ret = glfs_mkdir (fs, topdir, 0755);
-        if (ret) {
-                fprintf (stderr, "mkdir(%s): %s\n", topdir, strerror (errno));
-                return -1;
-        }
-
-        respath = glfs_getcwd (fs, pathbuf, 4096);
-        fprintf (stdout, "getcwd() = %s\n", respath);
-
-        ret = glfs_symlink (fs, topdir, linkdir);
-        if (ret) {
-                fprintf (stderr, "symlink(%s, %s): %s\n", topdir, linkdir, strerror (errno));
-                return -1;
-        }
-
-        ret = glfs_chdir (fs, linkdir);
-        if (ret) {
-                fprintf (stderr, "chdir(%s): %s\n", linkdir, strerror (errno));
-                return -1;
-        }
-
-        respath = glfs_getcwd (fs, pathbuf, 4096);
-        fprintf (stdout, "getcwd() = %s\n", respath);
-
-        respath = glfs_realpath (fs, subdir, pathbuf);
-        if (respath) {
-                fprintf (stderr, "realpath(%s) worked unexpectedly: %s\n", subdir, respath);
-                return -1;
-        }
-
-        ret = glfs_mkdir (fs, subdir, 0755);
-        if (ret) {
-                fprintf (stderr, "mkdir(%s): %s\n", subdir, strerror (errno));
-                return -1;
-        }
-
-        respath = glfs_realpath (fs, subdir, pathbuf);
-        if (!respath) {
-                fprintf (stderr, "realpath(%s): %s\n", subdir, strerror (errno));
-        } else {
-                fprintf (stdout, "realpath(%s) = %s\n", subdir, respath);
-        }
-
-        ret = glfs_chdir (fs, subdir);
-        if (ret) {
-                fprintf (stderr, "chdir(%s): %s\n", subdir, strerror (errno));
-                return -1;
-        }
-
-        respath = glfs_getcwd (fs, pathbuf, 4096);
-        fprintf (stdout, "getcwd() = %s\n", respath);
-
-        respath = glfs_realpath (fs, "/linkdir/subdir", pathbuf);
-        if (!respath) {
-                fprintf (stderr, "realpath(/linkdir/subdir): %s\n", strerror (errno));
-        } else {
-                fprintf (stdout, "realpath(/linkdir/subdir) = %s\n", respath);
-        }
-
-        return 0;
+    int ret = -1;
+    char *dir = "/dir";
+    char *topdir = "/topdir";
+    char *linkdir = "/linkdir";
+    char *linkdir2 = "/linkdir2";
+    char *subdir = "./subdir";
+    char *respath = NULL;
+    char pathbuf[4096];
+
+    ret = glfs_mkdir(fs, topdir, 0755);
+    fprintf(stderr, "mkdir(%s): %s\n", topdir, strerror(errno));
+    if (ret)
+        return -1;
+
+    ret = glfs_mkdir(fs, dir, 0755);
+    fprintf(stderr, "mkdir(%s): %s\n", dir, strerror(errno));
+    if (ret)
+        return -1;
+
+    respath = glfs_getcwd(fs, pathbuf, 4096);
+    fprintf(stdout, "getcwd() = %s\n", respath);
+
+    ret = glfs_symlink(fs, "topdir", linkdir);
+    if (ret) {
+        fprintf(stderr, "symlink(%s, %s): %s\n", topdir, linkdir,
+                strerror(errno));
+        return -1;
+    }
+
+    ret = glfs_chdir(fs, linkdir);
+    if (ret) {
+        fprintf(stderr, "chdir(%s): %s\n", linkdir, strerror(errno));
+        return -1;
+    }
+
+    respath = glfs_getcwd(fs, pathbuf, 4096);
+    fprintf(stdout, "getcwd() = %s\n", respath);
+
+    respath = glfs_realpath(fs, subdir, pathbuf);
+    if (respath) {
+        fprintf(stderr, "realpath(%s) worked unexpectedly: %s\n", subdir,
+                respath);
+        return -1;
+    }
+
+    ret = glfs_mkdir(fs, subdir, 0755);
+    if (ret) {
+        fprintf(stderr, "mkdir(%s): %s\n", subdir, strerror(errno));
+        return -1;
+    }
+
+    respath = glfs_realpath(fs, subdir, pathbuf);
+    if (!respath) {
+        fprintf(stderr, "realpath(%s): %s\n", subdir, strerror(errno));
+    } else {
+        fprintf(stdout, "realpath(%s) = %s\n", subdir, respath);
+    }
+
+    ret = glfs_chdir(fs, subdir);
+    if (ret) {
+        fprintf(stderr, "chdir(%s): %s\n", subdir, strerror(errno));
+        return -1;
+    }
+
+    respath = glfs_getcwd(fs, pathbuf, 4096);
+    fprintf(stdout, "getcwd() = %s\n", respath);
+
+    respath = glfs_realpath(fs, "/linkdir/subdir", pathbuf);
+    if (!respath) {
+        fprintf(stderr, "realpath(/linkdir/subdir): %s\n", strerror(errno));
+    } else {
+        fprintf(stdout, "realpath(/linkdir/subdir) = %s\n", respath);
+    }
+
+    return 0;
 }
 
 #ifdef DEBUG
 static void
-peek_stat (struct stat *sb)
+peek_stat(struct stat *sb)
 {
-        printf ("Dumping stat information:\n");
-        printf ("File type:                ");
-
-        switch (sb->st_mode & S_IFMT) {
-                case S_IFBLK:  printf ("block device\n");            break;
-                case S_IFCHR:  printf ("character device\n");        break;
-                case S_IFDIR:  printf ("directory\n");               break;
-                case S_IFIFO:  printf ("FIFO/pipe\n");               break;
-                case S_IFLNK:  printf ("symlink\n");                 break;
-                case S_IFREG:  printf ("regular file\n");            break;
-                case S_IFSOCK: printf ("socket\n");                  break;
-                default:       printf ("unknown?\n");                break;
-        }
-
-        printf ("I-node number:            %ld\n", (long) sb->st_ino);
-
-        printf ("Mode:                     %lo (octal)\n",
-                (unsigned long) sb->st_mode);
-
-        printf ("Link count:               %ld\n", (long) sb->st_nlink);
-        printf ("Ownership:                UID=%ld   GID=%ld\n",
-                (long) sb->st_uid, (long) sb->st_gid);
-
-        printf ("Preferred I/O block size: %ld bytes\n",
-                (long) sb->st_blksize);
-        printf ("File size:                %lld bytes\n",
-                (long long) sb->st_size);
-        printf ("Blocks allocated:         %lld\n",
-                (long long) sb->st_blocks);
-
-        printf ("Last status change:       %s", ctime(&sb->st_ctime));
-        printf ("Last file access:         %s", ctime(&sb->st_atime));
-        printf ("Last file modification:   %s", ctime(&sb->st_mtime));
-
-        return;
+    printf("Dumping stat information:\n");
+    printf("File type:                ");
+
+    switch (sb->st_mode & S_IFMT) {
+        case S_IFBLK:
+            printf("block device\n");
+            break;
+        case S_IFCHR:
+            printf("character device\n");
+            break;
+        case S_IFDIR:
+            printf("directory\n");
+            break;
+        case S_IFIFO:
+            printf("FIFO/pipe\n");
+            break;
+        case S_IFLNK:
+            printf("symlink\n");
+            break;
+        case S_IFREG:
+            printf("regular file\n");
+            break;
+        case S_IFSOCK:
+            printf("socket\n");
+            break;
+        default:
+            printf("unknown?\n");
+            break;
+    }
+
+    printf("I-node number:            %ld\n", (long)sb->st_ino);
+
+    printf("Mode:                     %lo (octal)\n",
+           (unsigned long)sb->st_mode);
+
+    printf("Link count:               %ld\n", (long)sb->st_nlink);
+    printf("Ownership:                UID=%ld   GID=%ld\n", (long)sb->st_uid,
+           (long)sb->st_gid);
+
+    printf("Preferred I/O block size: %ld bytes\n", (long)sb->st_blksize);
+    printf("File size:                %lld bytes\n", (long long)sb->st_size);
+    printf("Blocks allocated:         %lld\n", (long long)sb->st_blocks);
+
+    printf("Last status change:       %s", ctime(&sb->st_ctime));
+    printf("Last file access:         %s", ctime(&sb->st_atime));
+    printf("Last file modification:   %s", ctime(&sb->st_mtime));
+
+    return;
 }
 
 static void
-peek_handle (unsigned char *glid)
+peek_handle(unsigned char *glid)
 {
-        int i;
+    int i;
 
-        for (i = 0; i < GFAPI_HANDLE_LENGTH; i++)
-        {
-                printf (":%02x:", glid[i]);
-        }
-        printf ("\n");
+    for (i = 0; i < GFAPI_HANDLE_LENGTH; i++) {
+        printf(":%02x:", glid[i]);
+    }
+    printf("\n");
 }
-#else /* DEBUG */
+#else  /* DEBUG */
 static void
-peek_stat (struct stat *sb)
+peek_stat(struct stat *sb)
 {
-        return;
+    return;
 }
 
 static void
-peek_handle (unsigned char *id)
+peek_handle(unsigned char *id)
 {
-        return;
+    return;
 }
 #endif /* DEBUG */
 
-glfs_t    *fs = NULL;
-char      *full_parent_name = "/testdir", *parent_name = "testdir";
+glfs_t *fs = NULL;
+char *full_parent_name = "/testdir", *parent_name = "testdir";
 
 void
-test_h_unlink (void)
+test_h_unlink(void)
 {
-        char               *my_dir = "unlinkdir";
-        char               *my_file = "file.txt";
-        char               *my_subdir = "dir1";
-        struct glfs_object *parent = NULL, *leaf = NULL, *dir = NULL,
-                           *subdir = NULL, *subleaf = NULL;
-        struct stat         sb;
-        int                 ret;
-
-        printf ("glfs_h_unlink tests: In Progress\n");
-
-        /* Prepare tests */
-        parent = glfs_h_lookupat (fs, NULL, full_parent_name, &sb, 0);
-        if (parent == NULL) {
-                fprintf (stderr, "glfs_h_lookupat: error on lookup of %s: from (%p),%s\n",
-                         full_parent_name, NULL, strerror (errno));
-                printf ("glfs_h_lookupat tests: FAILED\n");
-                goto out;
-        }
-        peek_stat (&sb);
-
-        dir = glfs_h_mkdir (fs, parent, my_dir, 0644, &sb);
-        if (dir == NULL) {
-                fprintf (stderr, "glfs_h_mkdir: error creating %s: from (%p),%s\n",
-                         my_dir, parent, strerror (errno));
-                printf ("glfs_h_unlink tests: FAILED\n");
-                goto out;
-        }
-
-        leaf = glfs_h_creat (fs, dir, my_file, O_CREAT, 0644, &sb);
-        if (leaf == NULL) {
-                fprintf (stderr, "glfs_h_creat: error creating %s: from (%p),%s\n",
-                         my_file, dir, strerror (errno));
-                printf ("glfs_h_unlink tests: FAILED\n");
-                goto out;
-        }
-
-        subdir = glfs_h_mkdir (fs, dir, my_subdir, 0644, &sb);
-        if (subdir == NULL) {
-                fprintf (stderr, "glfs_h_mkdir: error creating %s: from (%p),%s\n",
-                         my_subdir, dir, strerror (errno));
-                printf ("glfs_h_unlink tests: FAILED\n");
-                goto out;
-        }
-
-        subleaf = glfs_h_creat (fs, subdir, my_file, O_CREAT, 0644, &sb);
-        if (subleaf == NULL) {
-                fprintf (stderr, "glfs_h_creat: error creating %s: from (%p),%s\n",
-                         my_file, subdir, strerror (errno));
-                printf ("glfs_h_unlink tests: FAILED\n");
-                goto out;
-        }
-
-        /* unlink non empty directory */
-        ret = glfs_h_unlink (fs, dir, my_subdir);
-        if ((ret && errno != ENOTEMPTY) || (ret == 0)) {
-                fprintf (stderr, "glfs_h_unlink: error unlinking %s: it is non empty: %s\n",
-                         my_subdir, strerror (errno));
-                printf ("glfs_h_unlink tests: FAILED\n");
-                goto out;
-        }
-
-        /* unlink regular file */
-        ret = glfs_h_unlink (fs, subdir, my_file);
-        if (ret) {
-                fprintf (stderr, "glfs_h_unlink: error unlinking %s: from (%p),%s\n",
-                         my_file, subdir, strerror (errno));
-                printf ("glfs_h_unlink tests: FAILED\n");
-                goto out;
-        }
-
-        /* unlink directory */
-        ret = glfs_h_unlink (fs, dir, my_subdir);
-        if (ret) {
-                fprintf (stderr, "glfs_h_unlink: error unlinking %s: from (%p),%s\n",
-                         my_subdir, dir, strerror (errno));
-                printf ("glfs_h_unlink tests: FAILED\n");
-                goto out;
-        }
-
-        /* unlink regular file */
-        ret = glfs_h_unlink (fs, dir, my_file);
-        if (ret) {
-                fprintf (stderr, "glfs_h_unlink: error unlinking %s: from (%p),%s\n",
-                         my_file, dir, strerror (errno));
-                printf ("glfs_h_unlink tests: FAILED\n");
-                goto out;
-        }
-
-        /* unlink non-existent regular file */
-        ret = glfs_h_unlink (fs, dir, my_file);
-        if ((ret && errno != ENOENT) || (ret == 0)) {
-                fprintf (stderr, "glfs_h_unlink: error unlinking non-existent %s: invalid errno ,%d, %s\n",
-                         my_file, ret, strerror (errno));
-                printf ("glfs_h_unlink tests: FAILED\n");
-                goto out;
-        }
-
-        /* unlink non-existent directory */
-        ret = glfs_h_unlink (fs, dir, my_subdir);
-        if ((ret && errno != ENOENT) || (ret == 0)) {
-                fprintf (stderr, "glfs_h_unlink: error unlinking non-existent %s:  invalid errno ,%d, %s\n",
-                         my_subdir, ret, strerror (errno));
-                printf ("glfs_h_unlink tests: FAILED\n");
-                goto out;
-        }
-
-        /* unlink directory */
-        ret = glfs_h_unlink (fs, parent, my_dir);
-        if (ret) {
-                fprintf (stderr, "glfs_h_unlink: error unlinking %s: from (%p),%s\n",
-                         my_dir, dir, strerror (errno));
-                printf ("glfs_h_unlink tests: FAILED\n");
-                goto out;
-        }
-
-        printf ("glfs_h_unlink tests: PASSED\n");
+    char *my_dir = "unlinkdir";
+    char *my_file = "file.txt";
+    char *my_subdir = "dir1";
+    struct glfs_object *parent = NULL, *leaf = NULL, *dir = NULL,
+                       *subdir = NULL, *subleaf = NULL;
+    struct stat sb;
+    int ret;
+
+    printf("glfs_h_unlink tests: In Progress\n");
+
+    /* Prepare tests */
+    parent = glfs_h_lookupat(fs, NULL, full_parent_name, &sb, 0);
+    if (parent == NULL) {
+        fprintf(stderr,
+                "glfs_h_lookupat: error on lookup of %s: from (%p),%s\n",
+                full_parent_name, NULL, strerror(errno));
+        printf("glfs_h_lookupat tests: FAILED\n");
+        goto out;
+    }
+    peek_stat(&sb);
+
+    dir = glfs_h_mkdir(fs, parent, my_dir, 0755, &sb);
+    if (dir == NULL) {
+        fprintf(stderr, "glfs_h_mkdir: error creating %s: from (%p),%s\n",
+                my_dir, parent, strerror(errno));
+        printf("glfs_h_unlink tests: FAILED\n");
+        goto out;
+    }
+
+    leaf = glfs_h_creat(fs, dir, my_file, O_CREAT, 0644, &sb);
+    if (leaf == NULL) {
+        fprintf(stderr, "glfs_h_creat: error creating %s: from (%p),%s\n",
+                my_file, dir, strerror(errno));
+        printf("glfs_h_unlink tests: FAILED\n");
+        goto out;
+    }
+
+    subdir = glfs_h_mkdir(fs, dir, my_subdir, 0755, &sb);
+    if (subdir == NULL) {
+        fprintf(stderr, "glfs_h_mkdir: error creating %s: from (%p),%s\n",
+                my_subdir, dir, strerror(errno));
+        printf("glfs_h_unlink tests: FAILED\n");
+        goto out;
+    }
+
+    subleaf = glfs_h_creat(fs, subdir, my_file, O_CREAT, 0644, &sb);
+    if (subleaf == NULL) {
+        fprintf(stderr, "glfs_h_creat: error creating %s: from (%p),%s\n",
+                my_file, subdir, strerror(errno));
+        printf("glfs_h_unlink tests: FAILED\n");
+        goto out;
+    }
+
+    /* unlink non empty directory */
+    ret = glfs_h_unlink(fs, dir, my_subdir);
+    if ((ret && errno != ENOTEMPTY) || (ret == 0)) {
+        fprintf(stderr,
+                "glfs_h_unlink: error unlinking %s: it is non empty: %s\n",
+                my_subdir, strerror(errno));
+        printf("glfs_h_unlink tests: FAILED\n");
+        goto out;
+    }
+
+    /* unlink regular file */
+    ret = glfs_h_unlink(fs, subdir, my_file);
+    if (ret) {
+        fprintf(stderr, "glfs_h_unlink: error unlinking %s: from (%p),%s\n",
+                my_file, subdir, strerror(errno));
+        printf("glfs_h_unlink tests: FAILED\n");
+        goto out;
+    }
+
+    /* unlink directory */
+    ret = glfs_h_unlink(fs, dir, my_subdir);
+    if (ret) {
+        fprintf(stderr, "glfs_h_unlink: error unlinking %s: from (%p),%s\n",
+                my_subdir, dir, strerror(errno));
+        printf("glfs_h_unlink tests: FAILED\n");
+        goto out;
+    }
+
+    /* unlink regular file */
+    ret = glfs_h_unlink(fs, dir, my_file);
+    if (ret) {
+        fprintf(stderr, "glfs_h_unlink: error unlinking %s: from (%p),%s\n",
+                my_file, dir, strerror(errno));
+        printf("glfs_h_unlink tests: FAILED\n");
+        goto out;
+    }
+
+    /* unlink non-existent regular file */
+    ret = glfs_h_unlink(fs, dir, my_file);
+    if ((ret && errno != ENOENT) || (ret == 0)) {
+        fprintf(stderr,
+                "glfs_h_unlink: error unlinking non-existent %s: invalid errno "
+                ",%d, %s\n",
+                my_file, ret, strerror(errno));
+        printf("glfs_h_unlink tests: FAILED\n");
+        goto out;
+    }
+
+    /* unlink non-existent directory */
+    ret = glfs_h_unlink(fs, dir, my_subdir);
+    if ((ret && errno != ENOENT) || (ret == 0)) {
+        fprintf(stderr,
+                "glfs_h_unlink: error unlinking non-existent %s:  invalid "
+                "errno ,%d, %s\n",
+                my_subdir, ret, strerror(errno));
+        printf("glfs_h_unlink tests: FAILED\n");
+        goto out;
+    }
+
+    /* unlink directory */
+    ret = glfs_h_unlink(fs, parent, my_dir);
+    if (ret) {
+        fprintf(stderr, "glfs_h_unlink: error unlinking %s: from (%p),%s\n",
+                my_dir, dir, strerror(errno));
+        printf("glfs_h_unlink tests: FAILED\n");
+        goto out;
+    }
+
+    printf("glfs_h_unlink tests: PASSED\n");
 
 out:
-        if (dir)
-                glfs_h_close (dir);
-        if (leaf)
-                glfs_h_close (leaf);
-        if (subdir)
-                glfs_h_close (subdir);
-        if (subleaf)
-                glfs_h_close (subleaf);
-        if (parent)
-                glfs_h_close (parent);
-
-        return;
+    if (dir)
+        glfs_h_close(dir);
+    if (leaf)
+        glfs_h_close(leaf);
+    if (subdir)
+        glfs_h_close(subdir);
+    if (subleaf)
+        glfs_h_close(subleaf);
+    if (parent)
+        glfs_h_close(parent);
+
+    return;
 }
 
 void
-test_h_getsetattrs (void)
+test_h_getsetattrs(void)
 {
-        char               *my_dir = "attrdir";
-        char               *my_file = "attrfile.txt";
-        struct glfs_object *parent = NULL, *leaf = NULL, *dir = NULL;
-        struct stat         sb, retsb;
-        int                 ret, valid;
-        struct timespec     timestamp;
-
-        printf("glfs_h_getattrs and setattrs tests: In Progress\n");
-
-        /* Prepare tests */
-        parent = glfs_h_lookupat (fs, NULL, full_parent_name, &sb, 0);
-        if (parent == NULL) {
-                fprintf (stderr, "glfs_h_lookupat: error on lookup of %s: from (%p),%s\n",
-                         full_parent_name, NULL, strerror (errno));
-                printf ("glfs_h_lookupat tests: FAILED\n");
-                goto out;
-        }
-        peek_stat (&sb);
-
-        dir = glfs_h_mkdir (fs, parent, my_dir, 0644, &sb);
-        if (dir == NULL) {
-                fprintf (stderr, "glfs_h_mkdir: error creating %s: from (%p),%s\n",
-                         my_dir, parent, strerror (errno));
-                printf ("glfs_h_unlink tests: FAILED\n");
-                goto out;
-        }
-        peek_stat (&sb);
-
-        leaf = glfs_h_creat (fs, dir, my_file, O_CREAT, 0644, &sb);
-        if (leaf == NULL) {
-                fprintf (stderr, "glfs_h_creat: error creating %s: from (%p),%s\n",
-                         my_file, dir, strerror (errno));
-                printf ("glfs_h_unlink tests: FAILED\n");
-                goto out;
-        }
-        peek_stat (&sb);
-
-        ret = glfs_h_getattrs (fs, dir, &retsb);
-        if (ret != 0) {
-                fprintf (stderr, "glfs_h_getattrs: error %s: from (%p),%s\n",
-                         my_dir, dir, strerror (errno));
-                printf ("glfs_h_getattrs and setattrs tests: FAILED\n");
-                goto out;
-        }
-        peek_stat (&retsb);
-        /* TODO: Compare stat information */
-
-        retsb.st_mode = 00666;
-        retsb.st_uid = 1000;
-        retsb.st_gid = 1001;
-        ret = clock_gettime (CLOCK_REALTIME, &timestamp);
-        if(ret != 0) {
-                fprintf (stderr, "clock_gettime: error %s\n", strerror (errno));
-                printf ("glfs_h_getattrs and setattrs tests: FAILED\n");
-                goto out;
-        }
-        retsb.st_atim = timestamp;
-        retsb.st_mtim = timestamp;
-        valid = GFAPI_SET_ATTR_MODE | GFAPI_SET_ATTR_UID | GFAPI_SET_ATTR_GID |
-        GFAPI_SET_ATTR_ATIME | GFAPI_SET_ATTR_MTIME;
-        peek_stat (&retsb);
-
-        ret = glfs_h_setattrs (fs, dir, &retsb, valid);
-        if (ret != 0) {
-                fprintf (stderr, "glfs_h_setattrs: error %s: from (%p),%s\n",
-                         my_dir, dir, strerror (errno));
-                printf ("glfs_h_getattrs and setattrs tests: FAILED\n");
-                goto out;
-        }
-
-        memset(&retsb, 0, sizeof (struct stat));
-        ret = glfs_h_stat (fs, dir, &retsb);
-        if (ret != 0) {
-                fprintf (stderr, "glfs_h_stat: error %s: from (%p),%s\n",
-                         my_dir, dir, strerror (errno));
-                printf ("glfs_h_getattrs and setattrs tests: FAILED\n");
-                goto out;
-        }
-        peek_stat (&retsb);
-
-        printf ("glfs_h_getattrs and setattrs tests: PASSED\n");
+    char *my_dir = "attrdir";
+    char *my_file = "attrfile.txt";
+    struct glfs_object *parent = NULL, *leaf = NULL, *dir = NULL;
+    struct stat sb, retsb;
+    int ret, valid;
+    struct timespec timestamp;
+
+    printf("glfs_h_getattrs and setattrs tests: In Progress\n");
+
+    /* Prepare tests */
+    parent = glfs_h_lookupat(fs, NULL, full_parent_name, &sb, 0);
+    if (parent == NULL) {
+        fprintf(stderr,
+                "glfs_h_lookupat: error on lookup of %s: from (%p),%s\n",
+                full_parent_name, NULL, strerror(errno));
+        printf("glfs_h_lookupat tests: FAILED\n");
+        goto out;
+    }
+    peek_stat(&sb);
+
+    dir = glfs_h_mkdir(fs, parent, my_dir, 0755, &sb);
+    if (dir == NULL) {
+        fprintf(stderr, "glfs_h_mkdir: error creating %s: from (%p),%s\n",
+                my_dir, parent, strerror(errno));
+        printf("glfs_h_unlink tests: FAILED\n");
+        goto out;
+    }
+    peek_stat(&sb);
+
+    leaf = glfs_h_creat(fs, dir, my_file, O_CREAT, 0644, &sb);
+    if (leaf == NULL) {
+        fprintf(stderr, "glfs_h_creat: error creating %s: from (%p),%s\n",
+                my_file, dir, strerror(errno));
+        printf("glfs_h_unlink tests: FAILED\n");
+        goto out;
+    }
+    peek_stat(&sb);
+
+    ret = glfs_h_getattrs(fs, dir, &retsb);
+    if (ret != 0) {
+        fprintf(stderr, "glfs_h_getattrs: error %s: from (%p),%s\n", my_dir,
+                dir, strerror(errno));
+        printf("glfs_h_getattrs and setattrs tests: FAILED\n");
+        goto out;
+    }
+    peek_stat(&retsb);
+    /* TODO: Compare stat information */
+
+    retsb.st_mode = 00666;
+    retsb.st_uid = 1000;
+    retsb.st_gid = 1001;
+    ret = clock_gettime(CLOCK_REALTIME, &timestamp);
+    if (ret != 0) {
+        fprintf(stderr, "clock_gettime: error %s\n", strerror(errno));
+        printf("glfs_h_getattrs and setattrs tests: FAILED\n");
+        goto out;
+    }
+    retsb.st_atim = timestamp;
+    retsb.st_mtim = timestamp;
+    valid = GFAPI_SET_ATTR_MODE | GFAPI_SET_ATTR_UID | GFAPI_SET_ATTR_GID |
+            GFAPI_SET_ATTR_ATIME | GFAPI_SET_ATTR_MTIME;
+    peek_stat(&retsb);
+
+    ret = glfs_h_setattrs(fs, dir, &retsb, valid);
+    if (ret != 0) {
+        fprintf(stderr, "glfs_h_setattrs: error %s: from (%p),%s\n", my_dir,
+                dir, strerror(errno));
+        printf("glfs_h_getattrs and setattrs tests: FAILED\n");
+        goto out;
+    }
+
+    memset(&retsb, 0, sizeof(struct stat));
+    ret = glfs_h_stat(fs, dir, &retsb);
+    if (ret != 0) {
+        fprintf(stderr, "glfs_h_stat: error %s: from (%p),%s\n", my_dir, dir,
+                strerror(errno));
+        printf("glfs_h_getattrs and setattrs tests: FAILED\n");
+        goto out;
+    }
+    peek_stat(&retsb);
+
+    printf("glfs_h_getattrs and setattrs tests: PASSED\n");
 out:
-        if (parent)
-                glfs_h_close (parent);
-        if (leaf)
-                glfs_h_close (leaf);
-        if (dir)
-                glfs_h_close (dir);
-
-        return;
+    if (parent)
+        glfs_h_close(parent);
+    if (leaf)
+        glfs_h_close(leaf);
+    if (dir)
+        glfs_h_close(dir);
+
+    return;
 }
 
 void
-test_h_truncate (void)
+test_h_truncate(void)
 {
-        char               *my_dir = "truncatedir";
-        char               *my_file = "file.txt";
-        struct glfs_object *root = NULL, *parent = NULL, *leaf = NULL;
-        struct stat         sb;
-        glfs_fd_t          *fd = NULL;
-        char                buf[32];
-        off_t               offset = 0;
-        int                 ret = 0;
-
-        printf("glfs_h_truncate tests: In Progress\n");
-
-        /* Prepare tests */
-        root = glfs_h_lookupat (fs, NULL, full_parent_name, &sb, 0);
-        if (root == NULL) {
-                fprintf (stderr, "glfs_h_lookupat: error on lookup of %s: from (%p),%s\n",
-                         full_parent_name, NULL, strerror (errno));
-                printf ("glfs_h_truncate tests: FAILED\n");
-                goto out;
-        }
-        peek_stat (&sb);
-
-        parent = glfs_h_mkdir (fs, root, my_dir, 0644, &sb);
-        if (parent == NULL) {
-                fprintf (stderr, "glfs_h_mkdir: error creating %s: from (%p),%s\n",
-                         my_dir, root, strerror (errno));
-                printf ("glfs_h_truncate tests: FAILED\n");
-                goto out;
-        }
-        peek_stat (&sb);
-
-        leaf = glfs_h_creat (fs, parent, my_file, O_CREAT, 0644, &sb);
-        if (leaf == NULL) {
-                fprintf (stderr, "glfs_h_creat: error creating %s: from (%p),%s\n",
-                         my_file, parent, strerror (errno));
-                printf ("glfs_h_truncate tests: FAILED\n");
-                goto out;
-        }
-        peek_stat (&sb);
-
-        fd = glfs_h_open (fs, leaf, O_RDWR);
-        if (fd == NULL) {
-                fprintf (stderr, "glfs_h_open: error on open of %s: %s\n",
-                         my_file, strerror (errno));
-                printf ("glfs_h_truncate tests: FAILED\n");
-                goto out;
-        }
-
-        memcpy (buf, "abcdefghijklmnopqrstuvwxyz012345", 32);
-        ret = glfs_write (fd, buf, 32, 0);
-
-        /* run tests */
-        /* truncate lower */
-        offset = 30;
-        ret = glfs_h_truncate (fs, leaf, offset);
-        if (ret != 0) {
-                fprintf (stderr, "glfs_h_truncate: error creating %s: from (%p),%s\n",
-                         my_file, parent, strerror (errno));
-                printf ("glfs_h_truncate tests: FAILED\n");
-                goto out;
-        }
-        ret = glfs_h_getattrs (fs, leaf, &sb);
-        if (ret != 0) {
-                fprintf (stderr, "glfs_h_getattrs: error for %s (%p),%s\n",
-                         my_file, leaf, strerror (errno));
-                printf ("glfs_h_truncate tests: FAILED\n");
-                goto out;
-        }
-        if (sb.st_size != offset) {
-                fprintf (stderr, "glfs_h_truncate: post size mismatch\n");
-                printf ("glfs_h_truncate tests: FAILED\n");
-                goto out;
-        }
-
-        /* truncate higher */
-        offset = 32;
-        ret = glfs_h_truncate (fs, leaf, offset);
-        if (ret != 0) {
-                fprintf (stderr, "glfs_h_truncate: error creating %s: from (%p),%s\n",
-                         my_file, parent, strerror (errno));
-                printf ("glfs_h_truncate tests: FAILED\n");
-                goto out;
-        }
-        ret = glfs_h_getattrs (fs, leaf, &sb);
-        if (ret != 0) {
-                fprintf (stderr, "glfs_h_getattrs: error for %s (%p),%s\n",
-                         my_file, leaf, strerror (errno));
-                printf ("glfs_h_truncate tests: FAILED\n");
-                goto out;
-        }
-        if (sb.st_size != offset) {
-                fprintf (stderr, "glfs_h_truncate: post size mismatch\n");
-                printf ("glfs_h_truncate tests: FAILED\n");
-                goto out;
-        }
-
-        /* truncate equal */
-        offset = 30;
-        ret = glfs_h_truncate (fs, leaf, offset);
-        if (ret != 0) {
-                fprintf (stderr, "glfs_h_truncate: error creating %s: from (%p),%s\n",
-                         my_file, parent, strerror (errno));
-                printf ("glfs_h_truncate tests: FAILED\n");
-                goto out;
-        }
-        ret = glfs_h_getattrs (fs, leaf, &sb);
-        if (ret != 0) {
-                fprintf (stderr, "glfs_h_getattrs: error for %s (%p),%s\n",
-                         my_file, leaf, strerror (errno));
-                printf ("glfs_h_truncate tests: FAILED\n");
-                goto out;
-        }
-        if (sb.st_size != offset) {
-                fprintf (stderr, "glfs_h_truncate: post size mismatch\n");
-                printf ("glfs_h_truncate tests: FAILED\n");
-                goto out;
-        }
-
-        printf ("glfs_h_truncate tests: PASSED\n");
+    char *my_dir = "truncatedir";
+    char *my_file = "file.txt";
+    struct glfs_object *root = NULL, *parent = NULL, *leaf = NULL;
+    struct stat sb;
+    glfs_fd_t *fd = NULL;
+    char buf[32];
+    off_t offset = 0;
+    int ret = 0;
+
+    printf("glfs_h_truncate tests: In Progress\n");
+
+    /* Prepare tests */
+    root = glfs_h_lookupat(fs, NULL, full_parent_name, &sb, 0);
+    if (root == NULL) {
+        fprintf(stderr,
+                "glfs_h_lookupat: error on lookup of %s: from (%p),%s\n",
+                full_parent_name, NULL, strerror(errno));
+        printf("glfs_h_truncate tests: FAILED\n");
+        goto out;
+    }
+    peek_stat(&sb);
+
+    parent = glfs_h_mkdir(fs, root, my_dir, 0755, &sb);
+    if (parent == NULL) {
+        fprintf(stderr, "glfs_h_mkdir: error creating %s: from (%p),%s\n",
+                my_dir, root, strerror(errno));
+        printf("glfs_h_truncate tests: FAILED\n");
+        goto out;
+    }
+    peek_stat(&sb);
+
+    leaf = glfs_h_creat(fs, parent, my_file, O_CREAT, 0644, &sb);
+    if (leaf == NULL) {
+        fprintf(stderr, "glfs_h_creat: error creating %s: from (%p),%s\n",
+                my_file, parent, strerror(errno));
+        printf("glfs_h_truncate tests: FAILED\n");
+        goto out;
+    }
+    peek_stat(&sb);
+
+    fd = glfs_h_open(fs, leaf, O_RDWR);
+    if (fd == NULL) {
+        fprintf(stderr, "glfs_h_open: error on open of %s: %s\n", my_file,
+                strerror(errno));
+        printf("glfs_h_truncate tests: FAILED\n");
+        goto out;
+    }
+
+    memcpy(buf, "abcdefghijklmnopqrstuvwxyz012345", 32);
+    ret = glfs_write(fd, buf, 32, 0);
+
+    /* run tests */
+    /* truncate lower */
+    offset = 30;
+    ret = glfs_h_truncate(fs, leaf, offset);
+    if (ret != 0) {
+        fprintf(stderr, "glfs_h_truncate: error creating %s: from (%p),%s\n",
+                my_file, parent, strerror(errno));
+        printf("glfs_h_truncate tests: FAILED\n");
+        goto out;
+    }
+    ret = glfs_h_getattrs(fs, leaf, &sb);
+    if (ret != 0) {
+        fprintf(stderr, "glfs_h_getattrs: error for %s (%p),%s\n", my_file,
+                leaf, strerror(errno));
+        printf("glfs_h_truncate tests: FAILED\n");
+        goto out;
+    }
+    if (sb.st_size != offset) {
+        fprintf(stderr, "glfs_h_truncate: post size mismatch\n");
+        printf("glfs_h_truncate tests: FAILED\n");
+        goto out;
+    }
+
+    /* truncate higher */
+    offset = 32;
+    ret = glfs_h_truncate(fs, leaf, offset);
+    if (ret != 0) {
+        fprintf(stderr, "glfs_h_truncate: error creating %s: from (%p),%s\n",
+                my_file, parent, strerror(errno));
+        printf("glfs_h_truncate tests: FAILED\n");
+        goto out;
+    }
+    ret = glfs_h_getattrs(fs, leaf, &sb);
+    if (ret != 0) {
+        fprintf(stderr, "glfs_h_getattrs: error for %s (%p),%s\n", my_file,
+                leaf, strerror(errno));
+        printf("glfs_h_truncate tests: FAILED\n");
+        goto out;
+    }
+    if (sb.st_size != offset) {
+        fprintf(stderr, "glfs_h_truncate: post size mismatch\n");
+        printf("glfs_h_truncate tests: FAILED\n");
+        goto out;
+    }
+
+    /* truncate equal */
+    offset = 30;
+    ret = glfs_h_truncate(fs, leaf, offset);
+    if (ret != 0) {
+        fprintf(stderr, "glfs_h_truncate: error creating %s: from (%p),%s\n",
+                my_file, parent, strerror(errno));
+        printf("glfs_h_truncate tests: FAILED\n");
+        goto out;
+    }
+    ret = glfs_h_getattrs(fs, leaf, &sb);
+    if (ret != 0) {
+        fprintf(stderr, "glfs_h_getattrs: error for %s (%p),%s\n", my_file,
+                leaf, strerror(errno));
+        printf("glfs_h_truncate tests: FAILED\n");
+        goto out;
+    }
+    if (sb.st_size != offset) {
+        fprintf(stderr, "glfs_h_truncate: post size mismatch\n");
+        printf("glfs_h_truncate tests: FAILED\n");
+        goto out;
+    }
+
+    printf("glfs_h_truncate tests: PASSED\n");
 out:
-        if (fd)
-                glfs_close (fd);
-        if (root)
-                glfs_h_close (root);
-        if (parent)
-                glfs_h_close (parent);
-        if (leaf)
-                glfs_h_close (leaf);
-
-        return;
+    if (fd)
+        glfs_close(fd);
+    if (root)
+        glfs_h_close(root);
+    if (parent)
+        glfs_h_close(parent);
+    if (leaf)
+        glfs_h_close(leaf);
+
+    return;
 }
 
 void
-test_h_links (void)
+test_h_links(void)
 {
-        char               *my_dir = "linkdir";
-        char               *my_file = "file.txt";
-        char               *my_symlnk = "slnk.txt";
-        char               *my_lnk = "lnk.txt";
-        char               *linksrc_dir = "dir1";
-        char               *linktgt_dir = "dir2";
-        struct glfs_object *root = NULL, *parent = NULL, *leaf = NULL,
-                           *dirsrc = NULL, *dirtgt = NULL, *dleaf = NULL;
-        struct glfs_object *ln1 = NULL;
-        struct stat         sb;
-        int                 ret;
-        char               *buf = NULL;
-
-        printf("glfs_h_link(s) tests: In Progress\n");
-
-        /* Prepare tests */
-        root = glfs_h_lookupat (fs, NULL, full_parent_name, &sb, 0);
-        if (root == NULL) {
-                fprintf (stderr, "glfs_h_lookupat: error on lookup of %s: from (%p),%s\n",
-                         full_parent_name, NULL, strerror (errno));
-                printf ("glfs_h_link(s) tests: FAILED\n");
-                goto out;
-        }
-        peek_stat (&sb);
-
-        parent = glfs_h_mkdir (fs, root, my_dir, 0644, &sb);
-        if (parent == NULL) {
-                fprintf (stderr, "glfs_h_mkdir: error creating %s: from (%p),%s\n",
-                         my_dir, root, strerror (errno));
-                printf ("glfs_h_link(s) tests: FAILED\n");
-                goto out;
-        }
-        peek_stat (&sb);
-
-        leaf = glfs_h_creat (fs, parent, my_file, O_CREAT, 0644, &sb);
-        if (leaf == NULL) {
-                fprintf (stderr, "glfs_h_creat: error creating %s: from (%p),%s\n",
-                         my_file, parent, strerror (errno));
-                printf ("glfs_h_link(s) tests: FAILED\n");
-                goto out;
-        }
-        peek_stat (&sb);
-
-        dirsrc = glfs_h_mkdir (fs, parent, linksrc_dir, 0644, &sb);
-        if (dirsrc == NULL) {
-                fprintf (stderr, "glfs_h_mkdir: error creating %s: from (%p),%s\n",
-                         linksrc_dir, parent, strerror (errno));
-                printf ("glfs_h_link(s) tests: FAILED\n");
-                goto out;
-        }
-        peek_stat (&sb);
-
-        dirtgt = glfs_h_mkdir (fs, parent, linktgt_dir, 0644, &sb);
-        if (dirtgt == NULL) {
-                fprintf (stderr, "glfs_h_mkdir: error creating %s: from (%p),%s\n",
-                         linktgt_dir, parent, strerror (errno));
-                printf ("glfs_h_link(s) tests: FAILED\n");
-                goto out;
-        }
-        peek_stat (&sb);
-
-        dleaf = glfs_h_creat (fs, dirsrc, my_file, O_CREAT, 0644, &sb);
-        if (dleaf == NULL) {
-                fprintf (stderr, "glfs_h_creat: error creating %s: from (%p),%s\n",
-                         my_file, dirsrc, strerror (errno));
-                printf ("glfs_h_link(s) tests: FAILED\n");
-                goto out;
-        }
-        peek_stat (&sb);
-
-        /* run tests */
-        /* sym link: /testdir/linkdir/file.txt to ./slnk.txt */
-        ln1 = glfs_h_symlink (fs, parent, my_symlnk, "./file.txt", &sb);
-        if (ln1 == NULL) {
-                fprintf (stderr, "glfs_h_symlink: error creating %s: from (%p),%s\n",
-                         my_symlnk, parent, strerror (errno));
-                printf ("glfs_h_link(s) tests: FAILED\n");
-                goto out;
-        }
-        peek_stat (&sb);
-
-        buf = calloc (1024, sizeof(char));
-        if (buf == NULL) {
-                fprintf (stderr, "Error allocating memory\n");
-                printf ("glfs_h_link(s) tests: FAILED\n");
-                goto out;
-        }
-
-        ret = glfs_h_readlink (fs, ln1, buf, 1024);
-        if (ret <= 0) {
-                fprintf (stderr, "glfs_h_readlink: error reading %s: from (%p),%s\n",
-                         my_symlnk, ln1, strerror (errno));
-                printf ("glfs_h_link(s) tests: FAILED\n");
-                goto out;
-        }
-        if (!(strncmp (buf, my_symlnk, strlen (my_symlnk)))) {
-                fprintf (stderr, "glfs_h_readlink: error mismatch in link name: actual %s: retrieved %s\n",
-                         my_symlnk, buf);
-                printf ("glfs_h_link(s) tests: FAILED\n");
-                goto out;
-        }
-
-        /* link: /testdir/linkdir/file.txt to ./lnk.txt */
-        ret = glfs_h_link (fs, leaf, parent, my_lnk);
-        if (ret != 0) {
-                fprintf (stderr, "glfs_h_link: error creating %s: from (%p),%s\n",
-                         my_lnk, parent, strerror (errno));
-                printf ("glfs_h_link(s) tests: FAILED\n");
-                goto out;
-        }
-        /* TODO: Should write content to a file and read from the link */
-
-        /* link: /testdir/linkdir/dir1/file.txt to ../dir2/slnk.txt */
-        ret = glfs_h_link (fs, dleaf, dirtgt, my_lnk);
-        if (ret != 0) {
-                fprintf (stderr, "glfs_h_link: error creating %s: from (%p),%s\n",
-                         my_lnk, dirtgt, strerror (errno));
-                printf ("glfs_h_link(s) tests: FAILED\n");
-                goto out;
-        }
-        /* TODO: Should write content to a file and read from the link */
-
-        printf ("glfs_h_link(s) tests: PASSED\n");
+    char *my_dir = "linkdir";
+    char *my_file = "file.txt";
+    char *my_symlnk = "slnk.txt";
+    char *my_lnk = "lnk.txt";
+    char *linksrc_dir = "dir1";
+    char *linktgt_dir = "dir2";
+    struct glfs_object *root = NULL, *parent = NULL, *leaf = NULL,
+                       *dirsrc = NULL, *dirtgt = NULL, *dleaf = NULL;
+    struct glfs_object *ln1 = NULL;
+    struct stat sb;
+    int ret;
+    char *buf = NULL;
+
+    printf("glfs_h_link(s) tests: In Progress\n");
+
+    /* Prepare tests */
+    root = glfs_h_lookupat(fs, NULL, full_parent_name, &sb, 0);
+    if (root == NULL) {
+        fprintf(stderr,
+                "glfs_h_lookupat: error on lookup of %s: from (%p),%s\n",
+                full_parent_name, NULL, strerror(errno));
+        printf("glfs_h_link(s) tests: FAILED\n");
+        goto out;
+    }
+    peek_stat(&sb);
+
+    parent = glfs_h_mkdir(fs, root, my_dir, 0755, &sb);
+    if (parent == NULL) {
+        fprintf(stderr, "glfs_h_mkdir: error creating %s: from (%p),%s\n",
+                my_dir, root, strerror(errno));
+        printf("glfs_h_link(s) tests: FAILED\n");
+        goto out;
+    }
+    peek_stat(&sb);
+
+    leaf = glfs_h_creat(fs, parent, my_file, O_CREAT, 0644, &sb);
+    if (leaf == NULL) {
+        fprintf(stderr, "glfs_h_creat: error creating %s: from (%p),%s\n",
+                my_file, parent, strerror(errno));
+        printf("glfs_h_link(s) tests: FAILED\n");
+        goto out;
+    }
+    peek_stat(&sb);
+
+    dirsrc = glfs_h_mkdir(fs, parent, linksrc_dir, 0755, &sb);
+    if (dirsrc == NULL) {
+        fprintf(stderr, "glfs_h_mkdir: error creating %s: from (%p),%s\n",
+                linksrc_dir, parent, strerror(errno));
+        printf("glfs_h_link(s) tests: FAILED\n");
+        goto out;
+    }
+    peek_stat(&sb);
+
+    dirtgt = glfs_h_mkdir(fs, parent, linktgt_dir, 0755, &sb);
+    if (dirtgt == NULL) {
+        fprintf(stderr, "glfs_h_mkdir: error creating %s: from (%p),%s\n",
+                linktgt_dir, parent, strerror(errno));
+        printf("glfs_h_link(s) tests: FAILED\n");
+        goto out;
+    }
+    peek_stat(&sb);
+
+    dleaf = glfs_h_creat(fs, dirsrc, my_file, O_CREAT, 0644, &sb);
+    if (dleaf == NULL) {
+        fprintf(stderr, "glfs_h_creat: error creating %s: from (%p),%s\n",
+                my_file, dirsrc, strerror(errno));
+        printf("glfs_h_link(s) tests: FAILED\n");
+        goto out;
+    }
+    peek_stat(&sb);
+
+    /* run tests */
+    /* sym link: /testdir/linkdir/file.txt to ./slnk.txt */
+    ln1 = glfs_h_symlink(fs, parent, my_symlnk, "./file.txt", &sb);
+    if (ln1 == NULL) {
+        fprintf(stderr, "glfs_h_symlink: error creating %s: from (%p),%s\n",
+                my_symlnk, parent, strerror(errno));
+        printf("glfs_h_link(s) tests: FAILED\n");
+        goto out;
+    }
+    peek_stat(&sb);
+
+    buf = calloc(1024, sizeof(char));
+    if (buf == NULL) {
+        fprintf(stderr, "Error allocating memory\n");
+        printf("glfs_h_link(s) tests: FAILED\n");
+        goto out;
+    }
+
+    ret = glfs_h_readlink(fs, ln1, buf, 1024);
+    if (ret <= 0) {
+        fprintf(stderr, "glfs_h_readlink: error reading %s: from (%p),%s\n",
+                my_symlnk, ln1, strerror(errno));
+        printf("glfs_h_link(s) tests: FAILED\n");
+        goto out;
+    }
+    if (!(strncmp(buf, my_symlnk, strlen(my_symlnk)))) {
+        fprintf(stderr,
+                "glfs_h_readlink: error mismatch in link name: actual %s: "
+                "retrieved %s\n",
+                my_symlnk, buf);
+        printf("glfs_h_link(s) tests: FAILED\n");
+        goto out;
+    }
+
+    /* link: /testdir/linkdir/file.txt to ./lnk.txt */
+    ret = glfs_h_link(fs, leaf, parent, my_lnk);
+    if (ret != 0) {
+        fprintf(stderr, "glfs_h_link: error creating %s: from (%p),%s\n",
+                my_lnk, parent, strerror(errno));
+        printf("glfs_h_link(s) tests: FAILED\n");
+        goto out;
+    }
+    /* TODO: Should write content to a file and read from the link */
+
+    /* link: /testdir/linkdir/dir1/file.txt to ../dir2/slnk.txt */
+    ret = glfs_h_link(fs, dleaf, dirtgt, my_lnk);
+    if (ret != 0) {
+        fprintf(stderr, "glfs_h_link: error creating %s: from (%p),%s\n",
+                my_lnk, dirtgt, strerror(errno));
+        printf("glfs_h_link(s) tests: FAILED\n");
+        goto out;
+    }
+    /* TODO: Should write content to a file and read from the link */
+
+    printf("glfs_h_link(s) tests: PASSED\n");
 
 out:
-        if (root)
-                glfs_h_close (root);
-        if (parent)
-                glfs_h_close (parent);
-        if (leaf)
-                glfs_h_close (leaf);
-        if (dirsrc)
-                glfs_h_close (dirsrc);
-        if (dirtgt)
-                glfs_h_close (dirtgt);
-        if (dleaf)
-                glfs_h_close (dleaf);
-        if (ln1)
-                glfs_h_close (ln1);
-        if (buf)
-                free (buf);
-
-        return;
+    if (root)
+        glfs_h_close(root);
+    if (parent)
+        glfs_h_close(parent);
+    if (leaf)
+        glfs_h_close(leaf);
+    if (dirsrc)
+        glfs_h_close(dirsrc);
+    if (dirtgt)
+        glfs_h_close(dirtgt);
+    if (dleaf)
+        glfs_h_close(dleaf);
+    if (ln1)
+        glfs_h_close(ln1);
+    if (buf)
+        free(buf);
+
+    return;
 }
 
 void
-test_h_rename (void)
+test_h_rename(void)
 {
-        char               *my_dir = "renamedir";
-        char               *my_file = "file.txt";
-        char               *src_dir = "dir1";
-        char               *tgt_dir = "dir2";
-        struct glfs_object *root = NULL, *parent = NULL, *leaf = NULL,
-                           *dirsrc = NULL, *dirtgt = NULL, *dleaf = NULL;
-        struct stat         sb;
-        int                 ret;
-
-        printf("glfs_h_rename tests: In Progress\n");
-
-        /* Prepare tests */
-        root = glfs_h_lookupat (fs, NULL, full_parent_name, &sb, 0);
-        if (root == NULL) {
-                fprintf (stderr, "glfs_h_lookupat: error on lookup of %s: from (%p),%s\n",
-                         full_parent_name, NULL, strerror (errno));
-                printf ("glfs_h_rename tests: FAILED\n");
-                goto out;
-        }
-        peek_stat (&sb);
-
-        parent = glfs_h_mkdir (fs, root, my_dir, 0644, &sb);
-        if (parent == NULL) {
-                fprintf (stderr, "glfs_h_mkdir: error creating %s: from (%p),%s\n",
-                         my_dir, root, strerror (errno));
-                printf ("glfs_h_rename tests: FAILED\n");
-                goto out;
-        }
-        peek_stat (&sb);
-
-        leaf = glfs_h_creat (fs, parent, my_file, O_CREAT, 0644, &sb);
-        if (leaf == NULL) {
-                fprintf (stderr, "glfs_h_creat: error creating %s: from (%p),%s\n",
-                         my_file, parent, strerror (errno));
-                printf ("glfs_h_rename tests: FAILED\n");
-                goto out;
-        }
-        peek_stat (&sb);
-
-        dirsrc = glfs_h_mkdir (fs, parent, src_dir, 0644, &sb);
-        if (dirsrc == NULL) {
-                fprintf (stderr, "glfs_h_mkdir: error creating %s: from (%p),%s\n",
-                         src_dir, parent, strerror (errno));
-                printf ("glfs_h_rename tests: FAILED\n");
-                goto out;
-        }
-        peek_stat (&sb);
-
-        dirtgt = glfs_h_mkdir (fs, parent, tgt_dir, 0644, &sb);
-        if (dirtgt == NULL) {
-                fprintf (stderr, "glfs_h_mkdir: error creating %s: from (%p),%s\n",
-                         tgt_dir, parent, strerror (errno));
-                printf ("glfs_h_rename tests: FAILED\n");
-                goto out;
-        }
-        peek_stat (&sb);
-
-        dleaf = glfs_h_creat (fs, dirsrc, my_file, O_CREAT, 0644, &sb);
-        if (dleaf == NULL) {
-                fprintf (stderr, "glfs_h_creat: error creating %s: from (%p),%s\n",
-                         my_file, dirsrc, strerror (errno));
-                printf ("glfs_h_rename tests: FAILED\n");
-                goto out;
-        }
-        peek_stat (&sb);
-
-        /* run tests */
-        /* Rename file.txt -> file1.txt */
-        ret = glfs_h_rename (fs, parent, "file.txt", parent, "file1.txt");
-        if (ret != 0) {
-                fprintf (stderr, "glfs_h_rename: error renaming %s to %s (%s)\n",
-                         "file.txt", "file1.txt", strerror (errno));
-                printf ("glfs_h_rename tests: FAILED\n");
-                goto out;
-        }
-
-        /* rename dir1/file.txt -> file.txt */
-        ret = glfs_h_rename (fs, dirsrc, "file.txt", parent, "file.txt");
-        if (ret != 0) {
-                fprintf (stderr, "glfs_h_rename: error renaming %s/%s to %s (%s)\n",
-                         src_dir, "file.txt", "file.txt", strerror (errno));
-                printf ("glfs_h_rename tests: FAILED\n");
-                goto out;
-        }
-
-        /* rename file1.txt -> file.txt (exists) */
-        ret = glfs_h_rename (fs, parent, "file1.txt", parent, "file.txt");
-        if (ret != 0) {
-                fprintf (stderr, "glfs_h_rename: error renaming %s to %s (%s)\n",
-                         "file.txt", "file.txt", strerror (errno));
-                printf ("glfs_h_rename tests: FAILED\n");
-                goto out;
-        }
-
-        /* rename dir1 -> dir3 */
-        ret = glfs_h_rename (fs, parent, "dir1", parent, "dir3");
-        if (ret != 0) {
-                fprintf (stderr, "glfs_h_rename: error renaming %s to %s (%s)\n",
-                         "dir1", "dir3", strerror (errno));
-                printf ("glfs_h_rename tests: FAILED\n");
-                goto out;
-        }
-
-        /* rename dir2 ->dir3 (exists) */
-        ret = glfs_h_rename (fs, parent, "dir2", parent, "dir3");
-        if (ret != 0) {
-                fprintf (stderr, "glfs_h_rename: error renaming %s to %s (%s)\n",
-                         "dir2", "dir3", strerror (errno));
-                printf ("glfs_h_rename tests: FAILED\n");
-                goto out;
-        }
-
-        /* rename file.txt -> dir3 (fail) */
-        ret = glfs_h_rename (fs, parent, "file.txt", parent, "dir3");
-        if (ret == 0) {
-                fprintf (stderr, "glfs_h_rename: NO error renaming %s to %s (%s)\n",
-                         "file.txt", "dir3", strerror (errno));
-                printf ("glfs_h_rename tests: FAILED\n");
-                goto out;
-        }
-
-        /* rename dir3 -> file.txt (fail) */
-        ret = glfs_h_rename (fs, parent, "dir3", parent, "file.txt");
-        if (ret == 0) {
-                fprintf (stderr, "glfs_h_rename: NO error renaming %s to %s (%s)\n",
-                         "dir3", "file.txt", strerror (errno));
-                printf ("glfs_h_rename tests: FAILED\n");
-                goto out;
-        }
-
-        printf ("glfs_h_rename tests: PASSED\n");
+    char *my_dir = "renamedir";
+    char *my_file = "file.txt";
+    char *src_dir = "dir1";
+    char *tgt_dir = "dir2";
+    struct glfs_object *root = NULL, *parent = NULL, *leaf = NULL,
+                       *dirsrc = NULL, *dirtgt = NULL, *dleaf = NULL;
+    struct stat sb;
+    int ret;
+
+    printf("glfs_h_rename tests: In Progress\n");
+
+    /* Prepare tests */
+    root = glfs_h_lookupat(fs, NULL, full_parent_name, &sb, 0);
+    if (root == NULL) {
+        fprintf(stderr,
+                "glfs_h_lookupat: error on lookup of %s: from (%p),%s\n",
+                full_parent_name, NULL, strerror(errno));
+        printf("glfs_h_rename tests: FAILED\n");
+        goto out;
+    }
+    peek_stat(&sb);
+
+    parent = glfs_h_mkdir(fs, root, my_dir, 0755, &sb);
+    if (parent == NULL) {
+        fprintf(stderr, "glfs_h_mkdir: error creating %s: from (%p),%s\n",
+                my_dir, root, strerror(errno));
+        printf("glfs_h_rename tests: FAILED\n");
+        goto out;
+    }
+    peek_stat(&sb);
+
+    leaf = glfs_h_creat(fs, parent, my_file, O_CREAT, 0644, &sb);
+    if (leaf == NULL) {
+        fprintf(stderr, "glfs_h_creat: error creating %s: from (%p),%s\n",
+                my_file, parent, strerror(errno));
+        printf("glfs_h_rename tests: FAILED\n");
+        goto out;
+    }
+    peek_stat(&sb);
+
+    dirsrc = glfs_h_mkdir(fs, parent, src_dir, 0755, &sb);
+    if (dirsrc == NULL) {
+        fprintf(stderr, "glfs_h_mkdir: error creating %s: from (%p),%s\n",
+                src_dir, parent, strerror(errno));
+        printf("glfs_h_rename tests: FAILED\n");
+        goto out;
+    }
+    peek_stat(&sb);
+
+    dirtgt = glfs_h_mkdir(fs, parent, tgt_dir, 0755, &sb);
+    if (dirtgt == NULL) {
+        fprintf(stderr, "glfs_h_mkdir: error creating %s: from (%p),%s\n",
+                tgt_dir, parent, strerror(errno));
+        printf("glfs_h_rename tests: FAILED\n");
+        goto out;
+    }
+    peek_stat(&sb);
+
+    dleaf = glfs_h_creat(fs, dirsrc, my_file, O_CREAT, 0644, &sb);
+    if (dleaf == NULL) {
+        fprintf(stderr, "glfs_h_creat: error creating %s: from (%p),%s\n",
+                my_file, dirsrc, strerror(errno));
+        printf("glfs_h_rename tests: FAILED\n");
+        goto out;
+    }
+    peek_stat(&sb);
+
+    /* run tests */
+    /* Rename file.txt -> file1.txt */
+    ret = glfs_h_rename(fs, parent, "file.txt", parent, "file1.txt");
+    if (ret != 0) {
+        fprintf(stderr, "glfs_h_rename: error renaming %s to %s (%s)\n",
+                "file.txt", "file1.txt", strerror(errno));
+        printf("glfs_h_rename tests: FAILED\n");
+        goto out;
+    }
+
+    /* rename dir1/file.txt -> file.txt */
+    ret = glfs_h_rename(fs, dirsrc, "file.txt", parent, "file.txt");
+    if (ret != 0) {
+        fprintf(stderr, "glfs_h_rename: error renaming %s/%s to %s (%s)\n",
+                src_dir, "file.txt", "file.txt", strerror(errno));
+        printf("glfs_h_rename tests: FAILED\n");
+        goto out;
+    }
+
+    /* rename file1.txt -> file.txt (exists) */
+    ret = glfs_h_rename(fs, parent, "file1.txt", parent, "file.txt");
+    if (ret != 0) {
+        fprintf(stderr, "glfs_h_rename: error renaming %s to %s (%s)\n",
+                "file.txt", "file.txt", strerror(errno));
+        printf("glfs_h_rename tests: FAILED\n");
+        goto out;
+    }
+
+    /* rename dir1 -> dir3 */
+    ret = glfs_h_rename(fs, parent, "dir1", parent, "dir3");
+    if (ret != 0) {
+        fprintf(stderr, "glfs_h_rename: error renaming %s to %s (%s)\n", "dir1",
+                "dir3", strerror(errno));
+        printf("glfs_h_rename tests: FAILED\n");
+        goto out;
+    }
+
+    /* rename dir2 ->dir3 (exists) */
+    ret = glfs_h_rename(fs, parent, "dir2", parent, "dir3");
+    if (ret != 0) {
+        fprintf(stderr, "glfs_h_rename: error renaming %s to %s (%s)\n", "dir2",
+                "dir3", strerror(errno));
+        printf("glfs_h_rename tests: FAILED\n");
+        goto out;
+    }
+
+    /* rename file.txt -> dir3 (fail) */
+    ret = glfs_h_rename(fs, parent, "file.txt", parent, "dir3");
+    if (ret == 0) {
+        fprintf(stderr, "glfs_h_rename: NO error renaming %s to %s (%s)\n",
+                "file.txt", "dir3", strerror(errno));
+        printf("glfs_h_rename tests: FAILED\n");
+        goto out;
+    }
+
+    /* rename dir3 -> file.txt (fail) */
+    ret = glfs_h_rename(fs, parent, "dir3", parent, "file.txt");
+    if (ret == 0) {
+        fprintf(stderr, "glfs_h_rename: NO error renaming %s to %s (%s)\n",
+                "dir3", "file.txt", strerror(errno));
+        printf("glfs_h_rename tests: FAILED\n");
+        goto out;
+    }
+
+    printf("glfs_h_rename tests: PASSED\n");
 
 out:
-        if (root)
-                glfs_h_close (root);
-        if (parent)
-                glfs_h_close (parent);
-        if (leaf)
-                glfs_h_close (leaf);
-        if (dirsrc)
-                glfs_h_close (dirsrc);
-        if (dirtgt)
-                glfs_h_close (dirtgt);
-        if (dleaf)
-                glfs_h_close (dleaf);
-
-        return;
+    if (root)
+        glfs_h_close(root);
+    if (parent)
+        glfs_h_close(parent);
+    if (leaf)
+        glfs_h_close(leaf);
+    if (dirsrc)
+        glfs_h_close(dirsrc);
+    if (dirtgt)
+        glfs_h_close(dirtgt);
+    if (dleaf)
+        glfs_h_close(dleaf);
+
+    return;
 }
 
 void
-assimilatetime (struct timespec *ts, struct timespec ts_st,
-                struct timespec ts_ed)
+assimilatetime(struct timespec *ts, struct timespec ts_st,
+               struct timespec ts_ed)
 {
-        if ((ts_ed.tv_nsec - ts_st.tv_nsec) < 0) {
-                ts->tv_sec += ts_ed.tv_sec - ts_st.tv_sec - 1;
-                ts->tv_nsec += 1000000000 + ts_ed.tv_nsec - ts_st.tv_nsec;
-        } else {
-                ts->tv_sec += ts_ed.tv_sec - ts_st.tv_sec;
-                ts->tv_nsec += ts_ed.tv_nsec - ts_st.tv_nsec;
-        }
-
-        if (ts->tv_nsec > 1000000000) {
-                ts->tv_nsec = ts->tv_nsec - 1000000000;
-                ts->tv_sec += 1;
-        }
-
-        return;
+    if ((ts_ed.tv_nsec - ts_st.tv_nsec) < 0) {
+        ts->tv_sec += ts_ed.tv_sec - ts_st.tv_sec - 1;
+        ts->tv_nsec += 1000000000 + ts_ed.tv_nsec - ts_st.tv_nsec;
+    } else {
+        ts->tv_sec += ts_ed.tv_sec - ts_st.tv_sec;
+        ts->tv_nsec += ts_ed.tv_nsec - ts_st.tv_nsec;
+    }
+
+    if (ts->tv_nsec > 1000000000) {
+        ts->tv_nsec = ts->tv_nsec - 1000000000;
+        ts->tv_sec += 1;
+    }
+
+    return;
 }
 
 #define MAX_FILES_CREATE 10
-#define MAXPATHNAME      512
+#define MAXPATHNAME 512
 void
-test_h_performance (void)
+test_h_performance(void)
 {
-        char               *my_dir = "perftest",
-                           *full_dir_path="/testdir/perftest";
-        char               *my_file = "file_", my_file_name[MAXPATHNAME];
-        struct glfs_object *parent = NULL, *leaf = NULL, *dir = NULL;
-        struct stat         sb;
-        int                 ret, i;
-        struct glfs_fd     *fd;
-        struct timespec     c_ts = {0, 0}, c_ts_st, c_ts_ed;
-        struct timespec     o_ts = {0, 0}, o_ts_st, o_ts_ed;
-
-        printf("glfs_h_performance tests: In Progress\n");
-
-        /* Prepare tests */
-        parent = glfs_h_lookupat (fs, NULL, full_parent_name, &sb, 0);
-        if (parent == NULL) {
-                fprintf (stderr, "glfs_h_lookupat: error on lookup of %s: from (%p),%s\n",
-                         full_parent_name, NULL, strerror (errno));
-                printf ("glfs_h_performance tests: FAILED\n");
-                goto out;
-        }
-
-        dir = glfs_h_mkdir (fs, parent, my_dir, 0644, &sb);
-        if (dir == NULL) {
-                fprintf (stderr, "glfs_h_mkdir: error creating %s: from (%p),%s\n",
-                         my_dir, parent, strerror (errno));
-                printf ("glfs_h_performance tests: FAILED\n");
-                goto out;
-        }
-        peek_stat (&sb);
-
-        /* create performance */
-        ret = clock_gettime (CLOCK_REALTIME, &o_ts_st);
-        if(ret != 0) {
-                fprintf (stderr, "clock_gettime: error %s\n", strerror (errno));
-                printf ("glfs_h_getattrs and setattrs tests: FAILED\n");
-                goto out;
+    char *my_dir = "perftest", *full_dir_path = "/testdir/perftest";
+    char *my_file = "file_", my_file_name[MAXPATHNAME];
+    struct glfs_object *parent = NULL, *leaf = NULL, *dir = NULL;
+    struct stat sb;
+    int ret, i;
+    struct glfs_fd *fd;
+    struct timespec c_ts = {0, 0}, c_ts_st, c_ts_ed;
+    struct timespec o_ts = {0, 0}, o_ts_st, o_ts_ed;
+
+    printf("glfs_h_performance tests: In Progress\n");
+
+    /* Prepare tests */
+    parent = glfs_h_lookupat(fs, NULL, full_parent_name, &sb, 0);
+    if (parent == NULL) {
+        fprintf(stderr,
+                "glfs_h_lookupat: error on lookup of %s: from (%p),%s\n",
+                full_parent_name, NULL, strerror(errno));
+        printf("glfs_h_performance tests: FAILED\n");
+        goto out;
+    }
+
+    dir = glfs_h_mkdir(fs, parent, my_dir, 0755, &sb);
+    if (dir == NULL) {
+        fprintf(stderr, "glfs_h_mkdir: error creating %s: from (%p),%s\n",
+                my_dir, parent, strerror(errno));
+        printf("glfs_h_performance tests: FAILED\n");
+        goto out;
+    }
+    peek_stat(&sb);
+
+    /* create performance */
+    ret = clock_gettime(CLOCK_REALTIME, &o_ts_st);
+    if (ret != 0) {
+        fprintf(stderr, "clock_gettime: error %s\n", strerror(errno));
+        printf("glfs_h_getattrs and setattrs tests: FAILED\n");
+        goto out;
+    }
+
+    for (i = 0; i < MAX_FILES_CREATE; i++) {
+        sprintf(my_file_name, "%s%d", my_file, i);
+
+        ret = clock_gettime(CLOCK_REALTIME, &c_ts_st);
+        if (ret != 0) {
+            fprintf(stderr, "clock_gettime: error %s\n", strerror(errno));
+            printf("glfs_h_getattrs and setattrs tests: FAILED\n");
+            goto out;
         }
 
-        for (i = 0; i < MAX_FILES_CREATE; i++) {
-                sprintf (my_file_name, "%s%d", my_file, i);
-
-                ret = clock_gettime (CLOCK_REALTIME, &c_ts_st);
-                if(ret != 0) {
-                        fprintf (stderr, "clock_gettime: error %s\n",
-                                 strerror (errno));
-                        printf ("glfs_h_getattrs and setattrs tests: FAILED\n");
-                        goto out;
-                }
-
-                leaf = glfs_h_lookupat (fs, dir, my_file_name, &sb, 0);
-                if (leaf != NULL) {
-                        fprintf (stderr, "glfs_h_lookup: exists %s\n",
-                                 my_file_name);
-                        printf ("glfs_h_performance tests: FAILED\n");
-                        goto out;
-                }
-
-                leaf = glfs_h_creat (fs, dir, my_file_name, O_CREAT, 0644, &sb);
-                if (leaf == NULL) {
-                        fprintf (stderr, "glfs_h_creat: error creating %s: from (%p),%s\n",
-                                 my_file, dir, strerror (errno));
-                        printf ("glfs_h_performance tests: FAILED\n");
-                        goto out;
-                }
-
-                ret = clock_gettime (CLOCK_REALTIME, &c_ts_ed);
-                if(ret != 0) {
-                        fprintf (stderr, "clock_gettime: error %s\n",
-                                 strerror (errno));
-                        printf ("glfs_h_getattrs and setattrs tests: FAILED\n");
-                        goto out;
-                }
-
-                assimilatetime (&c_ts, c_ts_st, c_ts_ed);
-                glfs_h_close (leaf); leaf = NULL;
+        leaf = glfs_h_lookupat(fs, dir, my_file_name, &sb, 0);
+        if (leaf != NULL) {
+            fprintf(stderr, "glfs_h_lookup: exists %s\n", my_file_name);
+            printf("glfs_h_performance tests: FAILED\n");
+            goto out;
         }
 
-        ret = clock_gettime (CLOCK_REALTIME, &o_ts_ed);
-        if(ret != 0) {
-                fprintf (stderr, "clock_gettime: error %s\n", strerror (errno));
-                printf ("glfs_h_getattrs and setattrs tests: FAILED\n");
-                goto out;
+        leaf = glfs_h_creat(fs, dir, my_file_name, O_CREAT, 0644, &sb);
+        if (leaf == NULL) {
+            fprintf(stderr, "glfs_h_creat: error creating %s: from (%p),%s\n",
+                    my_file, dir, strerror(errno));
+            printf("glfs_h_performance tests: FAILED\n");
+            goto out;
         }
 
-        assimilatetime (&o_ts, o_ts_st, o_ts_ed);
-
-        printf ("Creation performance (handle based):\n\t# empty files:%d\n",
-                MAX_FILES_CREATE);
-        printf ("\tOverall time:\n\t\tSecs:%ld\n\t\tnSecs:%ld\n",
-                o_ts.tv_sec, o_ts.tv_nsec);
-        printf ("\tcreate call time time:\n\t\tSecs:%ld\n\t\tnSecs:%ld\n",
-                c_ts.tv_sec, c_ts.tv_nsec);
-
-        /* create using path */
-        c_ts.tv_sec = o_ts.tv_sec = 0;
-        c_ts.tv_nsec = o_ts.tv_nsec = 0;
-
-        sprintf (my_file_name, "%s1", full_dir_path);
-        ret = glfs_mkdir (fs, my_file_name, 0644);
+        ret = clock_gettime(CLOCK_REALTIME, &c_ts_ed);
         if (ret != 0) {
-                fprintf (stderr, "glfs_mkdir: error creating %s: from (%p),%s\n",
-                         my_dir, parent, strerror (errno));
-                printf ("glfs_h_performance tests: FAILED\n");
-                goto out;
-        }
-        peek_stat (&sb);
-
-        ret = clock_gettime (CLOCK_REALTIME, &o_ts_st);
-        if(ret != 0) {
-                fprintf (stderr, "clock_gettime: error %s\n", strerror (errno));
-                printf ("glfs_h_getattrs and setattrs tests: FAILED\n");
-                goto out;
+            fprintf(stderr, "clock_gettime: error %s\n", strerror(errno));
+            printf("glfs_h_getattrs and setattrs tests: FAILED\n");
+            goto out;
+        }
+
+        assimilatetime(&c_ts, c_ts_st, c_ts_ed);
+        glfs_h_close(leaf);
+        leaf = NULL;
+    }
+
+    ret = clock_gettime(CLOCK_REALTIME, &o_ts_ed);
+    if (ret != 0) {
+        fprintf(stderr, "clock_gettime: error %s\n", strerror(errno));
+        printf("glfs_h_getattrs and setattrs tests: FAILED\n");
+        goto out;
+    }
+
+    assimilatetime(&o_ts, o_ts_st, o_ts_ed);
+
+    printf("Creation performance (handle based):\n\t# empty files:%d\n",
+           MAX_FILES_CREATE);
+    printf("\tOverall time:\n\t\tSecs:%ld\n\t\tnSecs:%ld\n", o_ts.tv_sec,
+           o_ts.tv_nsec);
+    printf("\tcreate call time time:\n\t\tSecs:%ld\n\t\tnSecs:%ld\n",
+           c_ts.tv_sec, c_ts.tv_nsec);
+
+    /* create using path */
+    c_ts.tv_sec = o_ts.tv_sec = 0;
+    c_ts.tv_nsec = o_ts.tv_nsec = 0;
+
+    sprintf(my_file_name, "%s1", full_dir_path);
+    ret = glfs_mkdir(fs, my_file_name, 0755);
+    if (ret != 0) {
+        fprintf(stderr, "glfs_mkdir: error creating %s: from (%p),%s\n", my_dir,
+                parent, strerror(errno));
+        printf("glfs_h_performance tests: FAILED\n");
+        goto out;
+    }
+    peek_stat(&sb);
+
+    ret = clock_gettime(CLOCK_REALTIME, &o_ts_st);
+    if (ret != 0) {
+        fprintf(stderr, "clock_gettime: error %s\n", strerror(errno));
+        printf("glfs_h_getattrs and setattrs tests: FAILED\n");
+        goto out;
+    }
+
+    for (i = 0; i < MAX_FILES_CREATE; i++) {
+        sprintf(my_file_name, "%s1/%sn%d", full_dir_path, my_file, i);
+
+        ret = clock_gettime(CLOCK_REALTIME, &c_ts_st);
+        if (ret != 0) {
+            fprintf(stderr, "clock_gettime: error %s\n", strerror(errno));
+            printf("glfs_h_getattrs and setattrs tests: FAILED\n");
+            goto out;
         }
 
-        for (i = 0; i < MAX_FILES_CREATE; i++) {
-                sprintf (my_file_name, "%s1/%sn%d", full_dir_path, my_file, i);
-
-                ret = clock_gettime (CLOCK_REALTIME, &c_ts_st);
-                if(ret != 0) {
-                        fprintf (stderr, "clock_gettime: error %s\n",
-                                 strerror (errno));
-                        printf ("glfs_h_getattrs and setattrs tests: FAILED\n");
-                        goto out;
-                }
-
-                ret = glfs_stat (fs, my_file_name, &sb);
-                if (ret == 0) {
-                        fprintf (stderr, "glfs_stat: exists %s\n",
-                                 my_file_name);
-                        printf ("glfs_h_performance tests: FAILED\n");
-                        goto out;
-                }
-
-                fd = glfs_creat (fs, my_file_name, O_CREAT, 0644);
-                if (fd == NULL) {
-                        fprintf (stderr, "glfs_creat: error creating %s: from (%p),%s\n",
-                                 my_file, dir, strerror (errno));
-                        printf ("glfs_h_performance tests: FAILED\n");
-                        goto out;
-                }
-
-                ret = clock_gettime (CLOCK_REALTIME, &c_ts_ed);
-                if(ret != 0) {
-                        fprintf (stderr, "clock_gettime: error %s\n",
-                                 strerror (errno));
-                        printf ("glfs_h_getattrs and setattrs tests: FAILED\n");
-                        goto out;
-                }
-
-                assimilatetime (&c_ts, c_ts_st, c_ts_ed);
-                glfs_close (fd);
+        ret = glfs_stat(fs, my_file_name, &sb);
+        if (ret == 0) {
+            fprintf(stderr, "glfs_stat: exists %s\n", my_file_name);
+            printf("glfs_h_performance tests: FAILED\n");
+            goto out;
         }
 
-        ret = clock_gettime (CLOCK_REALTIME, &o_ts_ed);
-        if(ret != 0) {
-                fprintf (stderr, "clock_gettime: error %s\n", strerror (errno));
-                printf ("glfs_h_getattrs and setattrs tests: FAILED\n");
-                goto out;
+        fd = glfs_creat(fs, my_file_name, O_CREAT, 0644);
+        if (fd == NULL) {
+            fprintf(stderr, "glfs_creat: error creating %s: from (%p),%s\n",
+                    my_file, dir, strerror(errno));
+            printf("glfs_h_performance tests: FAILED\n");
+            goto out;
         }
 
-        assimilatetime (&o_ts, o_ts_st, o_ts_ed);
-
-        printf ("Creation performance (path based):\n\t# empty files:%d\n",
-                MAX_FILES_CREATE);
-        printf ("\tOverall time:\n\t\tSecs:%ld\n\t\tnSecs:%ld\n",
-                o_ts.tv_sec, o_ts.tv_nsec);
-        printf ("\tcreate call time time:\n\t\tSecs:%ld\n\t\tnSecs:%ld\n",
-                c_ts.tv_sec, c_ts.tv_nsec);
+        ret = clock_gettime(CLOCK_REALTIME, &c_ts_ed);
+        if (ret != 0) {
+            fprintf(stderr, "clock_gettime: error %s\n", strerror(errno));
+            printf("glfs_h_getattrs and setattrs tests: FAILED\n");
+            goto out;
+        }
+
+        assimilatetime(&c_ts, c_ts_st, c_ts_ed);
+        glfs_close(fd);
+    }
+
+    ret = clock_gettime(CLOCK_REALTIME, &o_ts_ed);
+    if (ret != 0) {
+        fprintf(stderr, "clock_gettime: error %s\n", strerror(errno));
+        printf("glfs_h_getattrs and setattrs tests: FAILED\n");
+        goto out;
+    }
+
+    assimilatetime(&o_ts, o_ts_st, o_ts_ed);
+
+    printf("Creation performance (path based):\n\t# empty files:%d\n",
+           MAX_FILES_CREATE);
+    printf("\tOverall time:\n\t\tSecs:%ld\n\t\tnSecs:%ld\n", o_ts.tv_sec,
+           o_ts.tv_nsec);
+    printf("\tcreate call time time:\n\t\tSecs:%ld\n\t\tnSecs:%ld\n",
+           c_ts.tv_sec, c_ts.tv_nsec);
 out:
-        return;
+    return;
 }
 
 int
-test_handleops (int argc, char *argv[])
+test_handleops(int argc, char *argv[])
 {
-        int                 ret = 0;
-        glfs_fd_t          *fd = NULL;
-        struct stat         sb = {0, };
-        struct glfs_object *root = NULL, *parent = NULL, *leaf = NULL,
-                           *tmp = NULL;
-        char                readbuf[32], writebuf[32];
-        unsigned char       leaf_handle[GFAPI_HANDLE_LENGTH];
-
-        char *full_leaf_name = "/testdir/testfile.txt",
-             *leaf_name = "testfile.txt",
-             *relative_leaf_name = "testdir/testfile.txt";
-        char *leaf_name1 = "testfile1.txt";
-        char *full_newparent_name = "/testdir/dir1",
-             *newparent_name = "dir1";
-        char *full_newnod_name = "/testdir/nod1",
-             *newnod_name = "nod1";
-
-        /* Initialize test area */
-        ret = glfs_mkdir (fs, full_parent_name, 0644);
-        if (ret != 0 && errno != EEXIST) {
-                fprintf (stderr, "%s: (%p) %s\n", full_parent_name, fd,
-                        strerror (errno));
-                printf ("Test initialization failed on volume %s\n", argv[1]);
-                goto out;
-        }
-        else if (ret != 0) {
-                printf ("Found test directory %s to be existing\n",
-                        full_parent_name);
-                printf ("Cleanup test directory and restart tests\n");
-                goto out;
-        }
-
-        fd = glfs_creat (fs, full_leaf_name, O_CREAT, 0644);
-        if (fd == NULL) {
-                fprintf (stderr, "%s: (%p) %s\n", full_leaf_name, fd,
-                        strerror (errno));
-                printf ("Test initialization failed on volume %s\n", argv[1]);
-                goto out;
-        }
-        glfs_close (fd);
-
-        printf ("Initialized the test area, within volume %s\n", argv[1]);
-
-        /* Handle based APIs test area */
-
-        /* glfs_lookupat test */
-        printf ("glfs_h_lookupat tests: In Progress\n");
-        /* start at root of the volume */
-        root = glfs_h_lookupat (fs, NULL, "/", &sb, 0);
-        if (root == NULL) {
-                fprintf (stderr, "glfs_h_lookupat: error on lookup of %s: from (%p),%s\n",
-                         "/", NULL, strerror (errno));
-                printf ("glfs_h_lookupat tests: FAILED\n");
-                goto out;
-        }
-        peek_stat (&sb);
-
-        /* lookup a parent within root */
-        parent = glfs_h_lookupat (fs, root, parent_name, &sb, 0);
-        if (parent == NULL) {
-                fprintf (stderr, "glfs_h_lookupat: error on lookup of %s: from (%p),%s\n",
-                         parent_name, root, strerror (errno));
-                printf ("glfs_h_lookupat tests: FAILED\n");
-                goto out;
-        }
-        peek_stat (&sb);
-
-        /* lookup a leaf/child within the parent */
-        leaf = glfs_h_lookupat (fs, parent, leaf_name, &sb, 0);
-        if (leaf == NULL) {
-                fprintf (stderr, "glfs_h_lookupat: error on lookup of %s: from (%p),%s\n",
-                         leaf_name, parent, strerror (errno));
-                printf ("glfs_h_lookupat tests: FAILED\n");
-                goto out;
-        }
-        peek_stat (&sb);
-
-        /* reset */
-        glfs_h_close (root); root = NULL;
-        glfs_h_close (leaf); leaf = NULL;
-        glfs_h_close (parent); parent = NULL;
-
-        /* check absolute paths */
-        root = glfs_h_lookupat (fs, NULL, "/", &sb, 0);
-        if (root == NULL) {
-                fprintf (stderr, "glfs_h_lookupat: error on lookup of %s: from (%p),%s\n",
-                         "/", NULL, strerror (errno));
-                printf ("glfs_h_lookupat tests: FAILED\n");
-                goto out;
-        }
-        peek_stat (&sb);
-
-        parent = glfs_h_lookupat (fs, NULL, full_parent_name, &sb, 0);
-        if (parent == NULL) {
-                fprintf (stderr, "glfs_h_lookupat: error on lookup of %s: from (%p),%s\n",
-                         full_parent_name, root, strerror (errno));
-                printf ("glfs_h_lookupat tests: FAILED\n");
-                goto out;
-        }
-        peek_stat (&sb);
-
-        leaf = glfs_h_lookupat (fs, NULL, full_leaf_name, &sb, 0);
-        if (leaf == NULL) {
-                fprintf (stderr, "glfs_h_lookupat: error on lookup of %s: from (%p),%s\n",
-                         full_leaf_name, parent, strerror (errno));
-                printf ("glfs_h_lookupat tests: FAILED\n");
-                goto out;
-        }
-        peek_stat (&sb);
-
-        /* reset */
-        glfs_h_close (leaf); leaf = NULL;
-
-        /* check multiple component paths */
-        leaf = glfs_h_lookupat (fs, root, relative_leaf_name, &sb, 0);
-        if (leaf == NULL) {
-                fprintf (stderr, "glfs_h_lookupat: error on lookup of %s: from (%p),%s\n",
-                         relative_leaf_name, parent, strerror (errno));
-                goto out;
-        }
-        peek_stat (&sb);
-
-        /* reset */
-        glfs_h_close (root); root = NULL;
-        glfs_h_close (parent); parent = NULL;
-
-        /* check symlinks in path */
-
-        /* TODO: -ve test cases */
-        /* parent invalid
-        * path invalid
-        * path does not exist after some components
-        * no parent, but relative path
-        * parent and full path? -ve?
-        */
-
-        printf ("glfs_h_lookupat tests: PASSED\n");
-
-        /* glfs_openat test */
-        printf ("glfs_h_open tests: In Progress\n");
-        fd = glfs_h_open (fs, leaf, O_RDWR);
-        if (fd == NULL) {
-                fprintf (stderr, "glfs_h_open: error on open of %s: %s\n",
-                         full_leaf_name, strerror (errno));
-                printf ("glfs_h_open tests: FAILED\n");
-                goto out;
-        }
-
-        /* test read/write based on fd */
-        memcpy (writebuf, "abcdefghijklmnopqrstuvwxyz012345", 32);
-        ret = glfs_write (fd, writebuf, 32, 0);
-
-        glfs_lseek (fd, 0, SEEK_SET);
-
-        ret = glfs_read (fd, readbuf, 32, 0);
-        if (memcmp (readbuf, writebuf, 32)) {
-                printf ("Failed to read what I wrote: %s %s\n", readbuf,
-                        writebuf);
-                glfs_close (fd);
-                printf ("glfs_h_open tests: FAILED\n");
-                goto out;
-        }
-
-        glfs_h_close (leaf); leaf = NULL;
-        glfs_close (fd);
-
-        printf ("glfs_h_open tests: PASSED\n");
-
-        /* Create tests */
-        printf ("glfs_h_creat tests: In Progress\n");
-        parent = glfs_h_lookupat (fs, NULL, full_parent_name, &sb, 0);
-        if (parent == NULL) {
-                fprintf (stderr, "glfs_h_lookupat: error on lookup of %s: from (%p),%s\n",
-                         full_parent_name, root, strerror (errno));
-                printf ("glfs_h_creat tests: FAILED\n");
-                goto out;
-        }
-        peek_stat (&sb);
-
-        leaf = glfs_h_creat (fs, parent, leaf_name1, O_CREAT, 0644, &sb);
-        if (leaf == NULL) {
-                fprintf (stderr, "glfs_h_creat: error on create of %s: from (%p),%s\n",
-                         leaf_name1, parent, strerror (errno));
-                printf ("glfs_h_creat tests: FAILED\n");
-                goto out;
-        }
-        peek_stat (&sb);
-
-        glfs_h_close (leaf); leaf = NULL;
-
-        leaf = glfs_h_creat (fs, parent, leaf_name1, O_CREAT | O_EXCL, 0644,
-                            &sb);
-        if (leaf != NULL || errno != EEXIST) {
-                fprintf (stderr, "glfs_h_creat: existing file, leaf = (%p), errno = %s\n",
-                        leaf, strerror (errno));
-                printf ("glfs_h_creat tests: FAILED\n");
-                if (leaf != NULL) {
-                        glfs_h_close (leaf); leaf = NULL;
-                }
-        }
-
-        tmp = glfs_h_creat (fs, root, parent_name, O_CREAT, 0644, &sb);
-        if (tmp != NULL || !(errno == EISDIR || errno == EINVAL)) {
-                fprintf (stderr, "glfs_h_creat: dir create, tmp = (%p), errno = %s\n",
-                        leaf, strerror (errno));
-                printf ("glfs_h_creat tests: FAILED\n");
-                if (tmp != NULL) {
-                        glfs_h_close (tmp); tmp = NULL;
-                }
-        }
-
-        /* TODO: Other combinations and -ve cases as applicable */
-        printf ("glfs_h_creat tests: PASSED\n");
-
-        /* extract handle and create from handle test */
-        printf ("glfs_h_extract_handle and glfs_h_create_from_handle tests: In Progress\n");
-        /* TODO: Change the lookup to creat below for a GIFD recovery falure,
-         * that needs to be fixed */
-        leaf = glfs_h_lookupat (fs, parent, leaf_name1, &sb, 0);
-        if (leaf == NULL) {
-                fprintf (stderr, "glfs_h_lookupat: error on lookup of %s: from (%p),%s\n",
-                         leaf_name1, parent, strerror (errno));
-                printf ("glfs_h_extract_handle tests: FAILED\n");
-                goto out;
-        }
-        peek_stat (&sb);
-
-        ret = glfs_h_extract_handle (leaf, leaf_handle,
-                                             GFAPI_HANDLE_LENGTH);
-        if (ret < 0) {
-                fprintf (stderr, "glfs_h_extract_handle: error extracting handle of %s: %s\n",
-                         full_leaf_name, strerror (errno));
-                printf ("glfs_h_extract_handle tests: FAILED\n");
-                goto out;
-        }
-        peek_handle (leaf_handle);
+    int ret = 0;
+    glfs_fd_t *fd = NULL;
+    struct stat sb = {
+        0,
+    };
+    struct glfs_object *root = NULL, *parent = NULL, *leaf = NULL, *tmp = NULL;
+    char readbuf[32], writebuf[32];
+    unsigned char leaf_handle[GFAPI_HANDLE_LENGTH];
+
+    char *full_leaf_name = "/testdir/testfile.txt", *leaf_name = "testfile.txt",
+         *relative_leaf_name = "testdir/testfile.txt";
+    char *leaf_name1 = "testfile1.txt";
+    char *full_newparent_name = "/testdir/dir1", *newparent_name = "dir1";
+    char *full_newnod_name = "/testdir/nod1", *newnod_name = "nod1";
+
+    /* Initialize test area */
+    ret = glfs_mkdir(fs, full_parent_name, 0755);
+    if (ret != 0 && errno != EEXIST) {
+        fprintf(stderr, "%s: (%p) %s\n", full_parent_name, fd, strerror(errno));
+        printf("Test initialization failed on volume %s\n", argv[1]);
+        goto out;
+    } else if (ret != 0) {
+        printf("Found test directory %s to be existing\n", full_parent_name);
+        printf("Cleanup test directory and restart tests\n");
+        goto out;
+    }
+
+    fd = glfs_creat(fs, full_leaf_name, O_CREAT, 0644);
+    if (fd == NULL) {
+        fprintf(stderr, "%s: (%p) %s\n", full_leaf_name, fd, strerror(errno));
+        printf("Test initialization failed on volume %s\n", argv[1]);
+        goto out;
+    }
+    glfs_close(fd);
+
+    printf("Initialized the test area, within volume %s\n", argv[1]);
+
+    /* Handle based APIs test area */
+
+    /* glfs_lookupat test */
+    printf("glfs_h_lookupat tests: In Progress\n");
+    /* start at root of the volume */
+    root = glfs_h_lookupat(fs, NULL, "/", &sb, 0);
+    if (root == NULL) {
+        fprintf(stderr,
+                "glfs_h_lookupat: error on lookup of %s: from (%p),%s\n", "/",
+                NULL, strerror(errno));
+        printf("glfs_h_lookupat tests: FAILED\n");
+        goto out;
+    }
+    peek_stat(&sb);
+
+    /* lookup a parent within root */
+    parent = glfs_h_lookupat(fs, root, parent_name, &sb, 0);
+    if (parent == NULL) {
+        fprintf(stderr,
+                "glfs_h_lookupat: error on lookup of %s: from (%p),%s\n",
+                parent_name, root, strerror(errno));
+        printf("glfs_h_lookupat tests: FAILED\n");
+        goto out;
+    }
+    peek_stat(&sb);
+
+    /* lookup a leaf/child within the parent */
+    leaf = glfs_h_lookupat(fs, parent, leaf_name, &sb, 0);
+    if (leaf == NULL) {
+        fprintf(stderr,
+                "glfs_h_lookupat: error on lookup of %s: from (%p),%s\n",
+                leaf_name, parent, strerror(errno));
+        printf("glfs_h_lookupat tests: FAILED\n");
+        goto out;
+    }
+    peek_stat(&sb);
+
+    /* reset */
+    glfs_h_close(root);
+    root = NULL;
+    glfs_h_close(leaf);
+    leaf = NULL;
+    glfs_h_close(parent);
+    parent = NULL;
+
+    /* check absolute paths */
+    root = glfs_h_lookupat(fs, NULL, "/", &sb, 0);
+    if (root == NULL) {
+        fprintf(stderr,
+                "glfs_h_lookupat: error on lookup of %s: from (%p),%s\n", "/",
+                NULL, strerror(errno));
+        printf("glfs_h_lookupat tests: FAILED\n");
+        goto out;
+    }
+    peek_stat(&sb);
+
+    parent = glfs_h_lookupat(fs, NULL, full_parent_name, &sb, 0);
+    if (parent == NULL) {
+        fprintf(stderr,
+                "glfs_h_lookupat: error on lookup of %s: from (%p),%s\n",
+                full_parent_name, root, strerror(errno));
+        printf("glfs_h_lookupat tests: FAILED\n");
+        goto out;
+    }
+    peek_stat(&sb);
+
+    leaf = glfs_h_lookupat(fs, NULL, full_leaf_name, &sb, 0);
+    if (leaf == NULL) {
+        fprintf(stderr,
+                "glfs_h_lookupat: error on lookup of %s: from (%p),%s\n",
+                full_leaf_name, parent, strerror(errno));
+        printf("glfs_h_lookupat tests: FAILED\n");
+        goto out;
+    }
+    peek_stat(&sb);
+
+    /* reset */
+    glfs_h_close(leaf);
+    leaf = NULL;
+
+    /* check multiple component paths */
+    leaf = glfs_h_lookupat(fs, root, relative_leaf_name, &sb, 0);
+    if (leaf == NULL) {
+        fprintf(stderr,
+                "glfs_h_lookupat: error on lookup of %s: from (%p),%s\n",
+                relative_leaf_name, parent, strerror(errno));
+        goto out;
+    }
+    peek_stat(&sb);
+
+    /* reset */
+    glfs_h_close(root);
+    root = NULL;
+    glfs_h_close(parent);
+    parent = NULL;
+
+    /* check symlinks in path */
+
+    /* TODO: -ve test cases */
+    /* parent invalid
+     * path invalid
+     * path does not exist after some components
+     * no parent, but relative path
+     * parent and full path? -ve?
+     */
+
+    printf("glfs_h_lookupat tests: PASSED\n");
+
+    /* glfs_openat test */
+    printf("glfs_h_open tests: In Progress\n");
+    fd = glfs_h_open(fs, leaf, O_RDWR);
+    if (fd == NULL) {
+        fprintf(stderr, "glfs_h_open: error on open of %s: %s\n",
+                full_leaf_name, strerror(errno));
+        printf("glfs_h_open tests: FAILED\n");
+        goto out;
+    }
+
+    /* test read/write based on fd */
+    memcpy(writebuf, "abcdefghijklmnopqrstuvwxyz012345", 32);
+    ret = glfs_write(fd, writebuf, 32, 0);
+
+    glfs_lseek(fd, 0, SEEK_SET);
+
+    ret = glfs_read(fd, readbuf, 32, 0);
+    if (memcmp(readbuf, writebuf, 32)) {
+        printf("Failed to read what I wrote: %s %s\n", readbuf, writebuf);
+        glfs_close(fd);
+        printf("glfs_h_open tests: FAILED\n");
+        goto out;
+    }
+
+    glfs_h_close(leaf);
+    leaf = NULL;
+    glfs_close(fd);
+
+    printf("glfs_h_open tests: PASSED\n");
+
+    /* Create tests */
+    printf("glfs_h_creat tests: In Progress\n");
+    parent = glfs_h_lookupat(fs, NULL, full_parent_name, &sb, 0);
+    if (parent == NULL) {
+        fprintf(stderr,
+                "glfs_h_lookupat: error on lookup of %s: from (%p),%s\n",
+                full_parent_name, root, strerror(errno));
+        printf("glfs_h_creat tests: FAILED\n");
+        goto out;
+    }
+    peek_stat(&sb);
+
+    leaf = glfs_h_creat(fs, parent, leaf_name1, O_CREAT, 0644, &sb);
+    if (leaf == NULL) {
+        fprintf(stderr, "glfs_h_creat: error on create of %s: from (%p),%s\n",
+                leaf_name1, parent, strerror(errno));
+        printf("glfs_h_creat tests: FAILED\n");
+        goto out;
+    }
+    peek_stat(&sb);
+
+    glfs_h_close(leaf);
+    leaf = NULL;
+
+    leaf = glfs_h_creat(fs, parent, leaf_name1, O_CREAT | O_EXCL, 0644, &sb);
+    if (leaf != NULL || errno != EEXIST) {
+        fprintf(stderr,
+                "glfs_h_creat: existing file, leaf = (%p), errno = %s\n", leaf,
+                strerror(errno));
+        printf("glfs_h_creat tests: FAILED\n");
+        if (leaf != NULL) {
+            glfs_h_close(leaf);
+            leaf = NULL;
+        }
+    }
+
+    tmp = glfs_h_creat(fs, root, parent_name, O_CREAT, 0644, &sb);
+    if (tmp != NULL || !(errno == EISDIR || errno == EINVAL)) {
+        fprintf(stderr, "glfs_h_creat: dir create, tmp = (%p), errno = %s\n",
+                leaf, strerror(errno));
+        printf("glfs_h_creat tests: FAILED\n");
+        if (tmp != NULL) {
+            glfs_h_close(tmp);
+            tmp = NULL;
+        }
+    }
+
+    /* TODO: Other combinations and -ve cases as applicable */
+    printf("glfs_h_creat tests: PASSED\n");
+
+    /* extract handle and create from handle test */
+    printf(
+        "glfs_h_extract_handle and glfs_h_create_from_handle tests: In "
+        "Progress\n");
+    /* TODO: Change the lookup to create below for a GIFD recovery failure,
+     * that needs to be fixed */
+    leaf = glfs_h_lookupat(fs, parent, leaf_name1, &sb, 0);
+    if (leaf == NULL) {
+        fprintf(stderr,
+                "glfs_h_lookupat: error on lookup of %s: from (%p),%s\n",
+                leaf_name1, parent, strerror(errno));
+        printf("glfs_h_extract_handle tests: FAILED\n");
+        goto out;
+    }
+    peek_stat(&sb);
+
+    ret = glfs_h_extract_handle(leaf, leaf_handle, GFAPI_HANDLE_LENGTH);
+    if (ret < 0) {
+        fprintf(stderr,
+                "glfs_h_extract_handle: error extracting handle of %s: %s\n",
+                full_leaf_name, strerror(errno));
+        printf("glfs_h_extract_handle tests: FAILED\n");
+        goto out;
+    }
+    peek_handle(leaf_handle);
+
+    glfs_h_close(leaf);
+    leaf = NULL;
+
+    leaf = glfs_h_create_from_handle(fs, leaf_handle, GFAPI_HANDLE_LENGTH, &sb);
+    if (leaf == NULL) {
+        fprintf(
+            stderr,
+            "glfs_h_create_from_handle: error on create of %s: from (%p),%s\n",
+            leaf_name1, leaf_handle, strerror(errno));
+        printf("glfs_h_create_from_handle tests: FAILED\n");
+        goto out;
+    }
+    peek_stat(&sb);
+
+    fd = glfs_h_open(fs, leaf, O_RDWR);
+    if (fd == NULL) {
+        fprintf(stderr, "glfs_h_open: error on open of %s: %s\n",
+                full_leaf_name, strerror(errno));
+        printf("glfs_h_create_from_handle tests: FAILED\n");
+        goto out;
+    }
+
+    /* test read/write based on fd */
+    memcpy(writebuf, "abcdefghijklmnopqrstuvwxyz012345", 32);
+    ret = glfs_write(fd, writebuf, 32, 0);
+
+    glfs_lseek(fd, 0, SEEK_SET);
+
+    ret = glfs_read(fd, readbuf, 32, 0);
+    if (memcmp(readbuf, writebuf, 32)) {
+        printf("Failed to read what I wrote: %s %s\n", writebuf, writebuf);
+        printf("glfs_h_create_from_handle tests: FAILED\n");
+        glfs_close(fd);
+        goto out;
+    }
+
+    glfs_close(fd);
+    glfs_h_close(leaf);
+    leaf = NULL;
+    glfs_h_close(parent);
+    parent = NULL;
+
+    printf(
+        "glfs_h_extract_handle and glfs_h_create_from_handle tests: PASSED\n");
+
+    /* Mkdir tests */
+    printf("glfs_h_mkdir tests: In Progress\n");
+
+    ret = glfs_rmdir(fs, full_newparent_name);
+    if (ret && errno != ENOENT) {
+        fprintf(stderr, "glfs_rmdir: Failed for %s: %s\n", full_newparent_name,
+                strerror(errno));
+        printf("glfs_h_mkdir tests: FAILED\n");
+        goto out;
+    }
+
+    parent = glfs_h_lookupat(fs, NULL, full_parent_name, &sb, 0);
+    if (parent == NULL) {
+        fprintf(stderr,
+                "glfs_h_lookupat: error on lookup of %s: from (%p),%s\n",
+                full_parent_name, root, strerror(errno));
+        printf("glfs_h_mkdir tests: FAILED\n");
+        goto out;
+    }
+    peek_stat(&sb);
+
+    leaf = glfs_h_mkdir(fs, parent, newparent_name, 0755, &sb);
+    if (leaf == NULL) {
+        fprintf(stderr, "glfs_h_mkdir: error on mkdir of %s: from (%p),%s\n",
+                newparent_name, parent, strerror(errno));
+        printf("glfs_h_mkdir tests: FAILED\n");
+        goto out;
+    }
+    peek_stat(&sb);
+
+    glfs_h_close(leaf);
+    leaf = NULL;
+
+    leaf = glfs_h_mkdir(fs, parent, newparent_name, 0755, &sb);
+    if (leaf != NULL || errno != EEXIST) {
+        fprintf(stderr,
+                "glfs_h_mkdir: existing directory, leaf = (%p), errno = %s\n",
+                leaf, strerror(errno));
+        printf("glfs_h_mkdir tests: FAILED\n");
+        if (leaf != NULL) {
+            glfs_h_close(leaf);
+            leaf = NULL;
+        }
+    }
+
+    glfs_h_close(parent);
+    parent = NULL;
+
+    printf("glfs_h_mkdir tests: PASSED\n");
+
+    /* Mknod tests */
+    printf("glfs_h_mknod tests: In Progress\n");
+    ret = glfs_unlink(fs, full_newnod_name);
+    if (ret && errno != ENOENT) {
+        fprintf(stderr, "glfs_unlink: Failed for %s: %s\n", full_newnod_name,
+                strerror(errno));
+        printf("glfs_h_mknod tests: FAILED\n");
+        goto out;
+    }
+
+    parent = glfs_h_lookupat(fs, NULL, full_parent_name, &sb, 0);
+    if (parent == NULL) {
+        fprintf(stderr,
+                "glfs_h_lookupat: error on lookup of %s: from (%p),%s\n",
+                full_parent_name, root, strerror(errno));
+        printf("glfs_h_mknod tests: FAILED\n");
+        goto out;
+    }
+    peek_stat(&sb);
+
+    leaf = glfs_h_mknod(fs, parent, newnod_name, S_IFIFO, 0, &sb);
+    if (leaf == NULL) {
+        fprintf(stderr, "glfs_h_mkdir: error on mkdir of %s: from (%p),%s\n",
+                newnod_name, parent, strerror(errno));
+        printf("glfs_h_mknod tests: FAILED\n");
+        goto out;
+    }
+    peek_stat(&sb);
+
+    /* TODO: create op on a FIFO node hangs, need to check and fix
+    tmp = glfs_h_creat (fs, parent, newnod_name, O_CREAT, 0644, &sb);
+    if (tmp != NULL || errno != EINVAL) {
+            fprintf (stderr, "glfs_h_creat: node create, tmp = (%p), errno =
+    %s\n", tmp, strerror (errno)); printf ("glfs_h_creat/mknod tests:
+    FAILED\n"); if (tmp != NULL) { glfs_h_close(tmp); tmp = NULL;
+            }
+    } */
+
+    glfs_h_close(leaf);
+    leaf = NULL;
+
+    leaf = glfs_h_mknod(fs, parent, newnod_name, 0644, 0, &sb);
+    if (leaf != NULL || errno != EEXIST) {
+        fprintf(stderr,
+                "glfs_h_mknod: existing node, leaf = (%p), errno = %s\n", leaf,
+                strerror(errno));
+        printf("glfs_h_mknod tests: FAILED\n");
+        if (leaf != NULL) {
+            glfs_h_close(leaf);
+            leaf = NULL;
+        }
+    }
+
+    glfs_h_close(parent);
+    parent = NULL;
+
+    printf("glfs_h_mknod tests: PASSED\n");
+
+    /* unlink tests */
+    test_h_unlink();
+
+    /* TODO: opendir tests */
+
+    /* getattr tests */
+    test_h_getsetattrs();
+
+    /* TODO: setattr tests */
+
+    /* truncate tests */
+    test_h_truncate();
+
+    /* link tests */
+    test_h_links();
+
+    /* rename tests */
+    test_h_rename();
+
+    /* performance tests */
+    test_h_performance();
+
+    /* END: New APIs test area */
 
-        glfs_h_close (leaf); leaf = NULL;
-
-        leaf = glfs_h_create_from_handle (fs, leaf_handle, GFAPI_HANDLE_LENGTH,
-                                          &sb);
-        if (leaf == NULL) {
-                fprintf (stderr, "glfs_h_create_from_handle: error on create of %s: from (%p),%s\n",
-                         leaf_name1, leaf_handle, strerror (errno));
-                printf ("glfs_h_create_from_handle tests: FAILED\n");
-                goto out;
-        }
-        peek_stat (&sb);
-
-        fd = glfs_h_open (fs, leaf, O_RDWR);
-        if (fd == NULL) {
-                fprintf (stderr, "glfs_h_open: error on open of %s: %s\n",
-                         full_leaf_name, strerror (errno));
-                printf ("glfs_h_create_from_handle tests: FAILED\n");
-                goto out;
-        }
-
-        /* test read/write based on fd */
-        memcpy (writebuf, "abcdefghijklmnopqrstuvwxyz012345", 32);
-        ret = glfs_write (fd, writebuf, 32, 0);
-
-        glfs_lseek (fd, 0, SEEK_SET);
-
-        ret = glfs_read (fd, readbuf, 32, 0);
-        if (memcmp (readbuf, writebuf, 32)) {
-                printf ("Failed to read what I wrote: %s %s\n", writebuf,
-                        writebuf);
-                printf ("glfs_h_create_from_handle tests: FAILED\n");
-                glfs_close (fd);
-                goto out;
-        }
-
-        glfs_close (fd);
-        glfs_h_close (leaf); leaf = NULL;
-        glfs_h_close (parent); parent = NULL;
-
-        printf ("glfs_h_extract_handle and glfs_h_create_from_handle tests: PASSED\n");
-
-        /* Mkdir tests */
-        printf ("glfs_h_mkdir tests: In Progress\n");
+out:
+    /* Cleanup glfs handles */
+    if (root)
+        glfs_h_close(root);
+    if (parent)
+        glfs_h_close(parent);
+    if (leaf)
+        glfs_h_close(leaf);
+
+    return ret;
+}
 
-        ret = glfs_rmdir (fs, full_newparent_name);
-        if (ret && errno != ENOENT) {
-                fprintf (stderr, "glfs_rmdir: Failed for %s: %s\n",
-                         full_newparent_name, strerror (errno));
-                printf ("glfs_h_mkdir tests: FAILED\n");
-                goto out;
-        }
+int
+test_write_apis(glfs_t *fs)
+{
+    /* Add more content here */
+    /* Some apis we can get are */
+    /*
+      0. glfs_set_xlator_option()
+
+      Read/Write combinations:
+      . glfs_{p,}readv/{p,}writev
+      . glfs_pread/pwrite
+
+      tests/basic/gfapi/gfapi-async-calls-test.c
+      . glfs_read_async/write_async
+      . glfs_pread_async/pwrite_async
+      . glfs_readv_async/writev_async
+      . glfs_preadv_async/pwritev_async
+
+      . ftruncate/ftruncate_async
+      . fsync/fsync_async
+      . fdatasync/fdatasync_async
+
+    */
+
+    glfs_fd_t *fd = NULL;
+    char *filename = "/filename2";
+    int flags = O_RDWR;
+    char *buf = "some bytes!";
+    char writestr[TEST_STR_LEN];
+    struct iovec iov = {&writestr, TEST_STR_LEN};
+    int ret, i;
+
+    for (i = 0; i < TEST_STR_LEN; i++)
+        writestr[i] = 0x11;
+
+    fd = glfs_open(fs, filename, flags);
+    if (!fd)
+        fprintf(stderr, "open(%s): (%p) %s\n", filename, fd, strerror(errno));
+
+    ret = glfs_writev(fd, &iov, 1, flags);
+    if (ret < 0) {
+        fprintf(stderr, "writev(%s): %d (%s)\n", filename, ret,
+                strerror(errno));
+    }
+
+    ret = glfs_pwrite(fd, buf, 10, 4, flags, NULL, NULL);
+    if (ret < 0) {
+        fprintf(stderr, "pwrite(%s): %d (%s)\n", filename, ret,
+                strerror(errno));
+    }
+
+    ret = glfs_pwritev(fd, &iov, 1, 4, flags);
+    if (ret < 0) {
+        fprintf(stderr, "pwritev(%s): %d (%s)\n", filename, ret,
+                strerror(errno));
+    }
+
+    return 0;
+}
 
-        parent = glfs_h_lookupat (fs, NULL, full_parent_name, &sb, 0);
-        if (parent == NULL) {
-                fprintf (stderr, "glfs_h_lookupat: error on lookup of %s: from (%p),%s\n",
-                         full_parent_name, root, strerror (errno));
-                printf ("glfs_h_mkdir tests: FAILED\n");
-                goto out;
-        }
-        peek_stat (&sb);
+int
+test_metadata_ops(glfs_t *fs, glfs_t *fs2)
+{
+    glfs_fd_t *fd = NULL;
+    glfs_fd_t *fd2 = NULL;
+    struct stat sb = {
+        0,
+    };
+    struct glfs_stat gsb = {
+        0,
+    };
+    struct statvfs sfs;
+    char readbuf[32];
+    char writebuf[32];
 
-        leaf = glfs_h_mkdir (fs, parent, newparent_name, 0644, &sb);
-        if (leaf == NULL) {
-                fprintf (stderr, "glfs_h_mkdir: error on mkdir of %s: from (%p),%s\n",
-                         newparent_name, parent, strerror (errno));
-                printf ("glfs_h_mkdir tests: FAILED\n");
-                goto out;
-        }
-        peek_stat (&sb);
-
-        glfs_h_close (leaf); leaf = NULL;
-
-        leaf = glfs_h_mkdir (fs, parent, newparent_name, 0644, &sb);
-        if (leaf != NULL || errno != EEXIST) {
-                fprintf (stderr, "glfs_h_mkdir: existing directory, leaf = (%p), errno = %s\n",
-                         leaf, strerror (errno));
-                printf ("glfs_h_mkdir tests: FAILED\n");
-                if (leaf != NULL) {
-                        glfs_h_close (leaf); leaf = NULL;
-                }
-        }
+    char *filename = "/filename2";
+    int ret;
 
-        glfs_h_close (parent); parent = NULL;
+    ret = glfs_lstat(fs, filename, &sb);
+    fprintf(stderr, "lstat(%s): (%d) %s\n", filename, ret, strerror(errno));
 
-        printf ("glfs_h_mkdir tests: PASSED\n");
+    fd = glfs_creat(fs, filename, O_RDWR, 0644);
+    fprintf(stderr, "creat(%s): (%p) %s\n", filename, fd, strerror(errno));
 
-        /* Mknod tests */
-        printf ("glfs_h_mknod tests: In Progress\n");
-        ret = glfs_unlink (fs, full_newnod_name);
-        if (ret && errno != ENOENT) {
-                fprintf (stderr, "glfs_unlink: Failed for %s: %s\n",
-                         full_newnod_name, strerror (errno));
-                printf ("glfs_h_mknod tests: FAILED\n");
-                goto out;
-        }
+    fd2 = glfs_open(fs2, filename, O_RDWR);
+    fprintf(stderr, "open(%s): (%p) %s\n", filename, fd, strerror(errno));
 
-        parent = glfs_h_lookupat (fs, NULL, full_parent_name, &sb, 0);
-        if (parent == NULL) {
-                fprintf (stderr, "glfs_h_lookupat: error on lookup of %s: from (%p),%s\n",
-                         full_parent_name, root, strerror (errno));
-                printf ("glfs_h_mknod tests: FAILED\n");
-                goto out;
-        }
-        peek_stat (&sb);
+    glfs_lseek(fd2, 0, SEEK_SET);
 
-        leaf = glfs_h_mknod (fs, parent, newnod_name, S_IFIFO, 0, &sb);
-        if (leaf == NULL) {
-                fprintf (stderr, "glfs_h_mkdir: error on mkdir of %s: from (%p),%s\n",
-                         newnod_name, parent, strerror (errno));
-                printf ("glfs_h_mknod tests: FAILED\n");
-                goto out;
-        }
-        peek_stat (&sb);
-
-        /* TODO: creat op on a FIFO node hangs, need to check and fix
-        tmp = glfs_h_creat (fs, parent, newnod_name, O_CREAT, 0644, &sb);
-        if (tmp != NULL || errno != EINVAL) {
-                fprintf (stderr, "glfs_h_creat: node create, tmp = (%p), errno = %s\n",
-                        tmp, strerror (errno));
-                printf ("glfs_h_creat/mknod tests: FAILED\n");
-                if (tmp != NULL) {
-                        glfs_h_close(tmp); tmp = NULL;
-                }
-        } */
-
-        glfs_h_close (leaf); leaf = NULL;
-
-        leaf = glfs_h_mknod (fs, parent, newnod_name, 0644, 0, &sb);
-        if (leaf != NULL || errno != EEXIST) {
-                fprintf (stderr, "glfs_h_mknod: existing node, leaf = (%p), errno = %s\n",
-                         leaf, strerror (errno));
-                printf ("glfs_h_mknod tests: FAILED\n");
-                if (leaf != NULL) {
-                        glfs_h_close (leaf); leaf = NULL;
-                }
-        }
+    ret = glfs_read(fd2, readbuf, 32, 0);
 
-        glfs_h_close (parent); parent = NULL;
+    printf("read %d, %s", ret, readbuf);
 
-        printf ("glfs_h_mknod tests: PASSED\n");
+    /* get stat */
+    ret = glfs_fstat(fd2, &sb);
 
-        /* unlink tests */
-        test_h_unlink ();
+    ret = glfs_access(fs, filename, R_OK);
 
-        /* TODO: opendir tests */
+    /* set stat */
+    /* TODO: got some errors, need to fix */
+    /* ret = glfs_fsetattr(fd2, &gsb); */
 
-        /* getattr tests */
-        test_h_getsetattrs ();
+    glfs_close(fd);
+    glfs_close(fd2);
 
-        /* TODO: setattr tests */
+    filename = "/filename3";
+    ret = glfs_mknod(fs, filename, S_IFIFO, 0);
+    fprintf(stderr, "%s: (%d) %s\n", filename, ret, strerror(errno));
 
-        /* truncate tests */
-        test_h_truncate();
+    ret = glfs_lstat(fs, filename, &sb);
+    fprintf(stderr, "%s: (%d) %s\n", filename, ret, strerror(errno));
 
-        /* link tests */
-        test_h_links ();
+    ret = glfs_rename(fs, filename, "/filename4");
+    fprintf(stderr, "rename(%s): (%d) %s\n", filename, ret, strerror(errno));
 
-        /* rename tests */
-        test_h_rename ();
+    ret = glfs_unlink(fs, "/filename4");
+    fprintf(stderr, "unlink(%s): (%d) %s\n", "/filename4", ret,
+            strerror(errno));
 
-        /* performance tests */
-        test_h_performance ();
+    filename = "/dirname2";
+    ret = glfs_mkdir(fs, filename, 0);
+    fprintf(stderr, "%s: (%d) %s\n", filename, ret, strerror(errno));
 
-        /* END: New APIs test area */
+    ret = glfs_lstat(fs, filename, &sb);
+    fprintf(stderr, "lstat(%s): (%d) %s\n", filename, ret, strerror(errno));
 
-out:
-        /* Cleanup glfs handles */
-        if (root)
-                glfs_h_close (root);
-        if (parent)
-                glfs_h_close (parent);
-        if (leaf)
-                glfs_h_close (leaf);
-
-        return ret;
+    ret = glfs_rmdir(fs, filename);
+    fprintf(stderr, "rmdir(%s): (%d) %s\n", filename, ret, strerror(errno));
 }
-
 int
-main (int argc, char *argv[])
+main(int argc, char *argv[])
 {
-        glfs_t    *fs2 = NULL;
-        int        ret = 0;
-        glfs_fd_t *fd = NULL;
-        glfs_fd_t *fd2 = NULL;
-        struct stat sb = {0, };
-        char       readbuf[32];
-        char       writebuf[32];
-
-        char      *filename = "/filename2";
-
-        if (argc != 3) {
-                printf ("Expect following args\n\t%s <volname> <hostname>\n", argv[0]);
-                return -1;
-        }
-
-        fs = glfs_new (argv[1]);
-        if (!fs) {
-                fprintf (stderr, "glfs_new: returned NULL\n");
-                return 1;
-        }
-
-//      ret = glfs_set_volfile (fs, "/tmp/posix.vol");
-
-        ret = glfs_set_volfile_server (fs, "tcp", argv[2], 24007);
-
-//      ret = glfs_set_volfile_server (fs, "unix", "/tmp/gluster.sock", 0);
-
-        ret = glfs_set_logging (fs, "/dev/stderr", 7);
+    glfs_t *fs2 = NULL;
+    int ret = 0;
+    glfs_fd_t *fd = NULL;
+    glfs_fd_t *fd2 = NULL;
+    struct stat sb = {
+        0,
+    };
+    struct glfs_stat gsb = {
+        0,
+    };
+    struct statvfs sfs;
+    char readbuf[32];
+    char writebuf[32];
 
-        ret = glfs_init (fs);
+    char *filename = "/filename2";
 
-        fprintf (stderr, "glfs_init: returned %d\n", ret);
+    if (argc != 3) {
+        printf("Expect following args\n\t%s <volname> <hostname>\n", argv[0]);
+        return -1;
+    }
 
-        sleep (2);
+    fs = glfs_new(argv[1]);
+    if (!fs) {
+        fprintf(stderr, "glfs_new: returned NULL\n");
+        return 1;
+    }
 
-        fs2 = glfs_new (argv[1]);
-        if (!fs2) {
-                fprintf (stderr, "glfs_new: returned NULL\n");
-                return 1;
-        }
-
-
-//      ret = glfs_set_volfile (fs2, "/tmp/posix.vol");
-
-        ret = glfs_set_volfile_server (fs2, "tcp", argv[2], 24007);
+    //      ret = glfs_set_volfile (fs, "/tmp/posix.vol");
 
-        ret = glfs_set_logging (fs2, "/dev/stderr", 7);
+    ret = glfs_set_volfile_server(fs, "tcp", argv[2], 24007);
 
-        ret = glfs_init (fs2);
+    //      ret = glfs_set_volfile_server (fs, "unix", "/tmp/gluster.sock", 0);
 
-        fprintf (stderr, "glfs_init: returned %d\n", ret);
+    ret = glfs_set_logging(fs, "/dev/stderr", 7);
 
-        ret = glfs_lstat (fs, filename, &sb);
-        fprintf (stderr, "%s: (%d) %s\n", filename, ret, strerror (errno));
+    ret = glfs_init(fs);
 
-        fd = glfs_creat (fs, filename, O_RDWR, 0644);
-        fprintf (stderr, "%s: (%p) %s\n", filename, fd, strerror (errno));
+    fprintf(stderr, "glfs_init: returned %d\n", ret);
 
-        fd2 = glfs_open (fs2, filename, O_RDWR);
-        fprintf (stderr, "%s: (%p) %s\n", filename, fd, strerror (errno));
+    if (ret)
+        goto out;
 
-        sprintf (writebuf, "hi there\n");
-        ret = glfs_write (fd, writebuf, 32, 0);
+    sleep(2);
 
-        glfs_lseek (fd2, 0, SEEK_SET);
+    fs2 = glfs_new(argv[1]);
+    if (!fs2) {
+        fprintf(stderr, "glfs_new: returned NULL\n");
+        return 1;
+    }
 
-        ret = glfs_read (fd2, readbuf, 32, 0);
+    //      ret = glfs_set_volfile (fs2, "/tmp/posix.vol");
 
-        printf ("read %d, %s", ret, readbuf);
+    ret = glfs_set_volfile_server(fs2, "tcp", argv[2], 24007);
 
-        glfs_close (fd);
-        glfs_close (fd2);
+    ret = glfs_set_logging(fs2, "/dev/stderr", 7);
 
-        filename = "/filename3";
-        ret = glfs_mknod (fs, filename, S_IFIFO, 0);
-        fprintf (stderr, "%s: (%d) %s\n", filename, ret, strerror (errno));
+    ret = glfs_init(fs2);
 
-        ret = glfs_lstat (fs, filename, &sb);
-        fprintf (stderr, "%s: (%d) %s\n", filename, ret, strerror (errno));
+    fprintf(stderr, "glfs_init: returned %d\n", ret);
 
+    test_metadata_ops(fs, fs2);
 
-        ret = glfs_rename (fs, filename, "/filename4");
-        fprintf (stderr, "rename(%s): (%d) %s\n", filename, ret,
-                 strerror (errno));
+    test_dirops(fs);
 
-        ret = glfs_unlink (fs, "/filename4");
-        fprintf (stderr, "unlink(%s): (%d) %s\n", "/filename4", ret,
-                 strerror (errno));
+    test_xattr(fs);
 
-        filename = "/dirname2";
-        ret = glfs_mkdir (fs, filename, 0);
-        fprintf (stderr, "%s: (%d) %s\n", filename, ret, strerror (errno));
+    test_chdir(fs);
 
-        ret = glfs_lstat (fs, filename, &sb);
-        fprintf (stderr, "lstat(%s): (%d) %s\n", filename, ret, strerror (errno));
+    test_handleops(argc, argv);
+    // done
 
-        ret = glfs_rmdir (fs, filename);
-        fprintf (stderr, "rmdir(%s): (%d) %s\n", filename, ret, strerror (errno));
+    /* Test some extra apis */
+    test_write_apis(fs);
 
-        test_dirops (fs);
+    glfs_statvfs(fs, "/", &sfs);
 
-        test_xattr (fs);
+    glfs_fini(fs);
+    glfs_fini(fs2);
 
-        test_chdir (fs);
-
-        test_handleops (argc, argv);
-        // done
-
-        glfs_fini (fs);
-        glfs_fini (fs2);
-
-        return ret;
+    ret = 0;
+out:
+    return ret;
 }
diff --git a/api/src/Makefile.am b/api/src/Makefile.am
index e01b384726c..7f9a7d17b35 100644
--- a/api/src/Makefile.am
+++ b/api/src/Makefile.am
@@ -9,20 +9,20 @@ libgfapi_la_SOURCES = glfs.c glfs-mgmt.c glfs-fops.c glfs-resolve.c \
 	glfs-handleops.c
 libgfapi_la_LIBADD = $(top_builddir)/libglusterfs/src/libglusterfs.la \
 	$(top_builddir)/rpc/rpc-lib/src/libgfrpc.la \
-	$(top_builddir)/rpc/xdr/src/libgfxdr.la \
-	$(GF_LDADD)
+	$(top_builddir)/rpc/xdr/src/libgfxdr.la
+
+libgfapi_la_LDFLAGS = -version-info $(GFAPI_LT_VERSION) $(GF_LDFLAGS) \
+	$(GFAPI_EXTRA_LDFLAGS) $(ACL_LIBS)
 
-AM_CPPFLAGS = $(GF_CPPFLAGS) -I$(top_srcdir)/libglusterfs/src \
+libgfapi_la_CPPFLAGS = $(GF_CPPFLAGS) -I$(top_srcdir)/libglusterfs/src \
         -I$(top_srcdir)/rpc/rpc-lib/src \
         -I$(top_srcdir)/rpc/xdr/src \
+        -I$(top_builddir)/rpc/xdr/src \
         -DDATADIR=\"$(localstatedir)\" \
-        -D__USE_FILE_OFFSET64
+        -D__USE_FILE_OFFSET64 -D__USE_LARGEFILE64
 
 AM_CFLAGS = -Wall $(GF_CFLAGS)
 
-libgfapi_la_LDFLAGS = -version-info $(GFAPI_LT_VERSION) \
-	$(GFAPI_EXTRA_LDFLAGS) $(ACL_LIBS)
-
 xlator_LTLIBRARIES = api.la
 xlatordir = $(libdir)/glusterfs/$(PACKAGE_VERSION)/xlator/mount
 # workaround for broken parallel install support in automake with LTLIBRARIES
@@ -32,9 +32,12 @@ $(install_xlatorLTLIBRARIES): install-libLTLIBRARIES
 
 api_la_SOURCES = glfs-master.c
 api_la_DEPENDENCIES = libgfapi.la
+api_la_CPPFLAGS = $(GF_CPPFLAGS) -I$(top_srcdir)/libglusterfs/src \
+	-I$(top_srcdir)/rpc/xdr/src \
+	-I$(top_builddir)/rpc/xdr/src
 api_la_LDFLAGS = -module $(GF_XLATOR_DEFAULT_LDFLAGS)
+#api_la_LDFLAGS = -module $(GF_XLATOR_DEFAULT_LDFLAGS) $(GF_LDFLAGS)
 api_la_LIBADD = $(top_builddir)/libglusterfs/src/libglusterfs.la \
 	$(top_builddir)/rpc/rpc-lib/src/libgfrpc.la \
 	$(top_builddir)/rpc/xdr/src/libgfxdr.la \
 	$(top_builddir)/api/src/libgfapi.la
-
diff --git a/api/src/README.Symbol_Versions b/api/src/README.Symbol_Versions
index d5cdedd826b..b6ec95f9311 100644
--- a/api/src/README.Symbol_Versions
+++ b/api/src/README.Symbol_Versions
@@ -1,3 +1,3 @@
 
-See .../doc/gfapi-symbol-versions/gfapi-symbol-versions.md
+See ../../doc/developer-guide/gfapi-symbol-versions.md
 
diff --git a/api/src/gfapi-messages.h b/api/src/gfapi-messages.h
index 050b9766dea..b9223940416 100644
--- a/api/src/gfapi-messages.h
+++ b/api/src/gfapi-messages.h
@@ -1,5 +1,5 @@
 /*
- *   Copyright (c) 2015 Red Hat, Inc. <http://www.redhat.com>
+ *   Copyright (c) 2015-2018 Red Hat, Inc. <http://www.redhat.com>
  *   This file is part of GlusterFS.
  *
  *   This file is licensed to you under your choice of the GNU Lesser
@@ -11,90 +11,137 @@
 #ifndef _GFAPI_MESSAGES_H__
 #define _GFAPI_MESSAGES_H__
 
-#include "glfs-message-id.h"
+#include <glusterfs/glfs-message-id.h>
 
-/*! \file gfapi-messages.h
- *  \brief libgfapi log-message IDs and their descriptions
- */
-
-/* NOTE: Rules for message additions
- * 1) Each instance of a message is _better_ left with a unique message ID, even
- *    if the message format is the same. Reasoning is that, if the message
- *    format needs to change in one instance, the other instances are not
- *    impacted or the new change does not change the ID of the instance being
- *    modified.
- * 2) Addition of a message,
- *       - Should increment the GLFS_NUM_MESSAGES
- *       - Append to the list of messages defined, towards the end
- *       - Retain macro naming as glfs_msg_X (for redability across developers)
- * NOTE: Rules for message format modifications
- * 3) Check acorss the code if the message ID macro in question is reused
- *    anywhere. If reused then then the modifications should ensure correctness
- *    everywhere, or needs a new message ID as (1) above was not adhered to. If
- *    not used anywhere, proceed with the required modification.
- * NOTE: Rules for message deletion
- * 4) Check (3) and if used anywhere else, then cannot be deleted. If not used
- *    anywhere, then can be deleted, but will leave a hole by design, as
- *    addition rules specify modification to the end of the list and not filling
- *    holes.
+/* To add new message IDs, append new identifiers at the end of the list.
+ *
+ * Never remove a message ID. If it's not used anymore, you can rename it or
+ * leave it as it is, but not delete it. This is to prevent reutilization of
+ * IDs by other messages.
+ *
+ * The component name must match one of the entries defined in
+ * glfs-message-id.h.
  */
 
-#define GLFS_GFAPI_BASE             GLFS_MSGID_COMP_API
-#define GLFS_NUM_MESSAGES           47
-#define GLFS_MSGID_END              (GLFS_GFAPI_BASE + GLFS_NUM_MESSAGESi + 1)
-/* Messages with message IDs */
-#define glfs_msg_start_x GLFS_GFAPI_BASE, "Invalid: Start of messages"
-/*------------*/
-
-#define API_MSG_MEM_ACCT_INIT_FAILED            (GLFS_GFAPI_BASE + 1)
-#define API_MSG_MASTER_XLATOR_INIT_FAILED       (GLFS_GFAPI_BASE + 2)
-#define API_MSG_GFAPI_XLATOR_INIT_FAILED        (GLFS_GFAPI_BASE + 3)
-#define API_MSG_VOLFILE_OPEN_FAILED             (GLFS_GFAPI_BASE + 4)
-#define API_MSG_VOL_SPEC_FILE_ERROR             (GLFS_GFAPI_BASE + 5)
-#define API_MSG_GLFS_FSOBJ_NULL                 (GLFS_GFAPI_BASE + 6)
-#define API_MSG_INVALID_ENTRY                   (GLFS_GFAPI_BASE + 7)
-#define API_MSG_FSMUTEX_LOCK_FAILED             (GLFS_GFAPI_BASE + 8)
-#define API_MSG_COND_WAIT_FAILED                (GLFS_GFAPI_BASE + 9)
-#define API_MSG_FSMUTEX_UNLOCK_FAILED           (GLFS_GFAPI_BASE + 10)
-#define API_MSG_INODE_REFRESH_FAILED            (GLFS_GFAPI_BASE + 11)
-#define API_MSG_GRAPH_CONSTRUCT_FAILED          (GLFS_GFAPI_BASE + 12)
-#define API_MSG_FUSE_XLATOR_ERROR               (GLFS_GFAPI_BASE + 13)
-#define API_MSG_XDR_PAYLOAD_FAILED              (GLFS_GFAPI_BASE + 14)
-#define API_MSG_GET_VOLINFO_CBK_FAILED          (GLFS_GFAPI_BASE + 15)
-#define API_MSG_FETCH_VOLUUID_FAILED            (GLFS_GFAPI_BASE + 16)
-#define API_MSG_INSUFF_SIZE                     (GLFS_GFAPI_BASE + 17)
-#define API_MSG_FRAME_CREAT_FAILED              (GLFS_GFAPI_BASE + 18)
-#define API_MSG_DICT_SET_FAILED                 (GLFS_GFAPI_BASE + 19)
-#define API_MSG_XDR_DECODE_FAILED               (GLFS_GFAPI_BASE + 20)
-#define API_MSG_GET_VOLFILE_FAILED              (GLFS_GFAPI_BASE + 21)
-#define API_MSG_WRONG_OPVERSION                 (GLFS_GFAPI_BASE + 22)
-#define API_MSG_DICT_SERIALIZE_FAILED           (GLFS_GFAPI_BASE + 23)
-#define API_MSG_REMOTE_HOST_CONN_FAILED         (GLFS_GFAPI_BASE + 24)
-#define API_MSG_VOLFILE_SERVER_EXHAUST          (GLFS_GFAPI_BASE + 25)
-#define API_MSG_CREATE_RPC_CLIENT_FAILED        (GLFS_GFAPI_BASE + 26)
-#define API_MSG_REG_NOTIFY_FUNC_FAILED          (GLFS_GFAPI_BASE + 27)
-#define API_MSG_REG_CBK_FUNC_FAILED             (GLFS_GFAPI_BASE + 28)
-#define API_MSG_GET_CWD_FAILED                  (GLFS_GFAPI_BASE + 29)
-#define API_MSG_FGETXATTR_FAILED                (GLFS_GFAPI_BASE + 30)
-#define API_MSG_LOCKINFO_KEY_MISSING            (GLFS_GFAPI_BASE + 31)
-#define API_MSG_FSETXATTR_FAILED                (GLFS_GFAPI_BASE + 32)
-#define API_MSG_FSYNC_FAILED                    (GLFS_GFAPI_BASE + 33)
-#define API_MSG_FDCREATE_FAILED                 (GLFS_GFAPI_BASE + 34)
-#define API_MSG_INODE_PATH_FAILED               (GLFS_GFAPI_BASE + 35)
-#define API_MSG_SYNCOP_OPEN_FAILED              (GLFS_GFAPI_BASE + 36)
-#define API_MSG_LOCK_MIGRATE_FAILED             (GLFS_GFAPI_BASE + 37)
-#define API_MSG_OPENFD_SKIPPED                  (GLFS_GFAPI_BASE + 38)
-#define API_MSG_FIRST_LOOKUP_GRAPH_FAILED       (GLFS_GFAPI_BASE + 39)
-#define API_MSG_CWD_GRAPH_REF_FAILED            (GLFS_GFAPI_BASE + 40)
-#define API_MSG_SWITCHED_GRAPH                  (GLFS_GFAPI_BASE + 41)
-#define API_MSG_XDR_RESPONSE_DECODE_FAILED      (GLFS_GFAPI_BASE + 42)
-#define API_MSG_VOLFILE_INFO                    (GLFS_GFAPI_BASE + 43)
-#define API_MSG_VOLFILE_CONNECTING              (GLFS_GFAPI_BASE + 44)
-#define API_MSG_NEW_GRAPH                       (GLFS_GFAPI_BASE + 45)
-#define API_MSG_ALLOC_FAILED                    (GLFS_GFAPI_BASE + 46)
-#define API_MSG_CREATE_HANDLE_FAILED            (GLFS_GFAPI_BASE + 47)
+GLFS_MSGID(API, API_MSG_MEM_ACCT_INIT_FAILED, API_MSG_MASTER_XLATOR_INIT_FAILED,
+           API_MSG_GFAPI_XLATOR_INIT_FAILED, API_MSG_VOLFILE_OPEN_FAILED,
+           API_MSG_VOL_SPEC_FILE_ERROR, API_MSG_GLFS_FSOBJ_NULL,
+           API_MSG_INVALID_ENTRY, API_MSG_FSMUTEX_LOCK_FAILED,
+           API_MSG_COND_WAIT_FAILED, API_MSG_FSMUTEX_UNLOCK_FAILED,
+           API_MSG_INODE_REFRESH_FAILED, API_MSG_GRAPH_CONSTRUCT_FAILED,
+           API_MSG_API_XLATOR_ERROR, API_MSG_XDR_PAYLOAD_FAILED,
+           API_MSG_GET_VOLINFO_CBK_FAILED, API_MSG_FETCH_VOLUUID_FAILED,
+           API_MSG_INSUFF_SIZE, API_MSG_FRAME_CREAT_FAILED,
+           API_MSG_DICT_SET_FAILED, API_MSG_XDR_DECODE_FAILED,
+           API_MSG_GET_VOLFILE_FAILED, API_MSG_WRONG_OPVERSION,
+           API_MSG_DICT_SERIALIZE_FAILED, API_MSG_REMOTE_HOST_CONN_FAILED,
+           API_MSG_VOLFILE_SERVER_EXHAUST, API_MSG_CREATE_RPC_CLIENT_FAILED,
+           API_MSG_REG_NOTIFY_FUNC_FAILED, API_MSG_REG_CBK_FUNC_FAILED,
+           API_MSG_GET_CWD_FAILED, API_MSG_FGETXATTR_FAILED,
+           API_MSG_LOCKINFO_KEY_MISSING, API_MSG_FSETXATTR_FAILED,
+           API_MSG_FSYNC_FAILED, API_MSG_FDCREATE_FAILED,
+           API_MSG_INODE_PATH_FAILED, API_MSG_SYNCOP_OPEN_FAILED,
+           API_MSG_LOCK_MIGRATE_FAILED, API_MSG_OPENFD_SKIPPED,
+           API_MSG_FIRST_LOOKUP_GRAPH_FAILED, API_MSG_CWD_GRAPH_REF_FAILED,
+           API_MSG_SWITCHED_GRAPH, API_MSG_XDR_RESPONSE_DECODE_FAILED,
+           API_MSG_VOLFILE_INFO, API_MSG_VOLFILE_CONNECTING, API_MSG_NEW_GRAPH,
+           API_MSG_ALLOC_FAILED, API_MSG_CREATE_HANDLE_FAILED,
+           API_MSG_INODE_LINK_FAILED, API_MSG_STATEDUMP_FAILED,
+           API_MSG_XREADDIRP_R_FAILED, API_MSG_LOCK_INSERT_MERGE_FAILED,
+           API_MSG_SETTING_LOCK_TYPE_FAILED, API_MSG_INODE_FIND_FAILED,
+           API_MSG_FDCTX_SET_FAILED, API_MSG_UPCALL_SYNCOP_FAILED,
+           API_MSG_INVALID_ARG, API_MSG_UPCALL_EVENT_NULL_RECEIVED,
+           API_MSG_FLAGS_HANDLE, API_MSG_FDCREATE_FAILED_ON_GRAPH,
+           API_MSG_TRANS_RDMA_DEP, API_MSG_TRANS_NOT_SUPPORTED,
+           API_MSG_FS_NOT_INIT, API_MSG_INVALID_SYSRQ,
+           API_MSG_DECODE_XDR_FAILED, API_MSG_NULL, API_MSG_CALL_NOT_SUCCESSFUL,
+           API_MSG_CALL_NOT_VALID, API_MSG_UNABLE_TO_DEL,
+           API_MSG_REMOTE_HOST_DISCONN, API_MSG_HANDLE_NOT_SET);
 
-/*------------*/
-#define glfs_msg_end_x GLFS_MSGID_END, "Invalid: End of messages"
+#define API_MSG_ALLOC_FAILED_STR "Upcall allocation failed"
+#define API_MSG_LOCK_INSERT_MERGE_FAILED_STR                                   \
+    "Lock insertion and splitting/merging failed"
+#define API_MSG_SETTING_LOCK_TYPE_FAILED_STR "Setting lock type failed"
 
+#define API_MSG_INVALID_ARG_STR "Invalid"
+#define API_MSG_INVALID_ENTRY_STR "Upcall entry validation failed"
+#define API_MSG_INODE_FIND_FAILED_STR "Unable to find inode entry"
+#define API_MSG_CREATE_HANDLE_FAILED_STR "handle creation failed"
+#define API_MSG_UPCALL_EVENT_NULL_RECEIVED_STR                                 \
+    "Upcall_EVENT_NULL received. Skipping it"
+#define API_MSG_UPCALL_SYNCOP_FAILED_STR "Synctask for upcall failed"
+#define API_MSG_FDCREATE_FAILED_STR "Allocating anonymous fd failed"
+#define API_MSG_XREADDIRP_R_FAILED_STR "glfs_x_readdirp_r failed"
+#define API_MSG_FDCTX_SET_FAILED_STR "Setting fd ctx failed"
+#define API_MSG_FLAGS_HANDLE_STR "arg not set. Flags handled are"
+#define API_MSG_INODE_REFRESH_FAILED_STR "inode refresh failed"
+#define API_MSG_INODE_LINK_FAILED_STR "inode linking failed"
+#define API_MSG_GET_CWD_FAILED_STR "Failed to get cwd"
+#define API_MSG_FGETXATTR_FAILED_STR "fgetxattr failed"
+#define API_MSG_LOCKINFO_KEY_MISSING_STR "missing lockinfo key"
+#define API_MSG_FSYNC_FAILED_STR "fsync() failed"
+#define API_MSG_FDCREATE_FAILED_ON_GRAPH_STR "fd_create failed on graph"
+#define API_MSG_INODE_PATH_FAILED_STR "inode_path failed"
+#define API_MSG_SYNCOP_OPEN_FAILED_STR "syncop_open failed"
+#define API_MSG_LOCK_MIGRATE_FAILED_STR "lock migration failed on graph"
+#define API_MSG_OPENFD_SKIPPED_STR "skipping openfd in graph"
+#define API_MSG_FIRST_LOOKUP_GRAPH_FAILED_STR "first lookup on graph failed"
+#define API_MSG_CWD_GRAPH_REF_FAILED_STR "cwd refresh of graph failed"
+#define API_MSG_SWITCHED_GRAPH_STR "switched to graph"
+#define API_MSG_FSETXATTR_FAILED_STR "fsetxattr failed"
+#define API_MSG_MEM_ACCT_INIT_FAILED_STR "Memory accounting init failed"
+#define API_MSG_MASTER_XLATOR_INIT_FAILED_STR                                  \
+    "master xlator for initialization failed"
+#define API_MSG_GFAPI_XLATOR_INIT_FAILED_STR                                   \
+    "failed to initialize gfapi translator"
+#define API_MSG_VOLFILE_OPEN_FAILED_STR "volume file open failed"
+#define API_MSG_VOL_SPEC_FILE_ERROR_STR "Cannot reach volume specification file"
+#define API_MSG_TRANS_RDMA_DEP_STR                                             \
+    "transport RDMA is deprecated, falling back to tcp"
+#define API_MSG_TRANS_NOT_SUPPORTED_STR                                        \
+    "transport is not supported, possible values tcp|unix"
+#define API_MSG_GLFS_FSOBJ_NULL_STR "fs is NULL"
+#define API_MSG_FS_NOT_INIT_STR "fs is not properly initialized"
+#define API_MSG_FSMUTEX_LOCK_FAILED_STR                                        \
+    "pthread lock on glfs mutex, returned error"
+#define API_MSG_FSMUTEX_UNLOCK_FAILED_STR                                      \
+    "pthread unlock on glfs mutex, returned error"
+#define API_MSG_COND_WAIT_FAILED_STR "cond wait failed"
+#define API_MSG_INVALID_SYSRQ_STR "not a valid sysrq"
+#define API_MSG_GRAPH_CONSTRUCT_FAILED_STR "failed to construct the graph"
+#define API_MSG_API_XLATOR_ERROR_STR                                           \
+    "api master xlator cannot be specified in volume file"
+#define API_MSG_STATEDUMP_FAILED_STR "statedump failed"
+#define API_MSG_DECODE_XDR_FAILED_STR                                          \
+    "Failed to decode xdr response for GF_CBK_STATEDUMP"
+#define API_MSG_NULL_STR "NULL"
+#define API_MSG_XDR_PAYLOAD_FAILED_STR "failed to create XDR payload"
+#define API_MSG_CALL_NOT_SUCCESSFUL_STR                                        \
+    "GET_VOLUME_INFO RPC call is not successful"
+#define API_MSG_XDR_RESPONSE_DECODE_FAILED_STR                                 \
+    "Failed to decode xdr response for GET_VOLUME_INFO"
+#define API_MSG_CALL_NOT_VALID_STR                                             \
+    "Response received for GET_VOLUME_INFO RPC is not valid"
+#define API_MSG_GET_VOLINFO_CBK_FAILED_STR                                     \
+    "In GET_VOLUME_INFO cbk, received error"
+#define API_MSG_FETCH_VOLUUID_FAILED_STR "Unable to fetch volume UUID"
+#define API_MSG_INSUFF_SIZE_STR "Insufficient size passed"
+#define API_MSG_FRAME_CREAT_FAILED_STR "failed to create the frame"
+#define API_MSG_DICT_SET_FAILED_STR "failed to set"
+#define API_MSG_XDR_DECODE_FAILED_STR "XDR decoding error"
+#define API_MSG_GET_VOLFILE_FAILED_STR "failed to get the volume file"
+#define API_MSG_VOLFILE_INFO_STR "No change in volfile, continuing"
+#define API_MSG_UNABLE_TO_DEL_STR "unable to delete file"
+#define API_MSG_WRONG_OPVERSION_STR                                            \
+    "Server is operating at an op-version which is not supported"
+#define API_MSG_DICT_SERIALIZE_FAILED_STR "Failed to serialize dictionary"
+#define API_MSG_REMOTE_HOST_CONN_FAILED_STR "Failed to connect to remote-host"
+#define API_MSG_REMOTE_HOST_DISCONN_STR "disconnected from remote-host"
+#define API_MSG_VOLFILE_SERVER_EXHAUST_STR "Exhausted all volfile servers"
+#define API_MSG_VOLFILE_CONNECTING_STR "connecting to next volfile server"
+#define API_MSG_CREATE_RPC_CLIENT_FAILED_STR "failed to create rpc clnt"
+#define API_MSG_REG_NOTIFY_FUNC_FAILED_STR "failed to register notify function"
+#define API_MSG_REG_CBK_FUNC_FAILED_STR "failed to register callback function"
+#define API_MSG_NEW_GRAPH_STR "New graph coming up"
+#define API_MSG_HANDLE_NOT_SET_STR "handle not set. Flags handled for xstat are"
 #endif /* !_GFAPI_MESSAGES_H__ */
diff --git a/api/src/gfapi.aliases b/api/src/gfapi.aliases
index 40b6ed21192..bc639e6b99f 100644
--- a/api/src/gfapi.aliases
+++ b/api/src/gfapi.aliases
@@ -18,31 +18,24 @@ _pub_glfs_from_glfd _glfs_from_glfd$GFAPI_3.4.0
 _pub_glfs_set_xlator_option _glfs_set_xlator_option$GFAPI_3.4.0
 _pub_glfs_read _glfs_read$GFAPI_3.4.0
 _pub_glfs_write _glfs_write$GFAPI_3.4.0
-_pub_glfs_read_async _glfs_read_async$GFAPI_3.4.0
-_pub_glfs_write_async _glfs_write_async$GFAPI_3.4.0
 _pub_glfs_readv _glfs_readv$GFAPI_3.4.0
 _pub_glfs_writev _glfs_writev$GFAPI_3.4.0
-_pub_glfs_readv_async _glfs_readv_async$GFAPI_3.4.0
-_pub_glfs_writev_async _glfs_writev_async$GFAPI_3.4.0
-_pub_glfs_pread _glfs_pread$GFAPI_3.4.0
-_pub_glfs_pwrite _glfs_pwrite$GFAPI_3.4.0
-_pub_glfs_pread_async _glfs_pread_async$GFAPI_3.4.0
-_pub_glfs_pwrite_async _glfs_pwrite_async$GFAPI_3.4.0
+_pub_glfs_pread34 _glfs_pread$GFAPI_3.4.0
+_pub_glfs_pwrite34 _glfs_pwrite$GFAPI_3.4.0
+_pub_glfs_pread_async34 _glfs_pread_async$GFAPI_3.4.0
+_pub_glfs_pwrite_async34 _glfs_pwrite_async$GFAPI_3.4.0
 _pub_glfs_preadv _glfs_preadv$GFAPI_3.4.0
 _pub_glfs_pwritev _glfs_pwritev$GFAPI_3.4.0
-_pub_glfs_preadv_async _glfs_preadv_async$GFAPI_3.4.0
-_pub_glfs_pwritev_async _glfs_pwritev_async$GFAPI_3.4.0
 _pub_glfs_lseek _glfs_lseek$GFAPI_3.4.0
-_pub_glfs_truncate _glfs_truncate$GFAPI_3.4.0
-_pub_glfs_ftruncate _glfs_ftruncate$GFAPI_3.4.0
-_pub_glfs_ftruncate_async _glfs_ftruncate_async$GFAPI_3.4.0
+_pub_glfs_ftruncate34 _glfs_ftruncate$GFAPI_3.4.0
+_pub_glfs_ftruncate_async34 _glfs_ftruncate_async$GFAPI_3.4.0
 _pub_glfs_lstat _glfs_lstat$GFAPI_3.4.0
 _pub_glfs_stat _glfs_stat$GFAPI_3.4.0
 _pub_glfs_fstat _glfs_fstat$GFAPI_3.4.0
-_pub_glfs_fsync _glfs_fsync$GFAPI_3.4.0
-_pub_glfs_fsync_async _glfs_fsync_async$GFAPI_3.4.0
-_pub_glfs_fdatasync _glfs_fdatasync$GFAPI_3.4.0
-_pub_glfs_fdatasync_async _glfs_fdatasync_async$GFAPI_3.4.0
+_pub_glfs_fsync34 _glfs_fsync$GFAPI_3.4.0
+_pub_glfs_fsync_async34 _glfs_fsync_async$GFAPI_3.4.0
+_pub_glfs_fdatasync34 _glfs_fdatasync$GFAPI_3.4.0
+_pub_glfs_fdatasync_async34 _glfs_fdatasync_async$GFAPI_3.4.0
 _pub_glfs_access _glfs_access$GFAPI_3.4.0
 _pub_glfs_symlink _glfs_symlink$GFAPI_3.4.0
 _pub_glfs_readlink _glfs_readlink$GFAPI_3.4.0
@@ -82,7 +75,7 @@ _pub_glfs_fremovexattr _glfs_fremovexattr$GFAPI_3.4.0
 _pub_glfs_getcwd _glfs_getcwd$GFAPI_3.4.0
 _pub_glfs_chdir _glfs_chdir$GFAPI_3.4.0
 _pub_glfs_fchdir _glfs_fchdir$GFAPI_3.4.0
-_pub_glfs_realpath _glfs_realpath$GFAPI_3.4.0
+_pub_glfs_realpath34 _glfs_realpath$GFAPI_3.4.0
 _pub_glfs_posix_lock _glfs_posix_lock$GFAPI_3.4.0
 _pub_glfs_dup _glfs_dup$GFAPI_3.4.0
 
@@ -113,9 +106,8 @@ _pub_glfs_readdir _glfs_readdir$GFAPI_3.5.0
 _pub_glfs_readdirplus _glfs_readdirplus$GFAPI_3.5.0
 _pub_glfs_fallocate _glfs_fallocate$GFAPI_3.5.0
 _pub_glfs_discard _glfs_discard$GFAPI_3.5.0
-_pub_glfs_discard_async _glfs_discard_async$GFAPI_3.5.0
 _pub_glfs_zerofill _glfs_zerofill$GFAPI_3.5.0
-_pub_glfs_zerofill_async _glfs_zerofill_async$GFAPI_3.5.0
+_pub_glfs_caller_specific_init _glfs_caller_specific_init$GFAPI_3.5.0
 _pub_glfs_h_setxattrs _glfs_h_setxattrs$GFAPI_3.5.0
 
 _pub_glfs_unset_volfile_server _glfs_unset_volfile_server$GFAPI_3.5.1
@@ -125,12 +117,10 @@ _pub_glfs_h_removexattrs _glfs_h_removexattrs$GFAPI_3.5.1
 _pub_glfs_get_volfile _glfs_get_volfile$GFAPI_3.6.0
 _pub_glfs_h_access _glfs_h_access$GFAPI_3.6.0
 
-_pub_glfs_ipc _glfs_ipc$GFAPI_3.7.0
-_pub_glfs_h_poll_upcall _glfs_h_poll_upcall$GFAPI_3.7.0
+_pub_glfs_h_poll_upcall370 _glfs_h_poll_upcall$GFAPI_3.7.0
 _pub_glfs_h_acl_set _glfs_h_acl_set$GFAPI_3.7.0
 _pub_glfs_h_acl_get _glfs_h_acl_get$GFAPI_3.7.0
 _pub_glfs_h_statfs _glfs_h_statfs$GFAPI_3.7.0
-
 _pub_glfs_h_anonymous_read _glfs_h_anonymous_read$GFAPI_3.7.0
 _pub_glfs_h_anonymous_write _glfs_h_anonymous_write$GFAPI_3.7.0
 
@@ -140,3 +130,72 @@ _priv_glfs_resolve _glfs_resolve$GFAPI_PRIVATE_3.7.0
 _priv_glfs_process_upcall_event _glfs_process_upcall_event$GFAPI_PRIVATE_3.7.0
 
 _pub_glfs_h_lookupat _glfs_h_lookupat$GFAPI_3.7.4
+
+_pub_glfs_truncate _glfs_truncate$GFAPI_3.7.15
+
+_pub_glfs_free _glfs_free$GFAPI_3.7.16
+_pub_glfs_h_poll_upcall _glfs_h_poll_upcall$GFAPI_3.7.16
+_pub_glfs_upcall_get_fs _glfs_upcall_get_fs$GFAPI_3.7.16
+_pub_glfs_upcall_get_reason _glfs_upcall_get_reason$GFAPI_3.7.16
+_pub_glfs_upcall_get_event _glfs_upcall_get_event$GFAPI_3.7.16
+_pub_glfs_upcall_inode_get_object _glfs_upcall_inode_get_object$GFAPI_3.7.16
+_pub_glfs_upcall_inode_get_flags _glfs_upcall_inode_get_flags$GFAPI_3.7.16
+_pub_glfs_upcall_inode_get_stat _glfs_upcall_inode_get_stat$GFAPI_3.7.16
+_pub_glfs_upcall_inode_get_expire _glfs_upcall_inode_get_expire$GFAPI_3.7.16
+_pub_glfs_upcall_inode_get_pobject _glfs_upcall_inode_get_pobject$GFAPI_3.7.16
+_pub_glfs_upcall_inode_get_pstat _glfs_upcall_inode_get_pstat$GFAPI_3.7.16
+_pub_glfs_upcall_inode_get_oldpobject _glfs_upcall_inode_get_oldpobject$GFAPI_3.7.16
+_pub_glfs_upcall_inode_get_oldpstat _glfs_upcall_inode_get_oldpstat$GFAPI_3.7.16
+
+_pub_glfs_realpath _glfs_realpath$GFAPI_3.7.17
+
+_pub_glfs_sysrq _glfs_sysrq$GFAPI_3.10.0
+
+_pub_glfs_fd_set_lkowner _glfs_fd_set_lkowner$GFAPI_3.10.7
+
+_pub_glfs_xreaddirplus_r _glfs_xreaddirplus_r$GFAPI_3.11.0
+_pub_glfs_xreaddirplus_r_get_stat _glfs_xreaddirplus_r_get_stat$GFAPI_3.11.0
+_pub_glfs_xreaddirplus_r_get_object _glfs_xreaddirplus_r_get_object$GFAPI_3.11.0
+_pub_glfs_object_copy _glfs_object_copy$GFAPI_3.11.0
+
+_priv_glfs_ipc _glfs_ipc$GFAPI_3.12.0
+
+_pub_glfs_upcall_register _glfs_upcall_register$GFAPI_3.13.0
+_pub_glfs_upcall_unregister _glfs_upcall_unregister$GFAPI_3.13.0
+
+_pub_glfs_setfsleaseid _glfs_setfsleaseid$GFAPI_4.0.0
+_pub_glfs_file_lock _glfs_file_lock$GFAPI_4.0.0
+_pub_glfs_lease _glfs_lease$GFAPI_4.0.0
+_pub_glfs_h_lease _glfs_h_lease$GFAPI_4.0.0
+_pub_glfs_upcall_lease_get_object _glfs_upcall_lease_get_object$GFAPI_4.1.6
+_pub_glfs_upcall_lease_get_lease_type _glfs_upcall_lease_get_lease_type$GFAPI_4.1.6
+
+_priv_glfs_statx _glfs_statx$GFAPI_6.0
+_priv_glfs_iatt_from_statx _glfs_iatt_from_statx$GFAPI_6.0
+_priv_glfs_setfspid _glfs_setfspid$GFAPI_6.1
+
+_pub_glfs_read_async _glfs_read_async$GFAPI_6.0
+_pub_glfs_write_async _glfs_write_async$GFAPI_6.0
+_pub_glfs_readv_async _glfs_readv_async$GFAPI_6.0
+_pub_glfs_writev_async _glfs_writev_async$GFAPI_6.0
+_pub_glfs_pread _glfs_pread$GFAPI_6.0
+_pub_glfs_pwrite _glfs_pwrite$GFAPI_6.0
+_pub_glfs_pread_async _glfs_pread_async$GFAPI_6.0
+_pub_glfs_pwrite_async _glfs_pwrite_async$GFAPI_6.0
+_pub_glfs_preadv_async _glfs_preadv_async$GFAPI_6.0
+_pub_glfs_pwritev_async _glfs_pwritev_async$GFAPI_6.0
+_pub_glfs_fsync _glfs_fsync$GFAPI_6.0
+_pub_glfs_fsync_async _glfs_fsync_async$GFAPI_6.0
+_pub_glfs_fdatasync _glfs_fdatasync$GFAPI_6.0
+_pub_glfs_fdatasync_async _glfs_fdatasync_async$GFAPI_6.0
+_pub_glfs_ftruncate _glfs_ftruncate$GFAPI_6.0
+_pub_glfs_ftruncate_async _glfs_ftruncate_async$GFAPI_6.0
+_pub_glfs_discard_async _glfs_discard_async$GFAPI_6.0
+_pub_glfs_zerofill_async _glfs_zerofill_async$GFAPI_6.0
+_pub_glfs_copy_file_range _glfs_copy_file_range$GFAPI_6.0
+_pub_glfs_fsetattr _glfs_fsetattr$GFAPI_6.0
+_pub_glfs_setattr _glfs_setattr$GFAPI_6.0
+
+_pub_glfs_set_statedump_path _glfs_set_statedump_path@GFAPI_7.0
+
+_pub_glfs_h_creat_open _glfs_h_creat_open@GFAPI_6.6
diff --git a/api/src/gfapi.map b/api/src/gfapi.map
index d42ae2b97af..228ac47c084 100644
--- a/api/src/gfapi.map
+++ b/api/src/gfapi.map
@@ -39,7 +39,6 @@ GFAPI_3.4.0 {
 		glfs_preadv_async;
 		glfs_pwritev_async;
 		glfs_lseek;
-		glfs_truncate;
 		glfs_ftruncate;
 		glfs_ftruncate_async;
 		glfs_lstat;
@@ -115,6 +114,7 @@ GFAPI_3.4.2 {
 		glfs_h_create_from_handle;
 		glfs_h_opendir;
 		glfs_h_open;
+		glfs_h_lookupat;
 } GFAPI_3.4.0;
 
 GFAPI_3.5.0 {
@@ -146,7 +146,6 @@ GFAPI_3.6.0 {
 
 GFAPI_3.7.0 {
 	global:
-		glfs_ipc;
 		glfs_h_poll_upcall;
 		glfs_h_acl_set;
 		glfs_h_acl_get;
@@ -167,3 +166,118 @@ GFAPI_3.7.4 {
 	global:
 		glfs_h_lookupat;
 } GFAPI_PRIVATE_3.7.0;
+
+GFAPI_3.7.15 {
+	global:
+		glfs_truncate;
+} GFAPI_3.7.4;
+
+GFAPI_3.7.16 {
+	global:
+		glfs_free;
+		glfs_upcall_get_fs;
+		glfs_upcall_get_reason;
+		glfs_upcall_get_event;
+		glfs_upcall_inode_get_object;
+		glfs_upcall_inode_get_flags;
+		glfs_upcall_inode_get_stat;
+		glfs_upcall_inode_get_expire;
+		glfs_upcall_inode_get_pobject;
+		glfs_upcall_inode_get_pstat;
+		glfs_upcall_inode_get_oldpobject;
+		glfs_upcall_inode_get_oldpstat;
+		glfs_h_poll_upcall;
+} GFAPI_3.7.15;
+
+GFAPI_3.7.17 {
+	global:
+		glfs_realpath;
+} GFAPI_3.7.16;
+
+GFAPI_3.10.0 {
+	global:
+		glfs_sysrq;
+} GFAPI_3.7.17;
+
+GFAPI_3.10.7 {
+	global:
+		glfs_fd_set_lkowner;
+} GFAPI_3.10.0;
+
+GFAPI_3.11.0 {
+		glfs_xreaddirplus_r;
+		glfs_xreaddirplus_r_get_stat;
+		glfs_xreaddirplus_r_get_object;
+		glfs_object_copy;
+} GFAPI_3.10.7;
+
+GFAPI_PRIVATE_3.12.0 {
+	global:
+		glfs_ipc;
+} GFAPI_3.11.0;
+
+GFAPI_3.13.0 {
+	global:
+		glfs_upcall_register;
+		glfs_upcall_unregister;
+} GFAPI_PRIVATE_3.12.0;
+
+GFAPI_4.0.0 {
+	global:
+		glfs_setfsleaseid;
+		glfs_file_lock;
+		glfs_lease;
+		glfs_h_lease;
+} GFAPI_3.13.0;
+
+GFAPI_4.1.6 {
+	global:
+		glfs_upcall_lease_get_object;
+		glfs_upcall_lease_get_lease_type;
+} GFAPI_4.0.0;
+
+GFAPI_PRIVATE_6.0 {
+	global:
+		glfs_statx;
+		glfs_iatt_from_statx;
+} GFAPI_4.1.6;
+
+GFAPI_6.0 {
+	global:
+		glfs_read_async;
+		glfs_write_async;
+		glfs_readv_async;
+		glfs_writev_async;
+		glfs_pread;
+		glfs_pwrite;
+		glfs_pread_async;
+		glfs_pwrite_async;
+		glfs_preadv_async;
+		glfs_pwritev_async;
+		glfs_fsync;
+		glfs_fsync_async;
+		glfs_fdatasync;
+		glfs_fdatasync_async;
+		glfs_ftruncate;
+		glfs_ftruncate_async;
+		glfs_discard_async;
+		glfs_zerofill_async;
+		glfs_copy_file_range;
+		glfs_setattr;
+		glfs_fsetattr;
+} GFAPI_PRIVATE_6.0;
+
+GFAPI_PRIVATE_6.1 {
+	global:
+		glfs_setfspid;
+} GFAPI_6.0;
+
+GFAPI_6.6 {
+	global:
+		glfs_h_creat_open;
+} GFAPI_PRIVATE_6.1;
+
+GFAPI_7.0 {
+	global:
+		glfs_set_statedump_path;
+} GFAPI_6.6;
diff --git a/api/src/glfs-fops.c b/api/src/glfs-fops.c
index e57ad3f4a15..6aa3c5602d1 100644
--- a/api/src/glfs-fops.c
+++ b/api/src/glfs-fops.c
@@ -1,6 +1,6 @@
 
 /*
-  Copyright (c) 2012 Red Hat, Inc. <http://www.redhat.com>
+  Copyright (c) 2012-2018 Red Hat, Inc. <http://www.redhat.com>
   This file is part of GlusterFS.
 
   This file is licensed to you under your choice of the GNU Lesser
@@ -9,15 +9,22 @@
   cases as published by the Free Software Foundation.
 */
 
+/* for SEEK_HOLE and SEEK_DATA */
+#ifndef _GNU_SOURCE
+#define _GNU_SOURCE
+#endif
+
+#include <unistd.h>
 
 #include "glfs-internal.h"
 #include "glfs-mem-types.h"
-#include "syncop.h"
+#include <glusterfs/syncop.h>
 #include "glfs.h"
 #include "gfapi-messages.h"
-#include "compat-errno.h"
+#include <glusterfs/compat-errno.h>
 #include <limits.h>
 #include "glusterfs3.h"
+#include <glusterfs/iatt.h>
 
 #ifdef NAME_MAX
 #define GF_NAME_MAX NAME_MAX
@@ -25,8 +32,69 @@
 #define GF_NAME_MAX 255
 #endif
 
+struct upcall_syncop_args {
+    struct glfs *fs;
+    struct gf_upcall upcall_data;
+};
+
 #define READDIRBUF_SIZE (sizeof(struct dirent) + GF_NAME_MAX + 1)
 
+typedef void (*glfs_io_cbk34)(glfs_fd_t *fd, ssize_t ret, void *data);
+
+/*
+ * This function will mark glfd for deletion and decrement its refcount.
+ */
+int
+glfs_mark_glfd_for_deletion(struct glfs_fd *glfd)
+{
+    LOCK(&glfd->lock);
+    {
+        glfd->state = GLFD_CLOSE;
+    }
+    UNLOCK(&glfd->lock);
+
+    GF_REF_PUT(glfd);
+
+    return 0;
+}
+
+/* This function is useful for all async fops. There is chance that glfd is
+ * closed before async fop is completed. When glfd is closed we change the
+ * state to GLFD_CLOSE.
+ *
+ * This function will return _gf_true if the glfd is still valid else return
+ * _gf_false.
+ */
+gf_boolean_t
+glfs_is_glfd_still_valid(struct glfs_fd *glfd)
+{
+    gf_boolean_t ret = _gf_false;
+
+    LOCK(&glfd->lock);
+    {
+        if (glfd->state != GLFD_CLOSE)
+            ret = _gf_true;
+    }
+    UNLOCK(&glfd->lock);
+
+    return ret;
+}
+
+void
+glfd_set_state_bind(struct glfs_fd *glfd)
+{
+    LOCK(&glfd->lock);
+    {
+        glfd->state = GLFD_OPEN;
+    }
+    UNLOCK(&glfd->lock);
+
+    fd_bind(glfd->fd);
+    glfs_fd_bind(glfd);
+
+    return;
+}
+
 /*
  * This routine is called when an upcall event of type
  * 'GF_UPCALL_CACHE_INVALIDATION' is received.
@@ -35,4064 +103,6343 @@
  * maintained by gfapi.
  */
 int
-glfs_get_upcall_cache_invalidation (struct gf_upcall *to_up_data,
-                                    struct gf_upcall *from_up_data)
+glfs_get_upcall_cache_invalidation(struct gf_upcall *to_up_data,
+                                   struct gf_upcall *from_up_data)
 {
+    struct gf_upcall_cache_invalidation *ca_data = NULL;
+    struct gf_upcall_cache_invalidation *f_ca_data = NULL;
+    int ret = -1;
 
-        struct gf_upcall_cache_invalidation *ca_data = NULL;
-        struct gf_upcall_cache_invalidation *f_ca_data = NULL;
-        int                                 ret      = -1;
+    GF_VALIDATE_OR_GOTO(THIS->name, to_up_data, out);
+    GF_VALIDATE_OR_GOTO(THIS->name, from_up_data, out);
 
-        GF_VALIDATE_OR_GOTO (THIS->name, to_up_data, out);
-        GF_VALIDATE_OR_GOTO (THIS->name, from_up_data, out);
+    f_ca_data = from_up_data->data;
+    GF_VALIDATE_OR_GOTO(THIS->name, f_ca_data, out);
 
-        f_ca_data = from_up_data->data;
-        GF_VALIDATE_OR_GOTO (THIS->name, f_ca_data, out);
+    ca_data = GF_CALLOC(1, sizeof(*ca_data), glfs_mt_upcall_entry_t);
 
-        ca_data = GF_CALLOC (1, sizeof(*ca_data),
-                            glfs_mt_upcall_entry_t);
+    if (!ca_data) {
+        gf_smsg(THIS->name, GF_LOG_ERROR, errno, API_MSG_ALLOC_FAILED, "entry",
+                NULL);
+        goto out;
+    }
 
-        if (!ca_data) {
-                gf_msg (THIS->name, GF_LOG_ERROR, errno,
-                        API_MSG_ALLOC_FAILED,
-                        "Upcall entry allocation failed.");
-                goto out;
-        }
+    to_up_data->data = ca_data;
 
-        to_up_data->data = ca_data;
+    ca_data->flags = f_ca_data->flags;
+    ca_data->expire_time_attr = f_ca_data->expire_time_attr;
+    ca_data->stat = f_ca_data->stat;
+    ca_data->p_stat = f_ca_data->p_stat;
+    ca_data->oldp_stat = f_ca_data->oldp_stat;
 
-        ca_data->flags      = f_ca_data->flags;
-        ca_data->expire_time_attr = f_ca_data->expire_time_attr;
-        ca_data->stat       = f_ca_data->stat;
-        ca_data->p_stat     = f_ca_data->p_stat;
-        ca_data->oldp_stat  = f_ca_data->oldp_stat;
-
-        ret = 0;
+    ret = 0;
 out:
-        return ret;
+    return ret;
 }
 
 int
-glfs_loc_link (loc_t *loc, struct iatt *iatt)
+glfs_get_upcall_lease(struct gf_upcall *to_up_data,
+                      struct gf_upcall *from_up_data)
 {
-	int ret = -1;
-        inode_t *old_inode = NULL;
+    struct gf_upcall_recall_lease *ca_data = NULL;
+    struct gf_upcall_recall_lease *f_ca_data = NULL;
+    int ret = -1;
 
-	if (!loc->inode) {
-		errno = EINVAL;
-		return -1;
-	}
+    GF_VALIDATE_OR_GOTO(THIS->name, to_up_data, out);
+    GF_VALIDATE_OR_GOTO(THIS->name, from_up_data, out);
 
-        old_inode = loc->inode;
+    f_ca_data = from_up_data->data;
+    GF_VALIDATE_OR_GOTO(THIS->name, f_ca_data, out);
 
-        /* If the inode already exists in the cache, the inode
-         * returned here points to the existing one. We need
-         * to update loc.inode accordingly.
-         */
-	loc->inode = inode_link (loc->inode, loc->parent, loc->name, iatt);
-	if (loc->inode) {
-		inode_lookup (loc->inode);
-                inode_unref (old_inode);
-		ret = 0;
-	} else {
-		ret = -1;
-		errno = ENOMEM;
-	}
+    ca_data = GF_CALLOC(1, sizeof(*ca_data), glfs_mt_upcall_entry_t);
+
+    if (!ca_data) {
+        gf_smsg(THIS->name, GF_LOG_ERROR, errno, API_MSG_ALLOC_FAILED, "entry",
+                NULL);
+        goto out;
+    }
 
-	return ret;
+    to_up_data->data = ca_data;
+
+    ca_data->lease_type = f_ca_data->lease_type;
+    gf_uuid_copy(ca_data->tid, f_ca_data->tid);
+    ca_data->dict = f_ca_data->dict;
+
+    ret = 0;
+out:
+    return ret;
 }
+int
+glfs_loc_link(loc_t *loc, struct iatt *iatt)
+{
+    int ret = -1;
+    inode_t *old_inode = NULL;
+    uint64_t ctx_value = LOOKUP_NOT_NEEDED;
 
+    if (!loc->inode) {
+        errno = EINVAL;
+        return -1;
+    }
+
+    old_inode = loc->inode;
+
+    /* If the inode already exists in the cache, the inode
+     * returned here points to the existing one. We need
+     * to update loc.inode accordingly.
+     */
+    loc->inode = inode_link(loc->inode, loc->parent, loc->name, iatt);
+    if (loc->inode) {
+        inode_ctx_set(loc->inode, THIS, &ctx_value);
+        inode_lookup(loc->inode);
+        inode_unref(old_inode);
+        ret = 0;
+    } else {
+        ret = -1;
+    }
+
+    return ret;
+}
 
 void
-glfs_iatt_to_stat (struct glfs *fs, struct iatt *iatt, struct stat *stat)
+glfs_iatt_to_stat(struct glfs *fs, struct iatt *iatt, struct stat *stat)
 {
-	iatt_to_stat (iatt, stat);
-	stat->st_dev = fs->dev_id;
+    iatt_to_stat(iatt, stat);
+    stat->st_dev = fs->dev_id;
 }
 
-
-int
-glfs_loc_unlink (loc_t *loc)
+void
+glfs_iatt_to_statx(struct glfs *fs, const struct iatt *iatt,
+                   struct glfs_stat *statx)
 {
-	inode_unlink (loc->inode, loc->parent, loc->name);
+    statx->glfs_st_mask = 0;
+
+    statx->glfs_st_mode = 0;
+    if (IATT_TYPE_VALID(iatt->ia_flags)) {
+        statx->glfs_st_mode |= st_mode_type_from_ia(iatt->ia_type);
+        statx->glfs_st_mask |= GLFS_STAT_TYPE;
+    }
+
+    if (IATT_MODE_VALID(iatt->ia_flags)) {
+        statx->glfs_st_mode |= st_mode_prot_from_ia(iatt->ia_prot);
+        statx->glfs_st_mask |= GLFS_STAT_MODE;
+    }
+
+    if (IATT_NLINK_VALID(iatt->ia_flags)) {
+        statx->glfs_st_nlink = iatt->ia_nlink;
+        statx->glfs_st_mask |= GLFS_STAT_NLINK;
+    }
+
+    if (IATT_UID_VALID(iatt->ia_flags)) {
+        statx->glfs_st_uid = iatt->ia_uid;
+        statx->glfs_st_mask |= GLFS_STAT_UID;
+    }
+
+    if (IATT_GID_VALID(iatt->ia_flags)) {
+        statx->glfs_st_gid = iatt->ia_gid;
+        statx->glfs_st_mask |= GLFS_STAT_GID;
+    }
+
+    if (IATT_ATIME_VALID(iatt->ia_flags)) {
+        statx->glfs_st_atime.tv_sec = iatt->ia_atime;
+        statx->glfs_st_atime.tv_nsec = iatt->ia_atime_nsec;
+        statx->glfs_st_mask |= GLFS_STAT_ATIME;
+    }
+
+    if (IATT_MTIME_VALID(iatt->ia_flags)) {
+        statx->glfs_st_mtime.tv_sec = iatt->ia_mtime;
+        statx->glfs_st_mtime.tv_nsec = iatt->ia_mtime_nsec;
+        statx->glfs_st_mask |= GLFS_STAT_MTIME;
+    }
+
+    if (IATT_CTIME_VALID(iatt->ia_flags)) {
+        statx->glfs_st_ctime.tv_sec = iatt->ia_ctime;
+        statx->glfs_st_ctime.tv_nsec = iatt->ia_ctime_nsec;
+        statx->glfs_st_mask |= GLFS_STAT_CTIME;
+    }
+
+    if (IATT_BTIME_VALID(iatt->ia_flags)) {
+        statx->glfs_st_btime.tv_sec = iatt->ia_btime;
+        statx->glfs_st_btime.tv_nsec = iatt->ia_btime_nsec;
+        statx->glfs_st_mask |= GLFS_STAT_BTIME;
+    }
+
+    if (IATT_INO_VALID(iatt->ia_flags)) {
+        statx->glfs_st_ino = iatt->ia_ino;
+        statx->glfs_st_mask |= GLFS_STAT_INO;
+    }
+
+    if (IATT_SIZE_VALID(iatt->ia_flags)) {
+        statx->glfs_st_size = iatt->ia_size;
+        statx->glfs_st_mask |= GLFS_STAT_SIZE;
+    }
+
+    if (IATT_BLOCKS_VALID(iatt->ia_flags)) {
+        statx->glfs_st_blocks = iatt->ia_blocks;
+        statx->glfs_st_mask |= GLFS_STAT_BLOCKS;
+    }
+
+    /* unconditionally present, encode as is */
+    statx->glfs_st_blksize = iatt->ia_blksize;
+    statx->glfs_st_rdev_major = ia_major(iatt->ia_rdev);
+    statx->glfs_st_rdev_minor = ia_minor(iatt->ia_rdev);
+    statx->glfs_st_dev_major = ia_major(fs->dev_id);
+    statx->glfs_st_dev_minor = ia_minor(fs->dev_id);
+
+    /* At present we do not read any localFS attributes and pass them along,
+     * so setting this to 0. As we start supporting file attributes we can
+     * populate the same here as well */
+    statx->glfs_st_attributes = 0;
+    statx->glfs_st_attributes_mask = 0;
+}
 
-	return 0;
+GFAPI_SYMVER_PRIVATE_DEFAULT(glfs_iatt_from_statx, 6.0)
+void
+priv_glfs_iatt_from_statx(struct iatt *iatt, const struct glfs_stat *statx)
+{
+    /* Most code in xlators are not checking validity flags before accessing
+    the items. Hence zero everything before setting valid items */
+    memset(iatt, 0, sizeof(struct iatt));
+
+    if (GLFS_STAT_TYPE_VALID(statx->glfs_st_mask)) {
+        iatt->ia_type = ia_type_from_st_mode(statx->glfs_st_mode);
+        iatt->ia_flags |= IATT_TYPE;
+    }
+
+    if (GLFS_STAT_MODE_VALID(statx->glfs_st_mask)) {
+        iatt->ia_prot = ia_prot_from_st_mode(statx->glfs_st_mode);
+        iatt->ia_flags |= IATT_MODE;
+    }
+
+    if (GLFS_STAT_NLINK_VALID(statx->glfs_st_mask)) {
+        iatt->ia_nlink = statx->glfs_st_nlink;
+        iatt->ia_flags |= IATT_NLINK;
+    }
+
+    if (GLFS_STAT_UID_VALID(statx->glfs_st_mask)) {
+        iatt->ia_uid = statx->glfs_st_uid;
+        iatt->ia_flags |= IATT_UID;
+    }
+
+    if (GLFS_STAT_GID_VALID(statx->glfs_st_mask)) {
+        iatt->ia_gid = statx->glfs_st_gid;
+        iatt->ia_flags |= IATT_GID;
+    }
+
+    if (GLFS_STAT_ATIME_VALID(statx->glfs_st_mask)) {
+        iatt->ia_atime = statx->glfs_st_atime.tv_sec;
+        iatt->ia_atime_nsec = statx->glfs_st_atime.tv_nsec;
+        iatt->ia_flags |= IATT_ATIME;
+    }
+
+    if (GLFS_STAT_MTIME_VALID(statx->glfs_st_mask)) {
+        iatt->ia_mtime = statx->glfs_st_mtime.tv_sec;
+        iatt->ia_mtime_nsec = statx->glfs_st_mtime.tv_nsec;
+        iatt->ia_flags |= IATT_MTIME;
+    }
+
+    if (GLFS_STAT_CTIME_VALID(statx->glfs_st_mask)) {
+        iatt->ia_ctime = statx->glfs_st_ctime.tv_sec;
+        iatt->ia_ctime_nsec = statx->glfs_st_ctime.tv_nsec;
+        iatt->ia_flags |= IATT_CTIME;
+    }
+
+    if (GLFS_STAT_BTIME_VALID(statx->glfs_st_mask)) {
+        iatt->ia_btime = statx->glfs_st_btime.tv_sec;
+        iatt->ia_btime_nsec = statx->glfs_st_btime.tv_nsec;
+        iatt->ia_flags |= IATT_BTIME;
+    }
+
+    if (GLFS_STAT_INO_VALID(statx->glfs_st_mask)) {
+        iatt->ia_ino = statx->glfs_st_ino;
+        iatt->ia_flags |= IATT_INO;
+    }
+
+    if (GLFS_STAT_SIZE_VALID(statx->glfs_st_mask)) {
+        iatt->ia_size = statx->glfs_st_size;
+        iatt->ia_flags |= IATT_SIZE;
+    }
+
+    if (GLFS_STAT_BLOCKS_VALID(statx->glfs_st_mask)) {
+        iatt->ia_blocks = statx->glfs_st_blocks;
+        iatt->ia_flags |= IATT_BLOCKS;
+    }
+
+    /* unconditionally present, encode as is */
+    iatt->ia_blksize = statx->glfs_st_blksize;
+    iatt->ia_rdev = makedev(statx->glfs_st_rdev_major,
+                            statx->glfs_st_rdev_minor);
+    iatt->ia_dev = makedev(statx->glfs_st_dev_major, statx->glfs_st_dev_minor);
+    iatt->ia_attributes = statx->glfs_st_attributes;
+    iatt->ia_attributes_mask = statx->glfs_st_attributes_mask;
 }
 
+void
+glfsflags_from_gfapiflags(struct glfs_stat *stat, int *glvalid)
+{
+    *glvalid = 0;
+    if (stat->glfs_st_mask & GLFS_STAT_MODE) {
+        *glvalid |= GF_SET_ATTR_MODE;
+    }
+
+    if (stat->glfs_st_mask & GLFS_STAT_SIZE) {
+        *glvalid |= GF_SET_ATTR_SIZE;
+    }
+
+    if (stat->glfs_st_mask & GLFS_STAT_UID) {
+        *glvalid |= GF_SET_ATTR_UID;
+    }
+
+    if (stat->glfs_st_mask & GLFS_STAT_GID) {
+        *glvalid |= GF_SET_ATTR_GID;
+    }
+
+    if (stat->glfs_st_mask & GLFS_STAT_ATIME) {
+        *glvalid |= GF_SET_ATTR_ATIME;
+    }
+
+    if (stat->glfs_st_mask & GLFS_STAT_MTIME) {
+        *glvalid |= GF_SET_ATTR_MTIME;
+    }
+}
 
-struct glfs_fd *
-pub_glfs_open (struct glfs *fs, const char *path, int flags)
+int
+glfs_loc_unlink(loc_t *loc)
 {
-	int              ret = -1;
-	struct glfs_fd  *glfd = NULL;
-	xlator_t        *subvol = NULL;
-	loc_t            loc = {0, };
-	struct iatt      iatt = {0, };
-	int              reval = 0;
+    inode_unlink(loc->inode, loc->parent, loc->name);
 
-        DECLARE_OLD_THIS;
-        __GLFS_ENTRY_VALIDATE_FS (fs, invalid_fs);
+    /* since glfs_h_* objects hold a reference to inode
+     * it is safe to keep lookup count to '0' */
+    if (!inode_has_dentry(loc->inode))
+        inode_forget(loc->inode, 0);
 
-	subvol = glfs_active_subvol (fs);
-	if (!subvol) {
-		ret = -1;
-		errno = EIO;
-		goto out;
-	}
+    return 0;
+}
 
-	glfd = glfs_fd_new (fs);
-	if (!glfd)
-		goto out;
+GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_open, 3.4.0)
+struct glfs_fd *
+pub_glfs_open(struct glfs *fs, const char *path, int flags)
+{
+    int ret = -1;
+    struct glfs_fd *glfd = NULL;
+    xlator_t *subvol = NULL;
+    loc_t loc = {
+        0,
+    };
+    struct iatt iatt = {
+        0,
+    };
+    int reval = 0;
+    dict_t *fop_attr = NULL;
+
+    DECLARE_OLD_THIS;
+    __GLFS_ENTRY_VALIDATE_FS(fs, invalid_fs);
+
+    subvol = glfs_active_subvol(fs);
+    if (!subvol) {
+        ret = -1;
+        errno = EIO;
+        goto out;
+    }
+
+    glfd = glfs_fd_new(fs);
+    if (!glfd)
+        goto out;
 
 retry:
-	ret = glfs_resolve (fs, subvol, path, &loc, &iatt, reval);
-
-	ESTALE_RETRY (ret, errno, reval, &loc, retry);
-
-	if (ret)
-		goto out;
-
-	if (IA_ISDIR (iatt.ia_type)) {
-		ret = -1;
-		errno = EISDIR;
-		goto out;
-	}
-
-	if (!IA_ISREG (iatt.ia_type)) {
-		ret = -1;
-		errno = EINVAL;
-		goto out;
-	}
-
-	if (glfd->fd) {
-		/* Retry. Safe to touch glfd->fd as we
-		   still have not glfs_fd_bind() yet.
-		*/
-		fd_unref (glfd->fd);
-		glfd->fd = NULL;
-	}
-
-	glfd->fd = fd_create (loc.inode, getpid());
-	if (!glfd->fd) {
-		ret = -1;
-		errno = ENOMEM;
-		goto out;
-	}
-        glfd->fd->flags = flags;
-
-	ret = syncop_open (subvol, &loc, flags, glfd->fd, NULL, NULL);
-        DECODE_SYNCOP_ERR (ret);
-
-	ESTALE_RETRY (ret, errno, reval, &loc, retry);
-out:
-	loc_wipe (&loc);
+    ret = glfs_resolve(fs, subvol, path, &loc, &iatt, reval);
 
-	if (ret && glfd) {
-		glfs_fd_destroy (glfd);
-		glfd = NULL;
-	} else if (glfd) {
-		fd_bind (glfd->fd);
-		glfs_fd_bind (glfd);
-	}
+    ESTALE_RETRY(ret, errno, reval, &loc, retry);
 
-	glfs_subvol_done (fs, subvol);
+    if (ret)
+        goto out;
 
-        __GLFS_EXIT_FS;
+    if (IA_ISDIR(iatt.ia_type)) {
+        ret = -1;
+        errno = EISDIR;
+        goto out;
+    }
 
-invalid_fs:
-	return glfd;
-}
+    if (!IA_ISREG(iatt.ia_type)) {
+        ret = -1;
+        errno = EINVAL;
+        goto out;
+    }
 
-GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_open, 3.4.0);
+    if (glfd->fd) {
+        /* Retry. Safe to touch glfd->fd as we
+           still have not glfs_fd_bind() yet.
+        */
+        fd_unref(glfd->fd);
+        glfd->fd = NULL;
+    }
+
+    glfd->fd = fd_create(loc.inode, getpid());
+    if (!glfd->fd) {
+        ret = -1;
+        errno = ENOMEM;
+        goto out;
+    }
+    glfd->fd->flags = flags;
+
+    ret = get_fop_attr_thrd_key(&fop_attr);
+    if (ret)
+        gf_msg_debug("gfapi", 0, "Getting leaseid from thread failed");
+
+    ret = syncop_open(subvol, &loc, flags, glfd->fd, fop_attr, NULL);
+    DECODE_SYNCOP_ERR(ret);
+
+    ESTALE_RETRY(ret, errno, reval, &loc, retry);
+out:
+    loc_wipe(&loc);
 
+    if (fop_attr)
+        dict_unref(fop_attr);
 
-int
-pub_glfs_close (struct glfs_fd *glfd)
-{
-	xlator_t  *subvol = NULL;
-	int        ret = -1;
-	fd_t      *fd = NULL;
-	struct glfs *fs = NULL;
+    if (ret && glfd) {
+        GF_REF_PUT(glfd);
+        glfd = NULL;
+    } else if (glfd) {
+        glfd_set_state_bind(glfd);
+    }
 
-        DECLARE_OLD_THIS;
-	__GLFS_ENTRY_VALIDATE_FD (glfd, invalid_fs);
+    glfs_subvol_done(fs, subvol);
 
-	subvol = glfs_active_subvol (glfd->fs);
-        if (!subvol) {
-                ret = -1;
-                errno = EIO;
-                goto out;
-        }
+    __GLFS_EXIT_FS;
 
-	fd = glfs_resolve_fd (glfd->fs, subvol, glfd);
-	if (!fd) {
-		ret = -1;
-		errno = EBADFD;
-		goto out;
-	}
+invalid_fs:
+    return glfd;
+}
 
-	ret = syncop_flush (subvol, fd, NULL, NULL);
-        DECODE_SYNCOP_ERR (ret);
+GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_close, 3.4.0)
+int
+pub_glfs_close(struct glfs_fd *glfd)
+{
+    xlator_t *subvol = NULL;
+    int ret = -1;
+    fd_t *fd = NULL;
+    struct glfs *fs = NULL;
+    dict_t *fop_attr = NULL;
+
+    DECLARE_OLD_THIS;
+    __GLFS_ENTRY_VALIDATE_FD(glfd, invalid_fs);
+
+    subvol = glfs_active_subvol(glfd->fs);
+    if (!subvol) {
+        ret = -1;
+        errno = EIO;
+        goto out;
+    }
+
+    fd = glfs_resolve_fd(glfd->fs, subvol, glfd);
+    if (!fd) {
+        ret = -1;
+        errno = EBADFD;
+        goto out;
+    }
+
+    if (glfd->lk_owner.len != 0) {
+        ret = syncopctx_setfslkowner(&glfd->lk_owner);
+        if (ret)
+            goto out;
+    }
+    ret = get_fop_attr_thrd_key(&fop_attr);
+    if (ret)
+        gf_msg_debug("gfapi", 0, "Getting leaseid from thread failed");
+
+    ret = syncop_flush(subvol, fd, fop_attr, NULL);
+    DECODE_SYNCOP_ERR(ret);
 out:
-	fs = glfd->fs;
-	glfs_fd_destroy (glfd);
+    fs = glfd->fs;
 
-	if (fd)
-		fd_unref (fd);
+    if (fd)
+        fd_unref(fd);
+    if (fop_attr)
+        dict_unref(fop_attr);
 
-	glfs_subvol_done (fs, subvol);
+    glfs_mark_glfd_for_deletion(glfd);
+    glfs_subvol_done(fs, subvol);
 
-        __GLFS_EXIT_FS;
+    __GLFS_EXIT_FS;
 
 invalid_fs:
-	return ret;
+    return ret;
 }
 
-GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_close, 3.4.0);
-
-
+GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_lstat, 3.4.0)
 int
-pub_glfs_lstat (struct glfs *fs, const char *path, struct stat *stat)
-{
-	int              ret = -1;
-	xlator_t        *subvol = NULL;
-	loc_t            loc = {0, };
-	struct iatt      iatt = {0, };
-	int              reval = 0;
-
-        DECLARE_OLD_THIS;
-        __GLFS_ENTRY_VALIDATE_FS (fs, invalid_fs);
-
-	subvol = glfs_active_subvol (fs);
-	if (!subvol) {
-		ret = -1;
-		errno = EIO;
-		goto out;
-	}
+pub_glfs_lstat(struct glfs *fs, const char *path, struct stat *stat)
+{
+    int ret = -1;
+    xlator_t *subvol = NULL;
+    loc_t loc = {
+        0,
+    };
+    struct iatt iatt = {
+        0,
+    };
+    int reval = 0;
+
+    DECLARE_OLD_THIS;
+    __GLFS_ENTRY_VALIDATE_FS(fs, invalid_fs);
+
+    subvol = glfs_active_subvol(fs);
+    if (!subvol) {
+        ret = -1;
+        errno = EIO;
+        goto out;
+    }
 retry:
-	ret = glfs_lresolve (fs, subvol, path, &loc, &iatt, reval);
+    ret = glfs_lresolve(fs, subvol, path, &loc, &iatt, reval);
 
-	ESTALE_RETRY (ret, errno, reval, &loc, retry);
+    ESTALE_RETRY(ret, errno, reval, &loc, retry);
 
-	if (ret == 0 && stat)
-		glfs_iatt_to_stat (fs, &iatt, stat);
+    if (ret == 0 && stat)
+        glfs_iatt_to_stat(fs, &iatt, stat);
 out:
-	loc_wipe (&loc);
+    loc_wipe(&loc);
 
-	glfs_subvol_done (fs, subvol);
+    glfs_subvol_done(fs, subvol);
 
-        __GLFS_EXIT_FS;
+    __GLFS_EXIT_FS;
 
 invalid_fs:
-	return ret;
+    return ret;
 }
 
-GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_lstat, 3.4.0);
-
-
+GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_stat, 3.4.0)
 int
-pub_glfs_stat (struct glfs *fs, const char *path, struct stat *stat)
-{
-	int              ret = -1;
-	xlator_t        *subvol = NULL;
-	loc_t            loc = {0, };
-	struct iatt      iatt = {0, };
-	int              reval = 0;
-
-        DECLARE_OLD_THIS;
-        __GLFS_ENTRY_VALIDATE_FS (fs, invalid_fs);
-
-	subvol = glfs_active_subvol (fs);
-	if (!subvol) {
-		ret = -1;
-		errno = EIO;
-		goto out;
-	}
+pub_glfs_stat(struct glfs *fs, const char *path, struct stat *stat)
+{
+    int ret = -1;
+    xlator_t *subvol = NULL;
+    loc_t loc = {
+        0,
+    };
+    struct iatt iatt = {
+        0,
+    };
+    int reval = 0;
+
+    DECLARE_OLD_THIS;
+    __GLFS_ENTRY_VALIDATE_FS(fs, invalid_fs);
+
+    subvol = glfs_active_subvol(fs);
+    if (!subvol) {
+        ret = -1;
+        errno = EIO;
+        goto out;
+    }
 retry:
-	ret = glfs_resolve (fs, subvol, path, &loc, &iatt, reval);
+    ret = glfs_resolve(fs, subvol, path, &loc, &iatt, reval);
 
-	ESTALE_RETRY (ret, errno, reval, &loc, retry);
+    ESTALE_RETRY(ret, errno, reval, &loc, retry);
 
-	if (ret == 0 && stat)
-		glfs_iatt_to_stat (fs, &iatt, stat);
+    if (ret == 0 && stat)
+        glfs_iatt_to_stat(fs, &iatt, stat);
 out:
-	loc_wipe (&loc);
+    loc_wipe(&loc);
 
-	glfs_subvol_done (fs, subvol);
+    glfs_subvol_done(fs, subvol);
 
-        __GLFS_EXIT_FS;
+    __GLFS_EXIT_FS;
 
 invalid_fs:
-	return ret;
+    return ret;
 }
 
-GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_stat, 3.4.0);
+GFAPI_SYMVER_PRIVATE_DEFAULT(glfs_statx, 6.0)
+int
+priv_glfs_statx(struct glfs *fs, const char *path, const unsigned int mask,
+                struct glfs_stat *statxbuf)
+{
+    int ret = -1;
+    xlator_t *subvol = NULL;
+    loc_t loc = {
+        0,
+    };
+    struct iatt iatt = {
+        0,
+    };
+    int reval = 0;
+
+    DECLARE_OLD_THIS;
+    __GLFS_ENTRY_VALIDATE_FS(fs, invalid_fs);
+
+    if (path == NULL) {
+        ret = -1;
+        errno = EINVAL;
+        goto out;
+    }
+
+    if (mask & ~GLFS_STAT_ALL) {
+        ret = -1;
+        errno = EINVAL;
+        goto out;
+    }
+
+    subvol = glfs_active_subvol(fs);
+    if (!subvol) {
+        ret = -1;
+        errno = EIO;
+        goto out;
+    }
 
+retry:
+    ret = glfs_resolve(fs, subvol, path, &loc, &iatt, reval);
+    ESTALE_RETRY(ret, errno, reval, &loc, retry);
 
-int
-pub_glfs_fstat (struct glfs_fd *glfd, struct stat *stat)
-{
-	int              ret = -1;
-	xlator_t        *subvol = NULL;
-	struct iatt      iatt = {0, };
-	fd_t            *fd = NULL;
-
-        DECLARE_OLD_THIS;
-	__GLFS_ENTRY_VALIDATE_FD (glfd, invalid_fs);
-
-	subvol = glfs_active_subvol (glfd->fs);
-	if (!subvol) {
-		ret = -1;
-		errno = EIO;
-		goto out;
-	}
-
-	fd = glfs_resolve_fd (glfd->fs, subvol, glfd);
-	if (!fd) {
-		ret = -1;
-		errno = EBADFD;
-		goto out;
-	}
-
-	ret = syncop_fstat (subvol, fd, &iatt, NULL, NULL);
-        DECODE_SYNCOP_ERR (ret);
-
-	if (ret == 0 && stat)
-		glfs_iatt_to_stat (glfd->fs, &iatt, stat);
+    if (ret == 0 && statxbuf)
+        glfs_iatt_to_statx(fs, &iatt, statxbuf);
 out:
-	if (fd)
-		fd_unref (fd);
+    loc_wipe(&loc);
 
-	glfs_subvol_done (glfd->fs, subvol);
+    glfs_subvol_done(fs, subvol);
 
-        __GLFS_EXIT_FS;
+    __GLFS_EXIT_FS;
 
 invalid_fs:
-	return ret;
+    return ret;
 }
 
-GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_fstat, 3.4.0);
+GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_fstat, 3.4.0)
+int
+pub_glfs_fstat(struct glfs_fd *glfd, struct stat *stat)
+{
+    int ret = -1;
+    xlator_t *subvol = NULL;
+    struct iatt iatt = {
+        0,
+    };
+    fd_t *fd = NULL;
+
+    DECLARE_OLD_THIS;
+    __GLFS_ENTRY_VALIDATE_FD(glfd, invalid_fs);
+
+    GF_REF_GET(glfd);
+
+    subvol = glfs_active_subvol(glfd->fs);
+    if (!subvol) {
+        ret = -1;
+        errno = EIO;
+        goto out;
+    }
+
+    fd = glfs_resolve_fd(glfd->fs, subvol, glfd);
+    if (!fd) {
+        ret = -1;
+        errno = EBADFD;
+        goto out;
+    }
+
+    ret = syncop_fstat(subvol, fd, &iatt, NULL, NULL);
+    DECODE_SYNCOP_ERR(ret);
+
+    if (ret == 0 && stat)
+        glfs_iatt_to_stat(glfd->fs, &iatt, stat);
+out:
+    if (fd)
+        fd_unref(fd);
+    if (glfd)
+        GF_REF_PUT(glfd);
 
+    glfs_subvol_done(glfd->fs, subvol);
 
+    __GLFS_EXIT_FS;
+
+invalid_fs:
+    return ret;
+}
+
+GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_creat, 3.4.0)
 struct glfs_fd *
-pub_glfs_creat (struct glfs *fs, const char *path, int flags, mode_t mode)
-{
-	int              ret = -1;
-	struct glfs_fd  *glfd = NULL;
-	xlator_t        *subvol = NULL;
-	loc_t            loc = {0, };
-	struct iatt      iatt = {0, };
-	uuid_t           gfid;
-	dict_t          *xattr_req = NULL;
-	int              reval = 0;
-
-        DECLARE_OLD_THIS;
-        __GLFS_ENTRY_VALIDATE_FS (fs, invalid_fs);
-
-	subvol = glfs_active_subvol (fs);
-	if (!subvol) {
-		ret = -1;
-		errno = EIO;
-		goto out;
-	}
-
-	xattr_req = dict_new ();
-	if (!xattr_req) {
-		ret = -1;
-		errno = ENOMEM;
-		goto out;
-	}
-
-	gf_uuid_generate (gfid);
-	ret = dict_set_static_bin (xattr_req, "gfid-req", gfid, 16);
-	if (ret) {
-		ret = -1;
-		errno = ENOMEM;
-		goto out;
-	}
-
-	glfd = glfs_fd_new (fs);
-	if (!glfd)
-		goto out;
-
-	/* This must be glfs_resolve() and NOT glfs_lresolve().
-	   That is because open("name", O_CREAT) where "name"
-	   is a danging symlink must create the dangling
-	   destinataion.
-	*/
+pub_glfs_creat(struct glfs *fs, const char *path, int flags, mode_t mode)
+{
+    int ret = -1;
+    struct glfs_fd *glfd = NULL;
+    xlator_t *subvol = NULL;
+    loc_t loc = {
+        0,
+    };
+    struct iatt iatt = {
+        0,
+    };
+    uuid_t gfid;
+    dict_t *xattr_req = NULL;
+    int reval = 0;
+
+    DECLARE_OLD_THIS;
+    __GLFS_ENTRY_VALIDATE_FS(fs, invalid_fs);
+
+    subvol = glfs_active_subvol(fs);
+    if (!subvol) {
+        ret = -1;
+        errno = EIO;
+        goto out;
+    }
+
+    xattr_req = dict_new();
+    if (!xattr_req) {
+        ret = -1;
+        errno = ENOMEM;
+        goto out;
+    }
+
+    gf_uuid_generate(gfid);
+    ret = dict_set_gfuuid(xattr_req, "gfid-req", gfid, true);
+    if (ret) {
+        ret = -1;
+        errno = ENOMEM;
+        goto out;
+    }
+
+    glfd = glfs_fd_new(fs);
+    if (!glfd)
+        goto out;
+
+    /* This must be glfs_resolve() and NOT glfs_lresolve().
+       That is because open("name", O_CREAT) where "name"
+       is a danging symlink must create the dangling
+       destination.
+    */
 retry:
-	ret = glfs_resolve (fs, subvol, path, &loc, &iatt, reval);
-
-	ESTALE_RETRY (ret, errno, reval, &loc, retry);
-
-	if (ret == -1 && errno != ENOENT)
-		/* Any other type of error is fatal */
-		goto out;
-
-	if (ret == -1 && errno == ENOENT && !loc.parent)
-		/* The parent directory or an ancestor even
-		   higher does not exist
-		*/
-		goto out;
-
-	if (loc.inode) {
-		if (flags & O_EXCL) {
-			ret = -1;
-			errno = EEXIST;
-			goto out;
-		}
-
-		if (IA_ISDIR (iatt.ia_type)) {
-			ret = -1;
-			errno = EISDIR;
-			goto out;
-		}
-
-		if (!IA_ISREG (iatt.ia_type)) {
-			ret = -1;
-			errno = EINVAL;
-			goto out;
-		}
-	}
-
-	if (ret == -1 && errno == ENOENT) {
-		loc.inode = inode_new (loc.parent->table);
-		if (!loc.inode) {
-			ret = -1;
-			errno = ENOMEM;
-			goto out;
-		}
-	}
-
-	if (glfd->fd) {
-		/* Retry. Safe to touch glfd->fd as we
-		   still have not glfs_fd_bind() yet.
-		*/
-		fd_unref (glfd->fd);
-		glfd->fd = NULL;
-	}
-
-	glfd->fd = fd_create (loc.inode, getpid());
-	if (!glfd->fd) {
-		ret = -1;
-		errno = ENOMEM;
-		goto out;
-	}
-        glfd->fd->flags = flags;
-
-	if (ret == 0) {
-		ret = syncop_open (subvol, &loc, flags, glfd->fd, NULL, NULL);
-                DECODE_SYNCOP_ERR (ret);
-	} else {
-		ret = syncop_create (subvol, &loc, flags, mode, glfd->fd,
-				     &iatt, xattr_req, NULL);
-                DECODE_SYNCOP_ERR (ret);
-	}
-
-	ESTALE_RETRY (ret, errno, reval, &loc, retry);
-
-	if (ret == 0)
-		ret = glfs_loc_link (&loc, &iatt);
+    ret = glfs_resolve(fs, subvol, path, &loc, &iatt, reval);
+
+    ESTALE_RETRY(ret, errno, reval, &loc, retry);
+
+    if (ret == -1 && errno != ENOENT)
+        /* Any other type of error is fatal */
+        goto out;
+
+    if (ret == -1 && errno == ENOENT && !loc.parent)
+        /* The parent directory or an ancestor even
+           higher does not exist
+        */
+        goto out;
+
+    if (loc.inode) {
+        if (flags & O_EXCL) {
+            ret = -1;
+            errno = EEXIST;
+            goto out;
+        }
+
+        if (IA_ISDIR(iatt.ia_type)) {
+            ret = -1;
+            errno = EISDIR;
+            goto out;
+        }
+
+        if (!IA_ISREG(iatt.ia_type)) {
+            ret = -1;
+            errno = EINVAL;
+            goto out;
+        }
+    }
+
+    if (ret == -1 && errno == ENOENT) {
+        loc.inode = inode_new(loc.parent->table);
+        if (!loc.inode) {
+            ret = -1;
+            errno = ENOMEM;
+            goto out;
+        }
+    }
+
+    if (glfd->fd) {
+        /* Retry. Safe to touch glfd->fd as we
+           still have not glfs_fd_bind() yet.
+        */
+        fd_unref(glfd->fd);
+        glfd->fd = NULL;
+    }
+
+    glfd->fd = fd_create(loc.inode, getpid());
+    if (!glfd->fd) {
+        ret = -1;
+        errno = ENOMEM;
+        goto out;
+    }
+    glfd->fd->flags = flags;
+
+    if (get_fop_attr_thrd_key(&xattr_req))
+        gf_msg_debug("gfapi", 0, "Getting leaseid from thread failed");
+    if (ret == 0) {
+        ret = syncop_open(subvol, &loc, flags, glfd->fd, xattr_req, NULL);
+        DECODE_SYNCOP_ERR(ret);
+    } else {
+        ret = syncop_create(subvol, &loc, flags, mode, glfd->fd, &iatt,
+                            xattr_req, NULL);
+        DECODE_SYNCOP_ERR(ret);
+    }
+
+    ESTALE_RETRY(ret, errno, reval, &loc, retry);
+
+    if (ret == 0)
+        ret = glfs_loc_link(&loc, &iatt);
 out:
-	loc_wipe (&loc);
+    loc_wipe(&loc);
 
-	if (xattr_req)
-		dict_unref (xattr_req);
+    if (xattr_req)
+        dict_unref(xattr_req);
 
-	if (ret && glfd) {
-		glfs_fd_destroy (glfd);
-		glfd = NULL;
-	} else if (glfd) {
-		fd_bind (glfd->fd);
-		glfs_fd_bind (glfd);
-	}
+    if (ret && glfd) {
+        GF_REF_PUT(glfd);
+        glfd = NULL;
+    } else if (glfd) {
+        glfd_set_state_bind(glfd);
+    }
 
-	glfs_subvol_done (fs, subvol);
+    glfs_subvol_done(fs, subvol);
 
-        __GLFS_EXIT_FS;
+    __GLFS_EXIT_FS;
 
 invalid_fs:
-	return glfd;
+    return glfd;
 }
 
-GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_creat, 3.4.0);
+#ifdef HAVE_SEEK_HOLE
+static int
+glfs_seek(struct glfs_fd *glfd, off_t offset, int whence)
+{
+    int ret = -1;
+    xlator_t *subvol = NULL;
+    fd_t *fd = NULL;
+    gf_seek_what_t what = 0;
+    off_t off = -1;
+
+    switch (whence) {
+        case SEEK_DATA:
+            what = GF_SEEK_DATA;
+            break;
+        case SEEK_HOLE:
+            what = GF_SEEK_HOLE;
+            break;
+        default:
+            /* other SEEK_* do not make sense, all operations get an offset
+             * and the position in the fd is not tracked */
+            errno = EINVAL;
+            goto out;
+    }
 
+    subvol = glfs_active_subvol(glfd->fs);
+    if (!subvol) {
+        errno = EIO;
+        goto out;
+    }
 
-off_t
-pub_glfs_lseek (struct glfs_fd *glfd, off_t offset, int whence)
-{
-	struct stat sb = {0, };
-	int         ret = -1;
-
-        DECLARE_OLD_THIS;
-	__GLFS_ENTRY_VALIDATE_FD (glfd, invalid_fs);
-
-	switch (whence) {
-	case SEEK_SET:
-		glfd->offset = offset;
-		break;
-	case SEEK_CUR:
-		glfd->offset += offset;
-		break;
-	case SEEK_END:
-		ret = pub_glfs_fstat (glfd, &sb);
-		if (ret) {
-			/* seek cannot fail :O */
-			break;
-		}
-		glfd->offset = sb.st_size + offset;
-		break;
-	}
-
-        __GLFS_EXIT_FS;
-
-	return glfd->offset;
+    fd = glfs_resolve_fd(glfd->fs, subvol, glfd);
+    if (!fd) {
+        errno = EBADFD;
+        goto done;
+    }
 
-invalid_fs:
-        return -1;
-}
+    ret = syncop_seek(subvol, fd, offset, what, NULL, &off);
+    DECODE_SYNCOP_ERR(ret);
 
-GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_lseek, 3.4.0);
+    if (ret != -1)
+        glfd->offset = off;
 
+done:
+    if (fd)
+        fd_unref(fd);
 
-ssize_t
-pub_glfs_preadv (struct glfs_fd *glfd, const struct iovec *iovec, int iovcnt,
-                 off_t offset, int flags)
-{
-	xlator_t       *subvol = NULL;
-	ssize_t         ret = -1;
-	ssize_t         size = -1;
-	struct iovec   *iov = NULL;
-	int             cnt = 0;
-	struct iobref  *iobref = NULL;
-	fd_t           *fd = NULL;
+    glfs_subvol_done(glfd->fs, subvol);
 
-        DECLARE_OLD_THIS;
-	__GLFS_ENTRY_VALIDATE_FD (glfd, invalid_fs);
+out:
+    return ret;
+}
+#endif
 
-	subvol = glfs_active_subvol (glfd->fs);
-	if (!subvol) {
-		ret = -1;
-		errno = EIO;
-		goto out;
-	}
+GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_lseek, 3.4.0)
+off_t
+pub_glfs_lseek(struct glfs_fd *glfd, off_t offset, int whence)
+{
+    struct stat sb = {
+        0,
+    };
+    int ret = -1;
+    off_t off = -1;
+
+    DECLARE_OLD_THIS;
+    __GLFS_ENTRY_VALIDATE_FD(glfd, invalid_fs);
+
+    GF_REF_GET(glfd);
+
+    switch (whence) {
+        case SEEK_SET:
+            glfd->offset = offset;
+            ret = 0;
+            break;
+        case SEEK_CUR:
+            glfd->offset += offset;
+            ret = 0;
+            break;
+        case SEEK_END:
+            ret = pub_glfs_fstat(glfd, &sb);
+            if (ret) {
+                /* seek cannot fail :O */
+                break;
+            }
+            glfd->offset = sb.st_size + offset;
+            break;
+#ifdef HAVE_SEEK_HOLE
+        case SEEK_DATA:
+        case SEEK_HOLE:
+            ret = glfs_seek(glfd, offset, whence);
+            break;
+#endif
+        default:
+            errno = EINVAL;
+    }
 
-	fd = glfs_resolve_fd (glfd->fs, subvol, glfd);
-	if (!fd) {
-		ret = -1;
-		errno = EBADFD;
-		goto out;
-	}
+    if (glfd)
+        GF_REF_PUT(glfd);
 
-	size = iov_length (iovec, iovcnt);
+    __GLFS_EXIT_FS;
 
-	ret = syncop_readv (subvol, fd, size, offset, 0, &iov, &cnt, &iobref,
-                            NULL, NULL);
-        DECODE_SYNCOP_ERR (ret);
-	if (ret <= 0)
-		goto out;
+    if (ret != -1)
+        off = glfd->offset;
 
-	size = iov_copy (iovec, iovcnt, iov, cnt); /* FIXME!!! */
+    return off;
 
-	glfd->offset = (offset + size);
+invalid_fs:
+    return -1;
+}
 
-	ret = size;
+static ssize_t
+glfs_preadv_common(struct glfs_fd *glfd, const struct iovec *iovec, int iovcnt,
+                   off_t offset, int flags, struct glfs_stat *poststat)
+{
+    xlator_t *subvol = NULL;
+    ssize_t ret = -1;
+    ssize_t size = -1;
+    struct iovec *iov = NULL;
+    int cnt = 0;
+    struct iobref *iobref = NULL;
+    fd_t *fd = NULL;
+    struct iatt iatt = {
+        0,
+    };
+    dict_t *fop_attr = NULL;
+
+    DECLARE_OLD_THIS;
+    __GLFS_ENTRY_VALIDATE_FD(glfd, invalid_fs);
+
+    GF_REF_GET(glfd);
+
+    subvol = glfs_active_subvol(glfd->fs);
+    if (!subvol) {
+        ret = -1;
+        errno = EIO;
+        goto out;
+    }
+
+    fd = glfs_resolve_fd(glfd->fs, subvol, glfd);
+    if (!fd) {
+        ret = -1;
+        errno = EBADFD;
+        goto out;
+    }
+
+    size = iov_length(iovec, iovcnt);
+
+    ret = get_fop_attr_thrd_key(&fop_attr);
+    if (ret)
+        gf_msg_debug("gfapi", 0, "Getting leaseid from thread failed");
+
+    ret = syncop_readv(subvol, fd, size, offset, 0, &iov, &cnt, &iobref, &iatt,
+                       fop_attr, NULL);
+    DECODE_SYNCOP_ERR(ret);
+
+    if (ret >= 0 && poststat)
+        glfs_iatt_to_statx(glfd->fs, &iatt, poststat);
+
+    if (ret <= 0)
+        goto out;
+
+    size = iov_copy(iovec, iovcnt, iov, cnt); /* FIXME!!! */
+
+    glfd->offset = (offset + size);
+
+    ret = size;
 out:
-        if (iov)
-                GF_FREE (iov);
-        if (iobref)
-                iobref_unref (iobref);
+    if (iov)
+        GF_FREE(iov);
+    if (iobref)
+        iobref_unref(iobref);
 
-	if (fd)
-		fd_unref (fd);
+    if (fd)
+        fd_unref(fd);
+    if (glfd)
+        GF_REF_PUT(glfd);
+    if (fop_attr)
+        dict_unref(fop_attr);
 
-	glfs_subvol_done (glfd->fs, subvol);
+    glfs_subvol_done(glfd->fs, subvol);
 
-        __GLFS_EXIT_FS;
+    __GLFS_EXIT_FS;
 
 invalid_fs:
-	return ret;
+    return ret;
 }
 
-GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_preadv, 3.4.0);
-
-
+GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_preadv, 3.4.0)
 ssize_t
-pub_glfs_read (struct glfs_fd *glfd, void *buf, size_t count, int flags)
+pub_glfs_preadv(struct glfs_fd *glfd, const struct iovec *iovec, int iovcnt,
+                off_t offset, int flags)
 {
-	struct iovec iov = {0, };
-	ssize_t      ret = 0;
+    return glfs_preadv_common(glfd, iovec, iovcnt, offset, flags, NULL);
+}
 
-	iov.iov_base = buf;
-	iov.iov_len = count;
+GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_read, 3.4.0)
+ssize_t
+pub_glfs_read(struct glfs_fd *glfd, void *buf, size_t count, int flags)
+{
+    struct iovec iov = {
+        0,
+    };
+    ssize_t ret = 0;
 
-	ret = pub_glfs_preadv (glfd, &iov, 1, glfd->offset, flags);
+    if (glfd == NULL) {
+        errno = EBADF;
+        return -1;
+    }
 
-	return ret;
-}
+    iov.iov_base = buf;
+    iov.iov_len = count;
 
-GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_read, 3.4.0);
+    ret = pub_glfs_preadv(glfd, &iov, 1, glfd->offset, flags);
 
+    return ret;
+}
 
+GFAPI_SYMVER_PUBLIC(glfs_pread34, glfs_pread, 3.4.0)
 ssize_t
-pub_glfs_pread (struct glfs_fd *glfd, void *buf, size_t count, off_t offset,
-                int flags)
+pub_glfs_pread34(struct glfs_fd *glfd, void *buf, size_t count, off_t offset,
+                 int flags)
 {
-	struct iovec iov = {0, };
-	ssize_t      ret = 0;
+    struct iovec iov = {
+        0,
+    };
+    ssize_t ret = 0;
 
-	iov.iov_base = buf;
-	iov.iov_len = count;
+    iov.iov_base = buf;
+    iov.iov_len = count;
 
-	ret = pub_glfs_preadv (glfd, &iov, 1, offset, flags);
+    ret = pub_glfs_preadv(glfd, &iov, 1, offset, flags);
 
-	return ret;
+    return ret;
 }
 
-GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_pread, 3.4.0);
-
-
+GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_pread, 6.0)
 ssize_t
-pub_glfs_readv (struct glfs_fd *glfd, const struct iovec *iov, int count,
-                int flags)
+pub_glfs_pread(struct glfs_fd *glfd, void *buf, size_t count, off_t offset,
+               int flags, struct glfs_stat *poststat)
 {
-	ssize_t      ret = 0;
+    struct iovec iov = {
+        0,
+    };
+    ssize_t ret = 0;
 
-	ret = pub_glfs_preadv (glfd, iov, count, glfd->offset, flags);
+    iov.iov_base = buf;
+    iov.iov_len = count;
 
-	return ret;
+    ret = glfs_preadv_common(glfd, &iov, 1, offset, flags, poststat);
+
+    return ret;
 }
 
-GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_readv, 3.4.0);
+GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_readv, 3.4.0)
+ssize_t
+pub_glfs_readv(struct glfs_fd *glfd, const struct iovec *iov, int count,
+               int flags)
+{
+    ssize_t ret = 0;
 
+    if (glfd == NULL) {
+        errno = EBADF;
+        return -1;
+    }
+
+    ret = pub_glfs_preadv(glfd, iov, count, glfd->offset, flags);
+
+    return ret;
+}
 
 struct glfs_io {
-	struct glfs_fd      *glfd;
-	int                  op;
-	off_t                offset;
-	struct iovec        *iov;
-	int                  count;
-	int                  flags;
-	glfs_io_cbk          fn;
-	void                *data;
+    struct glfs_fd *glfd;
+    int op;
+    off_t offset;
+    struct iovec *iov;
+    int count;
+    int flags;
+    gf_boolean_t oldcb;
+    union {
+        glfs_io_cbk34 fn34;
+        glfs_io_cbk fn;
+    };
+    void *data;
 };
 
-
 static int
-glfs_io_async_cbk (int ret, call_frame_t *frame, void *data)
+glfs_io_async_cbk(int op_ret, int op_errno, call_frame_t *frame, void *cookie,
+                  struct iovec *iovec, int count, struct iatt *prebuf,
+                  struct iatt *postbuf)
 {
-	struct glfs_io  *gio = data;
+    struct glfs_io *gio = NULL;
+    xlator_t *subvol = NULL;
+    struct glfs *fs = NULL;
+    struct glfs_fd *glfd = NULL;
+    int ret = -1;
+    struct glfs_stat prestat = {}, *prestatp = NULL;
+    struct glfs_stat poststat = {}, *poststatp = NULL;
+
+    GF_VALIDATE_OR_GOTO("gfapi", frame, inval);
+    GF_VALIDATE_OR_GOTO("gfapi", cookie, inval);
+
+    gio = frame->local;
+    frame->local = NULL;
+    subvol = cookie;
+    glfd = gio->glfd;
+    fs = glfd->fs;
+
+    if (!glfs_is_glfd_still_valid(glfd))
+        goto err;
+
+    if (op_ret <= 0) {
+        goto out;
+    } else if (gio->op == GF_FOP_READ) {
+        if (!iovec) {
+            op_ret = -1;
+            op_errno = EINVAL;
+            goto out;
+        }
+
+        op_ret = iov_copy(gio->iov, gio->count, iovec, count);
+        glfd->offset = gio->offset + op_ret;
+    } else if (gio->op == GF_FOP_WRITE) {
+        glfd->offset = gio->offset + gio->iov->iov_len;
+    }
 
-	gio->fn (gio->glfd, ret, gio->data);
+out:
+    errno = op_errno;
+    if (gio->oldcb) {
+        gio->fn34(gio->glfd, op_ret, gio->data);
+    } else {
+        if (prebuf) {
+            prestatp = &prestat;
+            glfs_iatt_to_statx(fs, prebuf, prestatp);
+        }
 
-	GF_FREE (gio->iov);
-	GF_FREE (gio);
+        if (postbuf) {
+            poststatp = &poststat;
+            glfs_iatt_to_statx(fs, postbuf, poststatp);
+        }
 
-	return 0;
+        gio->fn(gio->glfd, op_ret, prestatp, poststatp, gio->data);
+    }
+err:
+    fd_unref(glfd->fd);
+    /* Since the async operation is complete
+     * release the ref taken during the start
+     * of async operation
+     */
+    GF_REF_PUT(glfd);
+
+    GF_FREE(gio->iov);
+    GF_FREE(gio);
+    STACK_DESTROY(frame->root);
+    glfs_subvol_done(fs, subvol);
+
+    ret = 0;
+inval:
+    return ret;
 }
 
-ssize_t
-pub_glfs_pwritev (struct glfs_fd *, const struct iovec *, int, off_t, int);
+static int
+glfs_preadv_async_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                      int op_ret, int op_errno, struct iovec *iovec, int count,
+                      struct iatt *stbuf, struct iobref *iobref, dict_t *xdata)
+{
+    glfs_io_async_cbk(op_ret, op_errno, frame, cookie, iovec, count, NULL,
+                      stbuf);
 
-int
-pub_glfs_ftruncate (struct glfs_fd *, off_t);
+    return 0;
+}
 
-int
-pub_glfs_fdatasync (struct glfs_fd *);
+static int
+glfs_preadv_async_common(struct glfs_fd *glfd, const struct iovec *iovec,
+                         int count, off_t offset, int flags, gf_boolean_t oldcb,
+                         glfs_io_cbk fn, void *data)
+{
+    struct glfs_io *gio = NULL;
+    int ret = 0;
+    call_frame_t *frame = NULL;
+    xlator_t *subvol = NULL;
+    struct glfs *fs = NULL;
+    fd_t *fd = NULL;
+    dict_t *fop_attr = NULL;
+
+    DECLARE_OLD_THIS;
+    __GLFS_ENTRY_VALIDATE_FD(glfd, invalid_fs);
+
+    GF_REF_GET(glfd);
+
+    subvol = glfs_active_subvol(glfd->fs);
+    if (!subvol) {
+        ret = -1;
+        errno = EIO;
+        goto out;
+    }
+
+    fd = glfs_resolve_fd(glfd->fs, subvol, glfd);
+    if (!fd) {
+        ret = -1;
+        errno = EBADFD;
+        goto out;
+    }
+
+    fs = glfd->fs;
+
+    frame = syncop_create_frame(THIS);
+    if (!frame) {
+        ret = -1;
+        errno = ENOMEM;
+        goto out;
+    }
+
+    gio = GF_CALLOC(1, sizeof(*gio), glfs_mt_glfs_io_t);
+    if (!gio) {
+        ret = -1;
+        errno = ENOMEM;
+        goto out;
+    }
+
+    gio->iov = iov_dup(iovec, count);
+    if (!gio->iov) {
+        ret = -1;
+        errno = ENOMEM;
+        goto out;
+    }
+
+    gio->op = GF_FOP_READ;
+    gio->glfd = glfd;
+    gio->count = count;
+    gio->offset = offset;
+    gio->flags = flags;
+    gio->oldcb = oldcb;
+    gio->fn = fn;
+    gio->data = data;
+
+    frame->local = gio;
+
+    ret = get_fop_attr_thrd_key(&fop_attr);
+    if (ret)
+        gf_msg_debug("gfapi", 0, "Getting leaseid from thread failed");
+
+    STACK_WIND_COOKIE(frame, glfs_preadv_async_cbk, subvol, subvol,
+                      subvol->fops->readv, fd, iov_length(iovec, count), offset,
+                      flags, fop_attr);
 
-int
-pub_glfs_fsync (struct glfs_fd *glfd);
+out:
+    if (ret) {
+        if (fd)
+            fd_unref(fd);
+        if (glfd)
+            GF_REF_PUT(glfd);
+        if (gio) {
+            GF_FREE(gio->iov);
+            GF_FREE(gio);
+        }
+        if (frame) {
+            STACK_DESTROY(frame->root);
+        }
+        glfs_subvol_done(fs, subvol);
+    }
+    if (fop_attr)
+        dict_unref(fop_attr);
 
-int
-pub_glfs_discard (struct glfs_fd *, off_t, size_t);
+    __GLFS_EXIT_FS;
 
-int
-pub_glfs_zerofill (struct glfs_fd *, off_t, off_t);
+    return ret;
 
-static int
-glfs_io_async_task (void *data)
-{
-	struct glfs_io *gio = data;
-	ssize_t         ret = 0;
-
-	switch (gio->op) {
-	case GF_FOP_WRITE:
-		ret = pub_glfs_pwritev (gio->glfd, gio->iov, gio->count,
-				    gio->offset, gio->flags);
-		break;
-	case GF_FOP_FTRUNCATE:
-		ret = pub_glfs_ftruncate (gio->glfd, gio->offset);
-		break;
-	case GF_FOP_FSYNC:
-		if (gio->flags)
-			ret = pub_glfs_fdatasync (gio->glfd);
-		else
-			ret = pub_glfs_fsync (gio->glfd);
-		break;
-	case GF_FOP_DISCARD:
-		ret = pub_glfs_discard (gio->glfd, gio->offset, gio->count);
-		break;
-        case GF_FOP_ZEROFILL:
-                ret = pub_glfs_zerofill(gio->glfd, gio->offset, gio->count);
-                break;
-	}
-
-	return (int) ret;
+invalid_fs:
+    return -1;
 }
 
-
+GFAPI_SYMVER_PUBLIC(glfs_preadv_async34, glfs_preadv_async, 3.4.0)
 int
-glfs_preadv_async_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-		       int op_ret, int op_errno, struct iovec *iovec,
-		       int count, struct iatt *stbuf, struct iobref *iobref,
-		       dict_t *xdata)
+pub_glfs_preadv_async34(struct glfs_fd *glfd, const struct iovec *iovec,
+                        int count, off_t offset, int flags, glfs_io_cbk34 fn,
+                        void *data)
 {
-	struct glfs_io *gio = NULL;
-	xlator_t       *subvol = NULL;
-	struct glfs    *fs = NULL;
-	struct glfs_fd *glfd = NULL;
-
+    return glfs_preadv_async_common(glfd, iovec, count, offset, flags, _gf_true,
+                                    (void *)fn, data);
+}
 
-	gio = frame->local;
-	frame->local = NULL;
-	subvol = cookie;
-	glfd = gio->glfd;
-	fs = glfd->fs;
+GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_preadv_async, 6.0)
+int
+pub_glfs_preadv_async(struct glfs_fd *glfd, const struct iovec *iovec,
+                      int count, off_t offset, int flags, glfs_io_cbk fn,
+                      void *data)
+{
+    return glfs_preadv_async_common(glfd, iovec, count, offset, flags,
+                                    _gf_false, fn, data);
+}
 
-	if (op_ret <= 0)
-		goto out;
+GFAPI_SYMVER_PUBLIC(glfs_read_async34, glfs_read_async, 3.4.0)
+int
+pub_glfs_read_async34(struct glfs_fd *glfd, void *buf, size_t count, int flags,
+                      glfs_io_cbk34 fn, void *data)
+{
+    struct iovec iov = {
+        0,
+    };
+    ssize_t ret = 0;
 
-	op_ret = iov_copy (gio->iov, gio->count, iovec, count);
+    if (glfd == NULL) {
+        errno = EBADF;
+        return -1;
+    }
 
-	glfd->offset = gio->offset + op_ret;
-out:
-	errno = op_errno;
-	gio->fn (gio->glfd, op_ret, gio->data);
+    iov.iov_base = buf;
+    iov.iov_len = count;
 
-	GF_FREE (gio->iov);
-	GF_FREE (gio);
-	STACK_DESTROY (frame->root);
-	glfs_subvol_done (fs, subvol);
+    ret = glfs_preadv_async_common(glfd, &iov, 1, glfd->offset, flags, _gf_true,
+                                   (void *)fn, data);
 
-	return 0;
+    return ret;
 }
 
-
+GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_read_async, 6.0)
 int
-pub_glfs_preadv_async (struct glfs_fd *glfd, const struct iovec *iovec,
-                       int count, off_t offset, int flags, glfs_io_cbk fn,
-                       void *data)
+pub_glfs_read_async(struct glfs_fd *glfd, void *buf, size_t count, int flags,
+                    glfs_io_cbk fn, void *data)
 {
-	struct glfs_io *gio = NULL;
-	int             ret = 0;
-	call_frame_t   *frame = NULL;
-	xlator_t       *subvol = NULL;
-	glfs_t         *fs = NULL;
-	fd_t           *fd = NULL;
-
-        DECLARE_OLD_THIS;
-	__GLFS_ENTRY_VALIDATE_FD (glfd, invalid_fs);
-
-	subvol = glfs_active_subvol (glfd->fs);
-	if (!subvol) {
-		ret = -1;
-		errno = EIO;
-		goto out;
-	}
-
-	fd = glfs_resolve_fd (glfd->fs, subvol, glfd);
-	if (!fd) {
-		ret = -1;
-		errno = EBADFD;
-		goto out;
-	}
-
-	fs = glfd->fs;
-
-	frame = syncop_create_frame (THIS);
-	if (!frame) {
-		ret = -1;
-		errno = ENOMEM;
-		goto out;
-	}
-
-	gio = GF_CALLOC (1, sizeof (*gio), glfs_mt_glfs_io_t);
-	if (!gio) {
-		ret = -1;
-		errno = ENOMEM;
-		goto out;
-	}
-
-	gio->iov = iov_dup (iovec, count);
-	if (!gio->iov) {
-		ret = -1;
-		errno = ENOMEM;
-		goto out;
-	}
-
-	gio->op     = GF_FOP_READ;
-	gio->glfd   = glfd;
-	gio->count  = count;
-	gio->offset = offset;
-	gio->flags  = flags;
-	gio->fn     = fn;
-	gio->data   = data;
-
-	frame->local = gio;
-
-	STACK_WIND_COOKIE (frame, glfs_preadv_async_cbk, subvol, subvol,
-			   subvol->fops->readv, fd, iov_length (iovec, count),
-			   offset, flags, NULL);
+    struct iovec iov = {
+        0,
+    };
+    ssize_t ret = 0;
 
-out:
-        if (ret) {
-                if (gio) {
-                        GF_FREE (gio->iov);
-                        GF_FREE (gio);
-                }
-                if (frame) {
-                        STACK_DESTROY (frame->root);
-                }
-		glfs_subvol_done (fs, subvol);
-	}
-
-	if (fd)
-		fd_unref (fd);
+    if (glfd == NULL) {
+        errno = EBADF;
+        return -1;
+    }
 
-        __GLFS_EXIT_FS;
+    iov.iov_base = buf;
+    iov.iov_len = count;
 
-	return ret;
+    ret = glfs_preadv_async_common(glfd, &iov, 1, glfd->offset, flags,
+                                   _gf_false, fn, data);
 
-invalid_fs:
-        return -1;
+    return ret;
 }
 
-GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_preadv_async, 3.4.0);
-
-
+GFAPI_SYMVER_PUBLIC(glfs_pread_async34, glfs_pread_async, 3.4.0)
 int
-pub_glfs_read_async (struct glfs_fd *glfd, void *buf, size_t count, int flags,
-                     glfs_io_cbk fn, void *data)
+pub_glfs_pread_async34(struct glfs_fd *glfd, void *buf, size_t count,
+                       off_t offset, int flags, glfs_io_cbk34 fn, void *data)
 {
-	struct iovec iov = {0, };
-	ssize_t      ret = 0;
+    struct iovec iov = {
+        0,
+    };
+    ssize_t ret = 0;
 
-	iov.iov_base = buf;
-	iov.iov_len = count;
+    iov.iov_base = buf;
+    iov.iov_len = count;
 
-	ret = pub_glfs_preadv_async (glfd, &iov, 1, glfd->offset, flags, fn, data);
+    ret = glfs_preadv_async_common(glfd, &iov, 1, offset, flags, _gf_true,
+                                   (void *)fn, data);
 
-	return ret;
+    return ret;
 }
 
-GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_read_async, 3.4.0);
-
-
+GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_pread_async, 6.0)
 int
-pub_glfs_pread_async (struct glfs_fd *glfd, void *buf, size_t count,
-                      off_t offset, int flags, glfs_io_cbk fn, void *data)
+pub_glfs_pread_async(struct glfs_fd *glfd, void *buf, size_t count,
+                     off_t offset, int flags, glfs_io_cbk fn, void *data)
 {
-	struct iovec iov = {0, };
-	ssize_t      ret = 0;
+    struct iovec iov = {
+        0,
+    };
+    ssize_t ret = 0;
 
-	iov.iov_base = buf;
-	iov.iov_len = count;
+    iov.iov_base = buf;
+    iov.iov_len = count;
 
-	ret = pub_glfs_preadv_async (glfd, &iov, 1, offset, flags, fn, data);
+    ret = glfs_preadv_async_common(glfd, &iov, 1, offset, flags, _gf_false, fn,
+                                   data);
 
-	return ret;
+    return ret;
 }
 
-GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_pread_async, 3.4.0);
-
-
+GFAPI_SYMVER_PUBLIC(glfs_readv_async34, glfs_readv_async, 3.4.0)
 int
-pub_glfs_readv_async (struct glfs_fd *glfd, const struct iovec *iov, int count,
-                      int flags, glfs_io_cbk fn, void *data)
+pub_glfs_readv_async34(struct glfs_fd *glfd, const struct iovec *iov, int count,
+                       int flags, glfs_io_cbk34 fn, void *data)
 {
-	ssize_t      ret = 0;
+    ssize_t ret = 0;
+
+    if (glfd == NULL) {
+        errno = EBADF;
+        return -1;
+    }
 
-	ret = pub_glfs_preadv_async (glfd, iov, count, glfd->offset, flags,
-				  fn, data);
-	return ret;
+    ret = glfs_preadv_async_common(glfd, iov, count, glfd->offset, flags,
+                                   _gf_true, (void *)fn, data);
+    return ret;
 }
 
-GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_readv_async, 3.4.0);
+GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_readv_async, 6.0)
+int
+pub_glfs_readv_async(struct glfs_fd *glfd, const struct iovec *iov, int count,
+                     int flags, glfs_io_cbk fn, void *data)
+{
+    ssize_t ret = 0;
 
+    if (glfd == NULL) {
+        errno = EBADF;
+        return -1;
+    }
 
-ssize_t
-pub_glfs_pwritev (struct glfs_fd *glfd, const struct iovec *iovec, int iovcnt,
-                  off_t offset, int flags)
+    ret = glfs_preadv_async_common(glfd, iov, count, glfd->offset, flags,
+                                   _gf_false, fn, data);
+    return ret;
+}
+
+static ssize_t
+glfs_pwritev_common(struct glfs_fd *glfd, const struct iovec *iovec, int iovcnt,
+                    off_t offset, int flags, struct glfs_stat *prestat,
+                    struct glfs_stat *poststat)
 {
-	xlator_t       *subvol = NULL;
-	int             ret = -1;
-	size_t          size = -1;
-	struct iobref  *iobref = NULL;
-	struct iobuf   *iobuf = NULL;
-	struct iovec    iov = {0, };
-	fd_t           *fd = NULL;
-
-        DECLARE_OLD_THIS;
-	__GLFS_ENTRY_VALIDATE_FD (glfd, invalid_fs);
-
-	subvol = glfs_active_subvol (glfd->fs);
-	if (!subvol) {
-		ret = -1;
-		errno = EIO;
-		goto out;
-	}
-
-	fd = glfs_resolve_fd (glfd->fs, subvol, glfd);
-	if (!fd) {
-		ret = -1;
-		errno = EBADFD;
-		goto out;
-	}
-
-	size = iov_length (iovec, iovcnt);
-
-	iobuf = iobuf_get2 (subvol->ctx->iobuf_pool, size);
-	if (!iobuf) {
-		ret = -1;
-		errno = ENOMEM;
-		goto out;
-	}
-
-	iobref = iobref_new ();
-	if (!iobref) {
-		iobuf_unref (iobuf);
-		errno = ENOMEM;
-		ret = -1;
-		goto out;
-	}
-
-	ret = iobref_add (iobref, iobuf);
-	if (ret) {
-		iobuf_unref (iobuf);
-		iobref_unref (iobref);
-		errno = ENOMEM;
-		ret = -1;
-		goto out;
-	}
-
-	iov_unload (iobuf_ptr (iobuf), iovec, iovcnt);  /* FIXME!!! */
-
-	iov.iov_base = iobuf_ptr (iobuf);
-	iov.iov_len = size;
-
-	ret = syncop_writev (subvol, fd, &iov, 1, offset, iobref, flags, NULL,
-                             NULL);
-        DECODE_SYNCOP_ERR (ret);
+    xlator_t *subvol = NULL;
+    int ret = -1;
+    struct iobref *iobref = NULL;
+    struct iobuf *iobuf = NULL;
+    struct iovec iov = {
+        0,
+    };
+    fd_t *fd = NULL;
+    struct iatt preiatt =
+                    {
+                        0,
+                    },
+                postiatt = {
+                    0,
+                };
+    dict_t *fop_attr = NULL;
+
+    DECLARE_OLD_THIS;
+    __GLFS_ENTRY_VALIDATE_FD(glfd, invalid_fs);
+
+    GF_REF_GET(glfd);
+
+    subvol = glfs_active_subvol(glfd->fs);
+    if (!subvol) {
+        ret = -1;
+        errno = EIO;
+        goto out;
+    }
+
+    fd = glfs_resolve_fd(glfd->fs, subvol, glfd);
+    if (!fd) {
+        ret = -1;
+        errno = EBADFD;
+        goto out;
+    }
+
+    ret = iobuf_copy(subvol->ctx->iobuf_pool, iovec, iovcnt, &iobref, &iobuf,
+                     &iov);
+    if (ret)
+        goto out;
+
+    ret = get_fop_attr_thrd_key(&fop_attr);
+    if (ret)
+        gf_msg_debug("gfapi", 0, "Getting leaseid from thread failed");
+
+    ret = syncop_writev(subvol, fd, &iov, 1, offset, iobref, flags, &preiatt,
+                        &postiatt, fop_attr, NULL);
+    DECODE_SYNCOP_ERR(ret);
+
+    if (ret >= 0) {
+        if (prestat)
+            glfs_iatt_to_statx(glfd->fs, &preiatt, prestat);
+        if (poststat)
+            glfs_iatt_to_statx(glfd->fs, &postiatt, poststat);
+    }
+
+    if (ret <= 0)
+        goto out;
+
+    glfd->offset = (offset + iov.iov_len);
+out:
+    if (iobuf)
+        iobuf_unref(iobuf);
+    if (iobref)
+        iobref_unref(iobref);
+    if (fd)
+        fd_unref(fd);
+    if (glfd)
+        GF_REF_PUT(glfd);
+    if (fop_attr)
+        dict_unref(fop_attr);
 
-	iobuf_unref (iobuf);
-	iobref_unref (iobref);
+    glfs_subvol_done(glfd->fs, subvol);
 
-	if (ret <= 0)
-		goto out;
+    __GLFS_EXIT_FS;
 
-	glfd->offset = (offset + size);
+invalid_fs:
+    return ret;
+}
+
+GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_copy_file_range, 6.0)
+ssize_t
+pub_glfs_copy_file_range(struct glfs_fd *glfd_in, off64_t *off_in,
+                         struct glfs_fd *glfd_out, off64_t *off_out, size_t len,
+                         unsigned int flags, struct glfs_stat *statbuf,
+                         struct glfs_stat *prestat, struct glfs_stat *poststat)
+{
+    xlator_t *subvol = NULL;
+    int ret = -1;
+    fd_t *fd_in = NULL;
+    fd_t *fd_out = NULL;
+    struct iatt preiatt =
+                    {
+                        0,
+                    },
+                iattbuf =
+                    {
+                        0,
+                    },
+                postiatt = {
+                    0,
+                };
+    dict_t *fop_attr = NULL;
+    off64_t pos_in;
+    off64_t pos_out;
+
+    DECLARE_OLD_THIS;
+    __GLFS_ENTRY_VALIDATE_FD(glfd_in, invalid_fs);
+    __GLFS_ENTRY_VALIDATE_FD(glfd_out, invalid_fs);
+
+    GF_REF_GET(glfd_in);
+    GF_REF_GET(glfd_out);
+
+    if (glfd_in->fs != glfd_out->fs) {
+        ret = -1;
+        errno = EXDEV;
+        goto out;
+    }
+
+    subvol = glfs_active_subvol(glfd_in->fs);
+    if (!subvol) {
+        ret = -1;
+        errno = EIO;
+        goto out;
+    }
+
+    fd_in = glfs_resolve_fd(glfd_in->fs, subvol, glfd_in);
+    if (!fd_in) {
+        ret = -1;
+        errno = EBADFD;
+        goto out;
+    }
+
+    fd_out = glfs_resolve_fd(glfd_out->fs, subvol, glfd_out);
+    if (!fd_out) {
+        ret = -1;
+        errno = EBADFD;
+        goto out;
+    }
+
+    /*
+     * This is based on how the vfs layer in the kernel handles
+     * copy_file_range call. Upon receiving it follows the
+     * below method to consider the offset.
+     * if (off_in != NULL)
+     *    use the value off_in to perform the op
+     * else if off_in == NULL
+     *    use the current file offset position to perform the op
+     *
+     * For gfapi, glfd->offset is used. For a freshly opened
+     * fd, the offset is set to 0.
+     */
+    if (off_in)
+        pos_in = *off_in;
+    else
+        pos_in = glfd_in->offset;
+
+    if (off_out)
+        pos_out = *off_out;
+    else
+        pos_out = glfd_out->offset;
+
+    ret = get_fop_attr_thrd_key(&fop_attr);
+    if (ret)
+        gf_msg_debug("gfapi", 0, "Getting leaseid from thread failed");
+
+    ret = syncop_copy_file_range(subvol, fd_in, pos_in, fd_out, pos_out, len,
+                                 flags, &iattbuf, &preiatt, &postiatt, fop_attr,
+                                 NULL);
+    DECODE_SYNCOP_ERR(ret);
+
+    if (ret >= 0) {
+        pos_in += ret;
+        pos_out += ret;
+
+        if (off_in)
+            *off_in = pos_in;
+        if (off_out)
+            *off_out = pos_out;
+
+        if (statbuf)
+            glfs_iatt_to_statx(glfd_in->fs, &iattbuf, statbuf);
+        if (prestat)
+            glfs_iatt_to_statx(glfd_in->fs, &preiatt, prestat);
+        if (poststat)
+            glfs_iatt_to_statx(glfd_in->fs, &postiatt, poststat);
+    }
+
+    if (ret <= 0)
+        goto out;
+
+    /*
+     * If *off_in is NULL, then there is no offset info that can
+     * obtained from the input argument. Hence follow below method.
+     *  If *off_in is NULL, then
+     *     glfd->offset = offset + ret;
+     * else
+     *     do nothing.
+     *
+     * According to the man page of copy_file_range, if off_in is
+     * NULL, then the offset of the source file is advanced by
+     * the return value of the fop. The same applies to off_out as
+     * well. Otherwise, if *off_in is not NULL, then the offset
+     * is not advanced by the filesystem. The entity which sends
+     * the copy_file_range call is supposed to advance the offset
+     * value in its buffer (pointed to by *off_in or *off_out)
+     * by the return value of copy_file_range.
+     */
+    if (!off_in)
+        glfd_in->offset += ret;
+
+    if (!off_out)
+        glfd_out->offset += ret;
 
 out:
-	if (fd)
-		fd_unref (fd);
+    if (fd_in)
+        fd_unref(fd_in);
+    if (fd_out)
+        fd_unref(fd_out);
+    if (glfd_in)
+        GF_REF_PUT(glfd_in);
+    if (glfd_out)
+        GF_REF_PUT(glfd_out);
+    if (fop_attr)
+        dict_unref(fop_attr);
 
-	glfs_subvol_done (glfd->fs, subvol);
+    glfs_subvol_done(glfd_in->fs, subvol);
 
-        __GLFS_EXIT_FS;
+    __GLFS_EXIT_FS;
 
 invalid_fs:
-	return ret;
+    return ret;
 }
 
-GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_pwritev, 3.4.0);
-
-
+GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_pwritev, 3.4.0)
 ssize_t
-pub_glfs_write (struct glfs_fd *glfd, const void *buf, size_t count, int flags)
+pub_glfs_pwritev(struct glfs_fd *glfd, const struct iovec *iovec, int iovcnt,
+                 off_t offset, int flags)
 {
-	struct iovec iov = {0, };
-	ssize_t      ret = 0;
+    return glfs_pwritev_common(glfd, iovec, iovcnt, offset, flags, NULL, NULL);
+}
 
-	iov.iov_base = (void *) buf;
-	iov.iov_len = count;
+GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_write, 3.4.0)
+ssize_t
+pub_glfs_write(struct glfs_fd *glfd, const void *buf, size_t count, int flags)
+{
+    struct iovec iov = {
+        0,
+    };
+    ssize_t ret = 0;
 
-	ret = pub_glfs_pwritev (glfd, &iov, 1, glfd->offset, flags);
+    if (glfd == NULL) {
+        errno = EBADF;
+        return -1;
+    }
 
-	return ret;
-}
+    iov.iov_base = (void *)buf;
+    iov.iov_len = count;
 
-GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_write, 3.4.0);
+    ret = pub_glfs_pwritev(glfd, &iov, 1, glfd->offset, flags);
 
+    return ret;
+}
 
+GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_writev, 3.4.0)
 ssize_t
-pub_glfs_writev (struct glfs_fd *glfd, const struct iovec *iov, int count,
-                 int flags)
+pub_glfs_writev(struct glfs_fd *glfd, const struct iovec *iov, int count,
+                int flags)
 {
-	ssize_t      ret = 0;
+    ssize_t ret = 0;
+
+    if (glfd == NULL) {
+        errno = EBADF;
+        return -1;
+    }
 
-	ret = pub_glfs_pwritev (glfd, iov, count, glfd->offset, flags);
+    ret = pub_glfs_pwritev(glfd, iov, count, glfd->offset, flags);
 
-	return ret;
+    return ret;
 }
 
-GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_writev, 3.4.0);
+GFAPI_SYMVER_PUBLIC(glfs_pwrite34, glfs_pwrite, 3.4.0)
+ssize_t
+pub_glfs_pwrite34(struct glfs_fd *glfd, const void *buf, size_t count,
+                  off_t offset, int flags)
+{
+    struct iovec iov = {
+        0,
+    };
+    ssize_t ret = 0;
 
+    iov.iov_base = (void *)buf;
+    iov.iov_len = count;
+
+    ret = pub_glfs_pwritev(glfd, &iov, 1, offset, flags);
+
+    return ret;
+}
 
+GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_pwrite, 6.0)
 ssize_t
-pub_glfs_pwrite (struct glfs_fd *glfd, const void *buf, size_t count,
-                 off_t offset, int flags)
+pub_glfs_pwrite(struct glfs_fd *glfd, const void *buf, size_t count,
+                off_t offset, int flags, struct glfs_stat *prestat,
+                struct glfs_stat *poststat)
 {
-	struct iovec iov = {0, };
-	ssize_t      ret = 0;
+    struct iovec iov = {
+        0,
+    };
+    ssize_t ret = 0;
 
-	iov.iov_base = (void *) buf;
-	iov.iov_len = count;
+    iov.iov_base = (void *)buf;
+    iov.iov_len = count;
 
-	ret = pub_glfs_pwritev (glfd, &iov, 1, offset, flags);
+    ret = glfs_pwritev_common(glfd, &iov, 1, offset, flags, prestat, poststat);
 
-	return ret;
+    return ret;
 }
 
-GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_pwrite, 3.4.0);
+extern glfs_t *
+pub_glfs_from_glfd(glfs_fd_t *);
 
+static int
+glfs_pwritev_async_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                       int op_ret, int op_errno, struct iatt *prebuf,
+                       struct iatt *postbuf, dict_t *xdata)
+{
+    glfs_io_async_cbk(op_ret, op_errno, frame, cookie, NULL, 0, prebuf,
+                      postbuf);
 
-extern glfs_t *pub_glfs_from_glfd (glfs_fd_t *);
+    return 0;
+}
 
-int
-pub_glfs_pwritev_async (struct glfs_fd *glfd, const struct iovec *iovec,
-                        int count, off_t offset, int flags, glfs_io_cbk fn,
-                        void *data)
+static int
+glfs_pwritev_async_common(struct glfs_fd *glfd, const struct iovec *iovec,
+                          int count, off_t offset, int flags,
+                          gf_boolean_t oldcb, glfs_io_cbk fn, void *data)
 {
-	struct glfs_io *gio = NULL;
-	int             ret = -1;
-
-        DECLARE_OLD_THIS;
-        __GLFS_ENTRY_VALIDATE_FD (glfd, invalid_fs);
-
-	gio = GF_CALLOC (1, sizeof (*gio), glfs_mt_glfs_io_t);
-	if (!gio) {
-		errno = ENOMEM;
-		goto out;
-	}
-
-	gio->iov = iov_dup (iovec, count);
-	if (!gio->iov) {
-		GF_FREE (gio);
-		errno = ENOMEM;
-		goto out;
-	}
-
-	gio->op     = GF_FOP_WRITE;
-	gio->glfd   = glfd;
-	gio->count  = count;
-	gio->offset = offset;
-	gio->flags  = flags;
-	gio->fn     = fn;
-	gio->data   = data;
-
-	ret = synctask_new (pub_glfs_from_glfd (glfd)->ctx->env,
-			    glfs_io_async_task, glfs_io_async_cbk,
-			    NULL, gio);
-
-	if (ret) {
-		GF_FREE (gio->iov);
-		GF_FREE (gio);
-	}
-
+    struct glfs_io *gio = NULL;
+    int ret = -1;
+    call_frame_t *frame = NULL;
+    xlator_t *subvol = NULL;
+    fd_t *fd = NULL;
+    struct iobref *iobref = NULL;
+    struct iobuf *iobuf = NULL;
+    dict_t *fop_attr = NULL;
+
+    DECLARE_OLD_THIS;
+    __GLFS_ENTRY_VALIDATE_FD(glfd, invalid_fs);
+
+    /* Need to take explicit ref so that the fd
+     * is not destroyed before the fop is complete
+     */
+    GF_REF_GET(glfd);
+
+    subvol = glfs_active_subvol(glfd->fs);
+    if (!subvol) {
+        errno = EIO;
+        goto out;
+    }
+
+    fd = glfs_resolve_fd(glfd->fs, subvol, glfd);
+    if (!fd) {
+        errno = EBADFD;
+        goto out;
+    }
+
+    gio = GF_CALLOC(1, sizeof(*gio), glfs_mt_glfs_io_t);
+    if (!gio) {
+        errno = ENOMEM;
+        goto out;
+    }
+
+    gio->op = GF_FOP_WRITE;
+    gio->glfd = glfd;
+    gio->offset = offset;
+    gio->flags = flags;
+    gio->oldcb = oldcb;
+    gio->fn = fn;
+    gio->data = data;
+    gio->count = 1;
+    gio->iov = GF_CALLOC(gio->count, sizeof(*(gio->iov)), gf_common_mt_iovec);
+    if (!gio->iov) {
+        errno = ENOMEM;
+        goto out;
+    }
+
+    ret = iobuf_copy(subvol->ctx->iobuf_pool, iovec, count, &iobref, &iobuf,
+                     gio->iov);
+    if (ret)
+        goto out;
+
+    frame = syncop_create_frame(THIS);
+    if (!frame) {
+        errno = ENOMEM;
+        ret = -1;
+        goto out;
+    }
+
+    frame->local = gio;
+
+    ret = get_fop_attr_thrd_key(&fop_attr);
+    if (ret)
+        gf_msg_debug("gfapi", 0, "Getting leaseid from thread failed");
+
+    STACK_WIND_COOKIE(frame, glfs_pwritev_async_cbk, subvol, subvol,
+                      subvol->fops->writev, fd, gio->iov, gio->count, offset,
+                      flags, iobref, fop_attr);
+
+    ret = 0;
 out:
-        __GLFS_EXIT_FS;
+    if (ret) {
+        if (fd)
+            fd_unref(fd);
+        if (glfd)
+            GF_REF_PUT(glfd);
+        GF_FREE(gio);
+        /*
+         * If there is any error condition check after the frame
+         * creation, we have to destroy the frame root.
+         */
+        glfs_subvol_done(glfd->fs, subvol);
+    }
+    if (fop_attr)
+        dict_unref(fop_attr);
+
+    if (iobuf)
+        iobuf_unref(iobuf);
+    if (iobref)
+        iobref_unref(iobref);
+
+    __GLFS_EXIT_FS;
 
 invalid_fs:
-	return ret;
+    return ret;
 }
 
-GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_pwritev_async, 3.4.0);
+GFAPI_SYMVER_PUBLIC(glfs_pwritev_async34, glfs_pwritev_async, 3.4.0)
+int
+pub_glfs_pwritev_async34(struct glfs_fd *glfd, const struct iovec *iovec,
+                         int count, off_t offset, int flags, glfs_io_cbk34 fn,
+                         void *data)
+{
+    return glfs_pwritev_async_common(glfd, iovec, count, offset, flags,
+                                     _gf_true, (void *)fn, data);
+}
 
+GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_pwritev_async, 6.0)
+int
+pub_glfs_pwritev_async(struct glfs_fd *glfd, const struct iovec *iovec,
+                       int count, off_t offset, int flags, glfs_io_cbk fn,
+                       void *data)
+{
+    return glfs_pwritev_async_common(glfd, iovec, count, offset, flags,
+                                     _gf_false, fn, data);
+}
 
+GFAPI_SYMVER_PUBLIC(glfs_write_async34, glfs_write_async, 3.4.0)
 int
-pub_glfs_write_async (struct glfs_fd *glfd, const void *buf, size_t count,
-                      int flags, glfs_io_cbk fn, void *data)
+pub_glfs_write_async34(struct glfs_fd *glfd, const void *buf, size_t count,
+                       int flags, glfs_io_cbk34 fn, void *data)
 {
-	struct iovec iov = {0, };
-	ssize_t      ret = 0;
+    struct iovec iov = {
+        0,
+    };
+    ssize_t ret = 0;
 
-	iov.iov_base = (void *) buf;
-	iov.iov_len = count;
+    if (glfd == NULL) {
+        errno = EBADF;
+        return -1;
+    }
+
+    iov.iov_base = (void *)buf;
+    iov.iov_len = count;
 
-	ret = pub_glfs_pwritev_async (glfd, &iov, 1, glfd->offset, flags, fn, data);
+    ret = glfs_pwritev_async_common(glfd, &iov, 1, glfd->offset, flags,
+                                    _gf_true, (void *)fn, data);
 
-	return ret;
+    return ret;
 }
 
-GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_write_async, 3.4.0);
+GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_write_async, 6.0)
+int
+pub_glfs_write_async(struct glfs_fd *glfd, const void *buf, size_t count,
+                     int flags, glfs_io_cbk fn, void *data)
+{
+    struct iovec iov = {
+        0,
+    };
+    ssize_t ret = 0;
+
+    if (glfd == NULL) {
+        errno = EBADF;
+        return -1;
+    }
+
+    iov.iov_base = (void *)buf;
+    iov.iov_len = count;
 
+    ret = glfs_pwritev_async_common(glfd, &iov, 1, glfd->offset, flags,
+                                    _gf_false, fn, data);
 
+    return ret;
+}
+
+GFAPI_SYMVER_PUBLIC(glfs_pwrite_async34, glfs_pwrite_async, 3.4.0)
 int
-pub_glfs_pwrite_async (struct glfs_fd *glfd, const void *buf, int count,
-                       off_t offset, int flags, glfs_io_cbk fn, void *data)
+pub_glfs_pwrite_async34(struct glfs_fd *glfd, const void *buf, int count,
+                        off_t offset, int flags, glfs_io_cbk34 fn, void *data)
 {
-	struct iovec iov = {0, };
-	ssize_t      ret = 0;
+    struct iovec iov = {
+        0,
+    };
+    ssize_t ret = 0;
 
-	iov.iov_base = (void *) buf;
-	iov.iov_len = count;
+    iov.iov_base = (void *)buf;
+    iov.iov_len = count;
 
-	ret = pub_glfs_pwritev_async (glfd, &iov, 1, offset, flags, fn, data);
+    ret = glfs_pwritev_async_common(glfd, &iov, 1, offset, flags, _gf_true,
+                                    (void *)fn, data);
 
-	return ret;
+    return ret;
 }
 
-GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_pwrite_async, 3.4.0);
+GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_pwrite_async, 6.0)
+int
+pub_glfs_pwrite_async(struct glfs_fd *glfd, const void *buf, int count,
+                      off_t offset, int flags, glfs_io_cbk fn, void *data)
+{
+    struct iovec iov = {
+        0,
+    };
+    ssize_t ret = 0;
+
+    iov.iov_base = (void *)buf;
+    iov.iov_len = count;
 
+    ret = glfs_pwritev_async_common(glfd, &iov, 1, offset, flags, _gf_false, fn,
+                                    data);
 
+    return ret;
+}
+
+GFAPI_SYMVER_PUBLIC(glfs_writev_async34, glfs_writev_async, 3.4.0)
 int
-pub_glfs_writev_async (struct glfs_fd *glfd, const struct iovec *iov, int count,
-                       int flags, glfs_io_cbk fn, void *data)
+pub_glfs_writev_async34(struct glfs_fd *glfd, const struct iovec *iov,
+                        int count, int flags, glfs_io_cbk34 fn, void *data)
 {
-	ssize_t      ret = 0;
+    ssize_t ret = 0;
+
+    if (glfd == NULL) {
+        errno = EBADF;
+        return -1;
+    }
 
-	ret = pub_glfs_pwritev_async (glfd, iov, count, glfd->offset, flags,
-				   fn, data);
-	return ret;
+    ret = glfs_pwritev_async_common(glfd, iov, count, glfd->offset, flags,
+                                    _gf_true, (void *)fn, data);
+    return ret;
 }
 
-GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_writev_async, 3.4.0);
+GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_writev_async, 6.0)
+int
+pub_glfs_writev_async(struct glfs_fd *glfd, const struct iovec *iov, int count,
+                      int flags, glfs_io_cbk fn, void *data)
+{
+    ssize_t ret = 0;
+
+    if (glfd == NULL) {
+        errno = EBADF;
+        return -1;
+    }
 
+    ret = glfs_pwritev_async_common(glfd, iov, count, glfd->offset, flags,
+                                    _gf_false, fn, data);
+    return ret;
+}
 
-int
-pub_glfs_fsync (struct glfs_fd *glfd)
-{
-	int              ret = -1;
-	xlator_t        *subvol = NULL;
-	fd_t            *fd = NULL;
-
-        DECLARE_OLD_THIS;
-	__GLFS_ENTRY_VALIDATE_FD (glfd, invalid_fs);
-
-	subvol = glfs_active_subvol (glfd->fs);
-	if (!subvol) {
-		ret = -1;
-		errno = EIO;
-		goto out;
-	}
-
-	fd = glfs_resolve_fd (glfd->fs, subvol, glfd);
-	if (!fd) {
-		ret = -1;
-		errno = EBADFD;
-		goto out;
-	}
-
-	ret = syncop_fsync (subvol, fd, 0, NULL, NULL);
-        DECODE_SYNCOP_ERR (ret);
+static int
+glfs_fsync_common(struct glfs_fd *glfd, struct glfs_stat *prestat,
+                  struct glfs_stat *poststat)
+{
+    int ret = -1;
+    xlator_t *subvol = NULL;
+    fd_t *fd = NULL;
+    struct iatt preiatt =
+                    {
+                        0,
+                    },
+                postiatt = {
+                    0,
+                };
+    dict_t *fop_attr = NULL;
+
+    DECLARE_OLD_THIS;
+    __GLFS_ENTRY_VALIDATE_FD(glfd, invalid_fs);
+
+    GF_REF_GET(glfd);
+
+    subvol = glfs_active_subvol(glfd->fs);
+    if (!subvol) {
+        ret = -1;
+        errno = EIO;
+        goto out;
+    }
+
+    fd = glfs_resolve_fd(glfd->fs, subvol, glfd);
+    if (!fd) {
+        ret = -1;
+        errno = EBADFD;
+        goto out;
+    }
+
+    ret = get_fop_attr_thrd_key(&fop_attr);
+    if (ret)
+        gf_msg_debug("gfapi", 0, "Getting leaseid from thread failed");
+
+    ret = syncop_fsync(subvol, fd, 0, &preiatt, &postiatt, fop_attr, NULL);
+    DECODE_SYNCOP_ERR(ret);
+
+    if (ret >= 0) {
+        if (prestat)
+            glfs_iatt_to_statx(glfd->fs, &preiatt, prestat);
+        if (poststat)
+            glfs_iatt_to_statx(glfd->fs, &postiatt, poststat);
+    }
 out:
-	if (fd)
-		fd_unref (fd);
+    if (fd)
+        fd_unref(fd);
+    if (glfd)
+        GF_REF_PUT(glfd);
+    if (fop_attr)
+        dict_unref(fop_attr);
 
-	glfs_subvol_done (glfd->fs, subvol);
+    glfs_subvol_done(glfd->fs, subvol);
 
-        __GLFS_EXIT_FS;
+    __GLFS_EXIT_FS;
 
 invalid_fs:
-	return ret;
+    return ret;
+}
+
+GFAPI_SYMVER_PUBLIC(glfs_fsync34, glfs_fsync, 3.4.0)
+int
+pub_glfs_fsync34(struct glfs_fd *glfd)
+{
+    return glfs_fsync_common(glfd, NULL, NULL);
+}
+
+GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_fsync, 6.0)
+int
+pub_glfs_fsync(struct glfs_fd *glfd, struct glfs_stat *prestat,
+               struct glfs_stat *poststat)
+{
+    return glfs_fsync_common(glfd, prestat, poststat);
 }
 
-GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_fsync, 3.4.0);
+static int
+glfs_fsync_async_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                     int32_t op_ret, int32_t op_errno, struct iatt *prebuf,
+                     struct iatt *postbuf, dict_t *xdata)
+{
+    glfs_io_async_cbk(op_ret, op_errno, frame, cookie, NULL, 0, prebuf,
+                      postbuf);
 
+    return 0;
+}
 
 static int
-glfs_fsync_async_common (struct glfs_fd *glfd, glfs_io_cbk fn, void *data,
-			 int dataonly)
+glfs_fsync_async_common(struct glfs_fd *glfd, gf_boolean_t oldcb,
+                        glfs_io_cbk fn, void *data, int dataonly)
 {
-	struct glfs_io *gio = NULL;
-	int             ret = 0;
+    struct glfs_io *gio = NULL;
+    int ret = 0;
+    call_frame_t *frame = NULL;
+    xlator_t *subvol = NULL;
+    fd_t *fd = NULL;
+
+    /* Need to take explicit ref so that the fd
+     * is not destroyed before the fop is complete
+     */
+    GF_REF_GET(glfd);
+
+    subvol = glfs_active_subvol(glfd->fs);
+    if (!subvol) {
+        ret = -1;
+        errno = EIO;
+        goto out;
+    }
+
+    fd = glfs_resolve_fd(glfd->fs, subvol, glfd);
+    if (!fd) {
+        ret = -1;
+        errno = EBADFD;
+        goto out;
+    }
+
+    frame = syncop_create_frame(THIS);
+    if (!frame) {
+        ret = -1;
+        errno = ENOMEM;
+        goto out;
+    }
+
+    gio = GF_CALLOC(1, sizeof(*gio), glfs_mt_glfs_io_t);
+    if (!gio) {
+        errno = ENOMEM;
+        ret = -1;
+        goto out;
+    }
+
+    gio->op = GF_FOP_FSYNC;
+    gio->glfd = glfd;
+    gio->flags = dataonly;
+    gio->oldcb = oldcb;
+    gio->fn = fn;
+    gio->data = data;
+
+    frame->local = gio;
+
+    STACK_WIND_COOKIE(frame, glfs_fsync_async_cbk, subvol, subvol,
+                      subvol->fops->fsync, fd, dataonly, NULL);
 
-	gio = GF_CALLOC (1, sizeof (*gio), glfs_mt_glfs_io_t);
-	if (!gio) {
-		errno = ENOMEM;
-		return -1;
-	}
+out:
+    if (ret) {
+        if (fd)
+            fd_unref(fd);
+        GF_REF_PUT(glfd);
+        GF_FREE(gio);
+        if (frame)
+            STACK_DESTROY(frame->root);
+        glfs_subvol_done(glfd->fs, subvol);
+    }
+
+    return ret;
+}
 
-	gio->op     = GF_FOP_FSYNC;
-	gio->glfd   = glfd;
-	gio->flags  = dataonly;
-	gio->fn     = fn;
-	gio->data   = data;
+GFAPI_SYMVER_PUBLIC(glfs_fsync_async34, glfs_fsync_async, 3.4.0)
+int
+pub_glfs_fsync_async34(struct glfs_fd *glfd, glfs_io_cbk34 fn, void *data)
+{
+    int ret = -1;
 
-	ret = synctask_new (pub_glfs_from_glfd (glfd)->ctx->env,
-			    glfs_io_async_task, glfs_io_async_cbk,
-			    NULL, gio);
+    DECLARE_OLD_THIS;
+    __GLFS_ENTRY_VALIDATE_FD(glfd, invalid_fs);
 
-	if (ret) {
-		GF_FREE (gio->iov);
-		GF_FREE (gio);
-	}
+    ret = glfs_fsync_async_common(glfd, _gf_true, (void *)fn, data, 0);
 
-	return ret;
+    __GLFS_EXIT_FS;
 
+invalid_fs:
+    return ret;
 }
 
-
+GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_fsync_async, 6.0)
 int
-pub_glfs_fsync_async (struct glfs_fd *glfd, glfs_io_cbk fn, void *data)
+pub_glfs_fsync_async(struct glfs_fd *glfd, glfs_io_cbk fn, void *data)
 {
-        int ret = -1;
+    int ret = -1;
 
-        DECLARE_OLD_THIS;
-        __GLFS_ENTRY_VALIDATE_FD (glfd, invalid_fs);
+    DECLARE_OLD_THIS;
+    __GLFS_ENTRY_VALIDATE_FD(glfd, invalid_fs);
 
-        ret = glfs_fsync_async_common (glfd, fn, data, 0);
+    ret = glfs_fsync_async_common(glfd, _gf_false, fn, data, 0);
 
-        __GLFS_EXIT_FS;
+    __GLFS_EXIT_FS;
 
 invalid_fs:
-        return ret;
+    return ret;
 }
 
-GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_fsync_async, 3.4.0);
-
-
-int
-pub_glfs_fdatasync (struct glfs_fd *glfd)
-{
-	int              ret = -1;
-	xlator_t        *subvol = NULL;
-	fd_t            *fd = NULL;
-
-        DECLARE_OLD_THIS;
-	__GLFS_ENTRY_VALIDATE_FD (glfd, invalid_fs);
-
-	subvol = glfs_active_subvol (glfd->fs);
-	if (!subvol) {
-		ret = -1;
-		errno = EIO;
-		goto out;
-	}
-
-	fd = glfs_resolve_fd (glfd->fs, subvol, glfd);
-	if (!fd) {
-		ret = -1;
-		errno = EBADFD;
-		goto out;
-	}
-
-	ret = syncop_fsync (subvol, fd, 1, NULL, NULL);
-        DECODE_SYNCOP_ERR (ret);
+static int
+glfs_fdatasync_common(struct glfs_fd *glfd, struct glfs_stat *prestat,
+                      struct glfs_stat *poststat)
+{
+    int ret = -1;
+    xlator_t *subvol = NULL;
+    fd_t *fd = NULL;
+    struct iatt preiatt =
+                    {
+                        0,
+                    },
+                postiatt = {
+                    0,
+                };
+    dict_t *fop_attr = NULL;
+
+    DECLARE_OLD_THIS;
+    __GLFS_ENTRY_VALIDATE_FD(glfd, invalid_fs);
+
+    GF_REF_GET(glfd);
+
+    subvol = glfs_active_subvol(glfd->fs);
+    if (!subvol) {
+        ret = -1;
+        errno = EIO;
+        goto out;
+    }
+
+    fd = glfs_resolve_fd(glfd->fs, subvol, glfd);
+    if (!fd) {
+        ret = -1;
+        errno = EBADFD;
+        goto out;
+    }
+
+    ret = get_fop_attr_thrd_key(&fop_attr);
+    if (ret)
+        gf_msg_debug("gfapi", 0, "Getting leaseid from thread failed");
+
+    ret = syncop_fsync(subvol, fd, 1, &preiatt, &postiatt, fop_attr, NULL);
+    DECODE_SYNCOP_ERR(ret);
+
+    if (ret >= 0) {
+        if (prestat)
+            glfs_iatt_to_statx(glfd->fs, &preiatt, prestat);
+        if (poststat)
+            glfs_iatt_to_statx(glfd->fs, &postiatt, poststat);
+    }
 out:
-	if (fd)
-		fd_unref (fd);
+    if (fd)
+        fd_unref(fd);
+    if (glfd)
+        GF_REF_PUT(glfd);
+    if (fop_attr)
+        dict_unref(fop_attr);
 
-	glfs_subvol_done (glfd->fs, subvol);
+    glfs_subvol_done(glfd->fs, subvol);
 
-        __GLFS_EXIT_FS;
+    __GLFS_EXIT_FS;
 
 invalid_fs:
-	return ret;
+    return ret;
 }
 
-GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_fdatasync, 3.4.0);
+GFAPI_SYMVER_PUBLIC(glfs_fdatasync34, glfs_fdatasync, 3.4.0)
+int
+pub_glfs_fdatasync34(struct glfs_fd *glfd)
+{
+    return glfs_fdatasync_common(glfd, NULL, NULL);
+}
 
+GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_fdatasync, 6.0)
+int
+pub_glfs_fdatasync(struct glfs_fd *glfd, struct glfs_stat *prestat,
+                   struct glfs_stat *poststat)
+{
+    return glfs_fdatasync_common(glfd, prestat, poststat);
+}
 
+GFAPI_SYMVER_PUBLIC(glfs_fdatasync_async34, glfs_fdatasync_async, 3.4.0)
 int
-pub_glfs_fdatasync_async (struct glfs_fd *glfd, glfs_io_cbk fn, void *data)
+pub_glfs_fdatasync_async34(struct glfs_fd *glfd, glfs_io_cbk34 fn, void *data)
 {
-        int ret = -1;
+    int ret = -1;
 
-        DECLARE_OLD_THIS;
-        __GLFS_ENTRY_VALIDATE_FD (glfd, invalid_fs);
+    DECLARE_OLD_THIS;
+    __GLFS_ENTRY_VALIDATE_FD(glfd, invalid_fs);
 
-	ret = glfs_fsync_async_common (glfd, fn, data, 1);
+    ret = glfs_fsync_async_common(glfd, _gf_true, (void *)fn, data, 1);
 
-        __GLFS_EXIT_FS;
+    __GLFS_EXIT_FS;
 
 invalid_fs:
-        return ret;
+    return ret;
 }
 
-GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_fdatasync_async, 3.4.0);
+GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_fdatasync_async, 6.0)
+int
+pub_glfs_fdatasync_async(struct glfs_fd *glfd, glfs_io_cbk fn, void *data)
+{
+    int ret = -1;
 
+    DECLARE_OLD_THIS;
+    __GLFS_ENTRY_VALIDATE_FD(glfd, invalid_fs);
 
-int
-pub_glfs_ftruncate (struct glfs_fd *glfd, off_t offset)
-{
-	int              ret = -1;
-	xlator_t        *subvol = NULL;
-	fd_t            *fd = NULL;
-
-        DECLARE_OLD_THIS;
-        __GLFS_ENTRY_VALIDATE_FD (glfd, invalid_fs);
-
-	subvol = glfs_active_subvol (glfd->fs);
-	if (!subvol) {
-		ret = -1;
-		errno = EIO;
-		goto out;
-	}
-
-	fd = glfs_resolve_fd (glfd->fs, subvol, glfd);
-	if (!fd) {
-		ret = -1;
-		errno = EBADFD;
-		goto out;
-	}
-
-	ret = syncop_ftruncate (subvol, fd, offset, NULL, NULL);
-        DECODE_SYNCOP_ERR (ret);
+    ret = glfs_fsync_async_common(glfd, _gf_false, fn, data, 1);
+
+    __GLFS_EXIT_FS;
+
+invalid_fs:
+    return ret;
+}
+
+static int
+glfs_ftruncate_common(struct glfs_fd *glfd, off_t offset,
+                      struct glfs_stat *prestat, struct glfs_stat *poststat)
+{
+    int ret = -1;
+    xlator_t *subvol = NULL;
+    fd_t *fd = NULL;
+    struct iatt preiatt =
+                    {
+                        0,
+                    },
+                postiatt = {
+                    0,
+                };
+    dict_t *fop_attr = NULL;
+
+    DECLARE_OLD_THIS;
+    __GLFS_ENTRY_VALIDATE_FD(glfd, invalid_fs);
+
+    GF_REF_GET(glfd);
+
+    subvol = glfs_active_subvol(glfd->fs);
+    if (!subvol) {
+        ret = -1;
+        errno = EIO;
+        goto out;
+    }
+
+    fd = glfs_resolve_fd(glfd->fs, subvol, glfd);
+    if (!fd) {
+        ret = -1;
+        errno = EBADFD;
+        goto out;
+    }
+
+    ret = get_fop_attr_thrd_key(&fop_attr);
+    if (ret)
+        gf_msg_debug("gfapi", 0, "Getting leaseid from thread failed");
+
+    ret = syncop_ftruncate(subvol, fd, offset, &preiatt, &postiatt, fop_attr,
+                           NULL);
+    DECODE_SYNCOP_ERR(ret);
+
+    if (ret >= 0) {
+        if (prestat)
+            glfs_iatt_to_statx(glfd->fs, &preiatt, prestat);
+        if (poststat)
+            glfs_iatt_to_statx(glfd->fs, &postiatt, poststat);
+    }
 out:
-	if (fd)
-		fd_unref (fd);
+    if (fd)
+        fd_unref(fd);
+    if (glfd)
+        GF_REF_PUT(glfd);
+    if (fop_attr)
+        dict_unref(fop_attr);
 
-	glfs_subvol_done (glfd->fs, subvol);
+    glfs_subvol_done(glfd->fs, subvol);
 
-        __GLFS_EXIT_FS;
+    __GLFS_EXIT_FS;
 
 invalid_fs:
-	return ret;
+    return ret;
 }
 
-GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_ftruncate, 3.4.0);
+GFAPI_SYMVER_PUBLIC(glfs_ftruncate34, glfs_ftruncate, 3.4.0)
+int
+pub_glfs_ftruncate34(struct glfs_fd *glfd, off_t offset)
+{
+    return glfs_ftruncate_common(glfd, offset, NULL, NULL);
+}
 
+GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_ftruncate, 6.0)
+int
+pub_glfs_ftruncate(struct glfs_fd *glfd, off_t offset,
+                   struct glfs_stat *prestat, struct glfs_stat *poststat)
+{
+    return glfs_ftruncate_common(glfd, offset, prestat, poststat);
+}
 
+GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_truncate, 3.7.15)
 int
-pub_glfs_ftruncate_async (struct glfs_fd *glfd, off_t offset, glfs_io_cbk fn,
-                          void *data)
+pub_glfs_truncate(struct glfs *fs, const char *path, off_t length)
 {
-	struct glfs_io *gio = NULL;
-	int             ret = -1;
+    int ret = -1;
+    xlator_t *subvol = NULL;
+    loc_t loc = {
+        0,
+    };
+    struct iatt iatt = {
+        0,
+    };
+    int reval = 0;
+
+    DECLARE_OLD_THIS;
+    __GLFS_ENTRY_VALIDATE_FS(fs, invalid_fs);
+
+    subvol = glfs_active_subvol(fs);
+    if (!subvol) {
+        ret = -1;
+        errno = EIO;
+        goto out;
+    }
+retry:
+    ret = glfs_resolve(fs, subvol, path, &loc, &iatt, reval);
 
-        DECLARE_OLD_THIS;
-        __GLFS_ENTRY_VALIDATE_FD (glfd, invalid_fs);
+    ESTALE_RETRY(ret, errno, reval, &loc, retry);
 
-	gio = GF_CALLOC (1, sizeof (*gio), glfs_mt_glfs_io_t);
-	if (!gio) {
-		errno = ENOMEM;
-		goto out;
-	}
+    if (ret)
+        goto out;
 
-	gio->op     = GF_FOP_FTRUNCATE;
-	gio->glfd   = glfd;
-	gio->offset = offset;
-	gio->fn     = fn;
-	gio->data   = data;
+    ret = syncop_truncate(subvol, &loc, length, NULL, NULL);
+    DECODE_SYNCOP_ERR(ret);
 
-	ret = synctask_new (pub_glfs_from_glfd (glfd)->ctx->env,
-			    glfs_io_async_task, glfs_io_async_cbk,
-			    NULL, gio);
+    ESTALE_RETRY(ret, errno, reval, &loc, retry);
+out:
+    loc_wipe(&loc);
+
+    glfs_subvol_done(fs, subvol);
 
-	if (ret) {
-		GF_FREE (gio->iov);
-		GF_FREE (gio);
-	}
+    __GLFS_EXIT_FS;
+
+invalid_fs:
+    return ret;
+}
+
+static int
+glfs_ftruncate_async_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                         int32_t op_ret, int32_t op_errno, struct iatt *prebuf,
+                         struct iatt *postbuf, dict_t *xdata)
+{
+    glfs_io_async_cbk(op_ret, op_errno, frame, cookie, NULL, 0, prebuf,
+                      postbuf);
+
+    return 0;
+}
+
+static int
+glfs_ftruncate_async_common(struct glfs_fd *glfd, off_t offset,
+                            gf_boolean_t oldcb, glfs_io_cbk fn, void *data)
+{
+    struct glfs_io *gio = NULL;
+    int ret = -1;
+    call_frame_t *frame = NULL;
+    xlator_t *subvol = NULL;
+    fd_t *fd = NULL;
+    dict_t *fop_attr = NULL;
+
+    DECLARE_OLD_THIS;
+    __GLFS_ENTRY_VALIDATE_FD(glfd, invalid_fs);
+
+    /* Need to take explicit ref so that the fd
+     * is not destroyed before the fop is complete
+     */
+    GF_REF_GET(glfd);
+
+    subvol = glfs_active_subvol(glfd->fs);
+    if (!subvol) {
+        errno = EIO;
+        goto out;
+    }
+
+    fd = glfs_resolve_fd(glfd->fs, subvol, glfd);
+    if (!fd) {
+        errno = EBADFD;
+        goto out;
+    }
+
+    frame = syncop_create_frame(THIS);
+    if (!frame) {
+        errno = ENOMEM;
+        goto out;
+    }
+
+    gio = GF_CALLOC(1, sizeof(*gio), glfs_mt_glfs_io_t);
+    if (!gio) {
+        errno = ENOMEM;
+        goto out;
+    }
+
+    gio->op = GF_FOP_FTRUNCATE;
+    gio->glfd = glfd;
+    gio->offset = offset;
+    gio->oldcb = oldcb;
+    gio->fn = fn;
+    gio->data = data;
+
+    frame->local = gio;
+
+    ret = get_fop_attr_thrd_key(&fop_attr);
+    if (ret)
+        gf_msg_debug("gfapi", 0, "Getting leaseid from thread failed");
+
+    STACK_WIND_COOKIE(frame, glfs_ftruncate_async_cbk, subvol, subvol,
+                      subvol->fops->ftruncate, fd, offset, fop_attr);
+
+    ret = 0;
 
 out:
-        __GLFS_EXIT_FS;
+    if (ret) {
+        if (fd)
+            fd_unref(fd);
+        if (glfd)
+            GF_REF_PUT(glfd);
+        GF_FREE(gio);
+        if (frame)
+            STACK_DESTROY(frame->root);
+        glfs_subvol_done(glfd->fs, subvol);
+    }
+    if (fop_attr)
+        dict_unref(fop_attr);
+
+    __GLFS_EXIT_FS;
 
 invalid_fs:
-	return ret;
+    return ret;
 }
 
-GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_ftruncate_async, 3.4.0);
+GFAPI_SYMVER_PUBLIC(glfs_ftruncate_async34, glfs_ftruncate_async, 3.4.0)
+int
+pub_glfs_ftruncate_async34(struct glfs_fd *glfd, off_t offset, glfs_io_cbk34 fn,
+                           void *data)
+{
+    return glfs_ftruncate_async_common(glfd, offset, _gf_true, (void *)fn,
+                                       data);
+}
 
+GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_ftruncate_async, 6.0)
+int
+pub_glfs_ftruncate_async(struct glfs_fd *glfd, off_t offset, glfs_io_cbk fn,
+                         void *data)
+{
+    return glfs_ftruncate_async_common(glfd, offset, _gf_false, fn, data);
+}
 
+GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_access, 3.4.0)
 int
-pub_glfs_access (struct glfs *fs, const char *path, int mode)
-{
-	int              ret = -1;
-	xlator_t        *subvol = NULL;
-	loc_t            loc = {0, };
-	struct iatt      iatt = {0, };
-	int              reval = 0;
-
-        DECLARE_OLD_THIS;
-        __GLFS_ENTRY_VALIDATE_FS (fs, invalid_fs);
-
-	subvol = glfs_active_subvol (fs);
-	if (!subvol) {
-		ret = -1;
-		errno = EIO;
-		goto out;
-	}
+pub_glfs_access(struct glfs *fs, const char *path, int mode)
+{
+    int ret = -1;
+    xlator_t *subvol = NULL;
+    loc_t loc = {
+        0,
+    };
+    struct iatt iatt = {
+        0,
+    };
+    int reval = 0;
+
+    DECLARE_OLD_THIS;
+    __GLFS_ENTRY_VALIDATE_FS(fs, invalid_fs);
+
+    subvol = glfs_active_subvol(fs);
+    if (!subvol) {
+        ret = -1;
+        errno = EIO;
+        goto out;
+    }
 retry:
-	ret = glfs_resolve (fs, subvol, path, &loc, &iatt, reval);
+    ret = glfs_resolve(fs, subvol, path, &loc, &iatt, reval);
 
-	ESTALE_RETRY (ret, errno, reval, &loc, retry);
+    ESTALE_RETRY(ret, errno, reval, &loc, retry);
 
-	if (ret)
-		goto out;
+    if (ret)
+        goto out;
 
-	ret = syncop_access (subvol, &loc, mode, NULL, NULL);
-        DECODE_SYNCOP_ERR (ret);
+    ret = syncop_access(subvol, &loc, mode, NULL, NULL);
+    DECODE_SYNCOP_ERR(ret);
 
-	ESTALE_RETRY (ret, errno, reval, &loc, retry);
+    ESTALE_RETRY(ret, errno, reval, &loc, retry);
 out:
-	loc_wipe (&loc);
+    loc_wipe(&loc);
 
-	glfs_subvol_done (fs, subvol);
+    glfs_subvol_done(fs, subvol);
 
-        __GLFS_EXIT_FS;
+    __GLFS_EXIT_FS;
 
 invalid_fs:
-	return ret;
+    return ret;
 }
 
-GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_access, 3.4.0);
-
-
+GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_symlink, 3.4.0)
 int
-pub_glfs_symlink (struct glfs *fs, const char *data, const char *path)
-{
-	int              ret = -1;
-	xlator_t        *subvol = NULL;
-	loc_t            loc = {0, };
-	struct iatt      iatt = {0, };
-	uuid_t           gfid;
-	dict_t          *xattr_req = NULL;
-	int              reval = 0;
-
-        DECLARE_OLD_THIS;
-        __GLFS_ENTRY_VALIDATE_FS (fs, invalid_fs);
-
-	subvol = glfs_active_subvol (fs);
-	if (!subvol) {
-		ret = -1;
-		errno = EIO;
-		goto out;
-	}
-
-	xattr_req = dict_new ();
-	if (!xattr_req) {
-		ret = -1;
-		errno = ENOMEM;
-		goto out;
-	}
-
-	gf_uuid_generate (gfid);
-	ret = dict_set_static_bin (xattr_req, "gfid-req", gfid, 16);
-	if (ret) {
-		ret = -1;
-		errno = ENOMEM;
-		goto out;
-	}
+pub_glfs_symlink(struct glfs *fs, const char *data, const char *path)
+{
+    int ret = -1;
+    xlator_t *subvol = NULL;
+    loc_t loc = {
+        0,
+    };
+    struct iatt iatt = {
+        0,
+    };
+    uuid_t gfid;
+    dict_t *xattr_req = NULL;
+    int reval = 0;
+
+    DECLARE_OLD_THIS;
+    __GLFS_ENTRY_VALIDATE_FS(fs, invalid_fs);
+
+    subvol = glfs_active_subvol(fs);
+    if (!subvol) {
+        ret = -1;
+        errno = EIO;
+        goto out;
+    }
+
+    xattr_req = dict_new();
+    if (!xattr_req) {
+        ret = -1;
+        errno = ENOMEM;
+        goto out;
+    }
+
+    gf_uuid_generate(gfid);
+    ret = dict_set_gfuuid(xattr_req, "gfid-req", gfid, true);
+    if (ret) {
+        ret = -1;
+        errno = ENOMEM;
+        goto out;
+    }
 retry:
-	ret = glfs_lresolve (fs, subvol, path, &loc, &iatt, reval);
+    ret = glfs_lresolve(fs, subvol, path, &loc, &iatt, reval);
 
-	ESTALE_RETRY (ret, errno, reval, &loc, retry);
+    ESTALE_RETRY(ret, errno, reval, &loc, retry);
 
-	if (loc.inode) {
-		errno = EEXIST;
-		ret = -1;
-		goto out;
-	}
+    if (loc.inode) {
+        errno = EEXIST;
+        ret = -1;
+        goto out;
+    }
 
-	if (ret == -1 && errno != ENOENT)
-		/* Any other type of error is fatal */
-		goto out;
+    if (ret == -1 && errno != ENOENT)
+        /* Any other type of error is fatal */
+        goto out;
 
-	if (ret == -1 && errno == ENOENT && !loc.parent)
-		/* The parent directory or an ancestor even
-		   higher does not exist
-		*/
-		goto out;
+    if (ret == -1 && errno == ENOENT && !loc.parent)
+        /* The parent directory or an ancestor even
+           higher does not exist
+        */
+        goto out;
 
-	/* ret == -1 && errno == ENOENT */
-	loc.inode = inode_new (loc.parent->table);
-	if (!loc.inode) {
-		ret = -1;
-		errno = ENOMEM;
-		goto out;
-	}
+    /* ret == -1 && errno == ENOENT */
+    loc.inode = inode_new(loc.parent->table);
+    if (!loc.inode) {
+        ret = -1;
+        errno = ENOMEM;
+        goto out;
+    }
 
-	ret = syncop_symlink (subvol, &loc, data, &iatt, xattr_req, NULL);
-        DECODE_SYNCOP_ERR (ret);
+    ret = syncop_symlink(subvol, &loc, data, &iatt, xattr_req, NULL);
+    DECODE_SYNCOP_ERR(ret);
 
-	ESTALE_RETRY (ret, errno, reval, &loc, retry);
+    ESTALE_RETRY(ret, errno, reval, &loc, retry);
 
-	if (ret == 0)
-		ret = glfs_loc_link (&loc, &iatt);
+    if (ret == 0)
+        ret = glfs_loc_link(&loc, &iatt);
 out:
-	loc_wipe (&loc);
+    loc_wipe(&loc);
 
-	if (xattr_req)
-		dict_unref (xattr_req);
+    if (xattr_req)
+        dict_unref(xattr_req);
 
-	glfs_subvol_done (fs, subvol);
+    glfs_subvol_done(fs, subvol);
 
-        __GLFS_EXIT_FS;
+    __GLFS_EXIT_FS;
 
 invalid_fs:
-	return ret;
+    return ret;
 }
 
-GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_symlink, 3.4.0);
-
-
+GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_readlink, 3.4.0)
 int
-pub_glfs_readlink (struct glfs *fs, const char *path, char *buf, size_t bufsiz)
-{
-	int              ret = -1;
-	xlator_t        *subvol = NULL;
-	loc_t            loc = {0, };
-	struct iatt      iatt = {0, };
-	int              reval = 0;
-	char            *linkval = NULL;
-
-        DECLARE_OLD_THIS;
-        __GLFS_ENTRY_VALIDATE_FS (fs, invalid_fs);
-
-	subvol = glfs_active_subvol (fs);
-	if (!subvol) {
-		ret = -1;
-		errno = EIO;
-		goto out;
-	}
+pub_glfs_readlink(struct glfs *fs, const char *path, char *buf, size_t bufsiz)
+{
+    int ret = -1;
+    xlator_t *subvol = NULL;
+    loc_t loc = {
+        0,
+    };
+    struct iatt iatt = {
+        0,
+    };
+    int reval = 0;
+    char *linkval = NULL;
+
+    DECLARE_OLD_THIS;
+    __GLFS_ENTRY_VALIDATE_FS(fs, invalid_fs);
+
+    subvol = glfs_active_subvol(fs);
+    if (!subvol) {
+        ret = -1;
+        errno = EIO;
+        goto out;
+    }
 retry:
-	ret = glfs_lresolve (fs, subvol, path, &loc, &iatt, reval);
+    ret = glfs_lresolve(fs, subvol, path, &loc, &iatt, reval);
 
-	ESTALE_RETRY (ret, errno, reval, &loc, retry);
+    ESTALE_RETRY(ret, errno, reval, &loc, retry);
 
-	if (ret)
-		goto out;
+    if (ret)
+        goto out;
 
-	if (iatt.ia_type != IA_IFLNK) {
-		ret = -1;
-		errno = EINVAL;
-		goto out;
-	}
+    if (iatt.ia_type != IA_IFLNK) {
+        ret = -1;
+        errno = EINVAL;
+        goto out;
+    }
 
-	ret = syncop_readlink (subvol, &loc, &linkval, bufsiz, NULL, NULL);
-        DECODE_SYNCOP_ERR (ret);
-	if (ret > 0) {
-		memcpy (buf, linkval, ret);
-		GF_FREE (linkval);
-	}
+    ret = syncop_readlink(subvol, &loc, &linkval, bufsiz, NULL, NULL);
+    DECODE_SYNCOP_ERR(ret);
+    if (ret > 0) {
+        memcpy(buf, linkval, ret);
+        GF_FREE(linkval);
+    }
 
-	ESTALE_RETRY (ret, errno, reval, &loc, retry);
+    ESTALE_RETRY(ret, errno, reval, &loc, retry);
 out:
-	loc_wipe (&loc);
+    loc_wipe(&loc);
 
-	glfs_subvol_done (fs, subvol);
+    glfs_subvol_done(fs, subvol);
 
-        __GLFS_EXIT_FS;
+    __GLFS_EXIT_FS;
 
 invalid_fs:
-	return ret;
+    return ret;
 }
 
-GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_readlink, 3.4.0);
-
-
+GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_mknod, 3.4.0)
 int
-pub_glfs_mknod (struct glfs *fs, const char *path, mode_t mode, dev_t dev)
-{
-	int              ret = -1;
-	xlator_t        *subvol = NULL;
-	loc_t            loc = {0, };
-	struct iatt      iatt = {0, };
-	uuid_t           gfid;
-	dict_t          *xattr_req = NULL;
-	int              reval = 0;
-
-        DECLARE_OLD_THIS;
-        __GLFS_ENTRY_VALIDATE_FS (fs, invalid_fs);
-
-	subvol = glfs_active_subvol (fs);
-	if (!subvol) {
-		ret = -1;
-		errno = EIO;
-		goto out;
-	}
-
-	xattr_req = dict_new ();
-	if (!xattr_req) {
-		ret = -1;
-		errno = ENOMEM;
-		goto out;
-	}
-
-	gf_uuid_generate (gfid);
-	ret = dict_set_static_bin (xattr_req, "gfid-req", gfid, 16);
-	if (ret) {
-		ret = -1;
-		errno = ENOMEM;
-		goto out;
-	}
+pub_glfs_mknod(struct glfs *fs, const char *path, mode_t mode, dev_t dev)
+{
+    int ret = -1;
+    xlator_t *subvol = NULL;
+    loc_t loc = {
+        0,
+    };
+    struct iatt iatt = {
+        0,
+    };
+    uuid_t gfid;
+    dict_t *xattr_req = NULL;
+    int reval = 0;
+
+    DECLARE_OLD_THIS;
+    __GLFS_ENTRY_VALIDATE_FS(fs, invalid_fs);
+
+    subvol = glfs_active_subvol(fs);
+    if (!subvol) {
+        ret = -1;
+        errno = EIO;
+        goto out;
+    }
+
+    xattr_req = dict_new();
+    if (!xattr_req) {
+        ret = -1;
+        errno = ENOMEM;
+        goto out;
+    }
+
+    gf_uuid_generate(gfid);
+    ret = dict_set_gfuuid(xattr_req, "gfid-req", gfid, true);
+    if (ret) {
+        ret = -1;
+        errno = ENOMEM;
+        goto out;
+    }
 retry:
-	ret = glfs_lresolve (fs, subvol, path, &loc, &iatt, reval);
+    ret = glfs_lresolve(fs, subvol, path, &loc, &iatt, reval);
 
-	ESTALE_RETRY (ret, errno, reval, &loc, retry);
+    ESTALE_RETRY(ret, errno, reval, &loc, retry);
 
-	if (loc.inode) {
-		errno = EEXIST;
-		ret = -1;
-		goto out;
-	}
+    if (loc.inode) {
+        errno = EEXIST;
+        ret = -1;
+        goto out;
+    }
 
-	if (ret == -1 && errno != ENOENT)
-		/* Any other type of error is fatal */
-		goto out;
+    if (ret == -1 && errno != ENOENT)
+        /* Any other type of error is fatal */
+        goto out;
 
-	if (ret == -1 && errno == ENOENT && !loc.parent)
-		/* The parent directory or an ancestor even
-		   higher does not exist
-		*/
-		goto out;
+    if (ret == -1 && errno == ENOENT && !loc.parent)
+        /* The parent directory or an ancestor even
+           higher does not exist
+        */
+        goto out;
 
-	/* ret == -1 && errno == ENOENT */
-	loc.inode = inode_new (loc.parent->table);
-	if (!loc.inode) {
-		ret = -1;
-		errno = ENOMEM;
-		goto out;
-	}
+    /* ret == -1 && errno == ENOENT */
+    loc.inode = inode_new(loc.parent->table);
+    if (!loc.inode) {
+        ret = -1;
+        errno = ENOMEM;
+        goto out;
+    }
 
-	ret = syncop_mknod (subvol, &loc, mode, dev, &iatt, xattr_req, NULL);
-        DECODE_SYNCOP_ERR (ret);
+    ret = syncop_mknod(subvol, &loc, mode, dev, &iatt, xattr_req, NULL);
+    DECODE_SYNCOP_ERR(ret);
 
-	ESTALE_RETRY (ret, errno, reval, &loc, retry);
+    ESTALE_RETRY(ret, errno, reval, &loc, retry);
 
-	if (ret == 0)
-		ret = glfs_loc_link (&loc, &iatt);
+    if (ret == 0)
+        ret = glfs_loc_link(&loc, &iatt);
 out:
-	loc_wipe (&loc);
+    loc_wipe(&loc);
 
-	if (xattr_req)
-		dict_unref (xattr_req);
+    if (xattr_req)
+        dict_unref(xattr_req);
 
-	glfs_subvol_done (fs, subvol);
+    glfs_subvol_done(fs, subvol);
 
-        __GLFS_EXIT_FS;
+    __GLFS_EXIT_FS;
 
 invalid_fs:
-	return ret;
+    return ret;
 }
 
-GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_mknod, 3.4.0);
-
-
+GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_mkdir, 3.4.0)
 int
-pub_glfs_mkdir (struct glfs *fs, const char *path, mode_t mode)
-{
-	int              ret = -1;
-	xlator_t        *subvol = NULL;
-	loc_t            loc = {0, };
-	struct iatt      iatt = {0, };
-	uuid_t           gfid;
-	dict_t          *xattr_req = NULL;
-	int              reval = 0;
-
-        DECLARE_OLD_THIS;
-        __GLFS_ENTRY_VALIDATE_FS (fs, invalid_fs);
-
-	subvol = glfs_active_subvol (fs);
-	if (!subvol) {
-		ret = -1;
-		errno = EIO;
-		goto out;
-	}
-
-	xattr_req = dict_new ();
-	if (!xattr_req) {
-		ret = -1;
-		errno = ENOMEM;
-		goto out;
-	}
-
-	gf_uuid_generate (gfid);
-	ret = dict_set_static_bin (xattr_req, "gfid-req", gfid, 16);
-	if (ret) {
-		ret = -1;
-		errno = ENOMEM;
-		goto out;
-	}
+pub_glfs_mkdir(struct glfs *fs, const char *path, mode_t mode)
+{
+    int ret = -1;
+    xlator_t *subvol = NULL;
+    loc_t loc = {
+        0,
+    };
+    struct iatt iatt = {
+        0,
+    };
+    uuid_t gfid;
+    dict_t *xattr_req = NULL;
+    int reval = 0;
+
+    DECLARE_OLD_THIS;
+    __GLFS_ENTRY_VALIDATE_FS(fs, invalid_fs);
+
+    subvol = glfs_active_subvol(fs);
+    if (!subvol) {
+        ret = -1;
+        errno = EIO;
+        goto out;
+    }
+
+    xattr_req = dict_new();
+    if (!xattr_req) {
+        ret = -1;
+        errno = ENOMEM;
+        goto out;
+    }
+
+    gf_uuid_generate(gfid);
+    ret = dict_set_gfuuid(xattr_req, "gfid-req", gfid, true);
+    if (ret) {
+        ret = -1;
+        errno = ENOMEM;
+        goto out;
+    }
 retry:
-	ret = glfs_lresolve (fs, subvol, path, &loc, &iatt, reval);
+    ret = glfs_lresolve(fs, subvol, path, &loc, &iatt, reval);
 
-	ESTALE_RETRY (ret, errno, reval, &loc, retry);
+    ESTALE_RETRY(ret, errno, reval, &loc, retry);
 
-	if (loc.inode) {
-		errno = EEXIST;
-		ret = -1;
-		goto out;
-	}
+    if (loc.inode) {
+        errno = EEXIST;
+        ret = -1;
+        goto out;
+    }
 
-	if (ret == -1 && errno != ENOENT)
-		/* Any other type of error is fatal */
-		goto out;
+    if (ret == -1 && errno != ENOENT)
+        /* Any other type of error is fatal */
+        goto out;
 
-	if (ret == -1 && errno == ENOENT && !loc.parent)
-		/* The parent directory or an ancestor even
-		   higher does not exist
-		*/
-		goto out;
+    if (ret == -1 && errno == ENOENT && !loc.parent)
+        /* The parent directory or an ancestor even
+           higher does not exist
+        */
+        goto out;
 
-	/* ret == -1 && errno == ENOENT */
-	loc.inode = inode_new (loc.parent->table);
-	if (!loc.inode) {
-		ret = -1;
-		errno = ENOMEM;
-		goto out;
-	}
+    /* ret == -1 && errno == ENOENT */
+    loc.inode = inode_new(loc.parent->table);
+    if (!loc.inode) {
+        ret = -1;
+        errno = ENOMEM;
+        goto out;
+    }
 
-	ret = syncop_mkdir (subvol, &loc, mode, &iatt, xattr_req, NULL);
-        DECODE_SYNCOP_ERR (ret);
+    ret = syncop_mkdir(subvol, &loc, mode, &iatt, xattr_req, NULL);
+    DECODE_SYNCOP_ERR(ret);
 
-	ESTALE_RETRY (ret, errno, reval, &loc, retry);
+    ESTALE_RETRY(ret, errno, reval, &loc, retry);
 
-	if (ret == 0)
-		ret = glfs_loc_link (&loc, &iatt);
+    if (ret == 0)
+        ret = glfs_loc_link(&loc, &iatt);
 out:
-	loc_wipe (&loc);
+    loc_wipe(&loc);
 
-	if (xattr_req)
-		dict_unref (xattr_req);
+    if (xattr_req)
+        dict_unref(xattr_req);
 
-	glfs_subvol_done (fs, subvol);
+    glfs_subvol_done(fs, subvol);
 
-        __GLFS_EXIT_FS;
+    __GLFS_EXIT_FS;
 
 invalid_fs:
-	return ret;
+    return ret;
 }
 
-GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_mkdir, 3.4.0);
-
-
+GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_unlink, 3.4.0)
 int
-pub_glfs_unlink (struct glfs *fs, const char *path)
-{
-	int              ret = -1;
-	xlator_t        *subvol = NULL;
-	loc_t            loc = {0, };
-	struct iatt      iatt = {0, };
-	int              reval = 0;
-
-        DECLARE_OLD_THIS;
-        __GLFS_ENTRY_VALIDATE_FS (fs, invalid_fs);
-
-	subvol = glfs_active_subvol (fs);
-	if (!subvol) {
-		ret = -1;
-		errno = EIO;
-		goto out;
-	}
+pub_glfs_unlink(struct glfs *fs, const char *path)
+{
+    int ret = -1;
+    xlator_t *subvol = NULL;
+    loc_t loc = {
+        0,
+    };
+    struct iatt iatt = {
+        0,
+    };
+    int reval = 0;
+
+    DECLARE_OLD_THIS;
+    __GLFS_ENTRY_VALIDATE_FS(fs, invalid_fs);
+
+    subvol = glfs_active_subvol(fs);
+    if (!subvol) {
+        ret = -1;
+        errno = EIO;
+        goto out;
+    }
 retry:
-	ret = glfs_lresolve (fs, subvol, path, &loc, &iatt, reval);
+    ret = glfs_lresolve(fs, subvol, path, &loc, &iatt, reval);
 
-	ESTALE_RETRY (ret, errno, reval, &loc, retry);
+    ESTALE_RETRY(ret, errno, reval, &loc, retry);
 
-	if (ret)
-		goto out;
+    if (ret)
+        goto out;
 
-	if (iatt.ia_type == IA_IFDIR) {
-		ret = -1;
-		errno = EISDIR;
-		goto out;
-	}
+    if (iatt.ia_type == IA_IFDIR) {
+        ret = -1;
+        errno = EISDIR;
+        goto out;
+    }
 
-	ret = syncop_unlink (subvol, &loc, NULL, NULL);
-        DECODE_SYNCOP_ERR (ret);
+    /* TODO: Add leaseid */
+    ret = syncop_unlink(subvol, &loc, NULL, NULL);
+    DECODE_SYNCOP_ERR(ret);
 
-	ESTALE_RETRY (ret, errno, reval, &loc, retry);
+    ESTALE_RETRY(ret, errno, reval, &loc, retry);
 
-	if (ret == 0)
-		ret = glfs_loc_unlink (&loc);
+    if (ret == 0)
+        ret = glfs_loc_unlink(&loc);
 out:
-	loc_wipe (&loc);
+    loc_wipe(&loc);
 
-	glfs_subvol_done (fs, subvol);
+    glfs_subvol_done(fs, subvol);
 
-        __GLFS_EXIT_FS;
+    __GLFS_EXIT_FS;
 
 invalid_fs:
-	return ret;
+    return ret;
 }
 
-GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_unlink, 3.4.0);
-
-
+GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_rmdir, 3.4.0)
 int
-pub_glfs_rmdir (struct glfs *fs, const char *path)
-{
-	int              ret = -1;
-	xlator_t        *subvol = NULL;
-	loc_t            loc = {0, };
-	struct iatt      iatt = {0, };
-	int              reval = 0;
-
-        DECLARE_OLD_THIS;
-        __GLFS_ENTRY_VALIDATE_FS (fs, invalid_fs);
-
-	subvol = glfs_active_subvol (fs);
-	if (!subvol) {
-		ret = -1;
-		errno = EIO;
-		goto out;
-	}
+pub_glfs_rmdir(struct glfs *fs, const char *path)
+{
+    int ret = -1;
+    xlator_t *subvol = NULL;
+    loc_t loc = {
+        0,
+    };
+    struct iatt iatt = {
+        0,
+    };
+    int reval = 0;
+
+    DECLARE_OLD_THIS;
+    __GLFS_ENTRY_VALIDATE_FS(fs, invalid_fs);
+
+    subvol = glfs_active_subvol(fs);
+    if (!subvol) {
+        ret = -1;
+        errno = EIO;
+        goto out;
+    }
 retry:
-	ret = glfs_lresolve (fs, subvol, path, &loc, &iatt, reval);
+    ret = glfs_lresolve(fs, subvol, path, &loc, &iatt, reval);
 
-	ESTALE_RETRY (ret, errno, reval, &loc, retry);
+    ESTALE_RETRY(ret, errno, reval, &loc, retry);
 
-	if (ret)
-		goto out;
+    if (ret)
+        goto out;
 
-	if (iatt.ia_type != IA_IFDIR) {
-		ret = -1;
-		errno = ENOTDIR;
-		goto out;
-	}
+    if (iatt.ia_type != IA_IFDIR) {
+        ret = -1;
+        errno = ENOTDIR;
+        goto out;
+    }
 
-	ret = syncop_rmdir (subvol, &loc, 0, NULL, NULL);
-        DECODE_SYNCOP_ERR (ret);
+    ret = syncop_rmdir(subvol, &loc, 0, NULL, NULL);
+    DECODE_SYNCOP_ERR(ret);
 
-	ESTALE_RETRY (ret, errno, reval, &loc, retry);
+    ESTALE_RETRY(ret, errno, reval, &loc, retry);
 
-	if (ret == 0)
-		ret = glfs_loc_unlink (&loc);
+    if (ret == 0)
+        ret = glfs_loc_unlink(&loc);
 out:
-	loc_wipe (&loc);
+    loc_wipe(&loc);
 
-	glfs_subvol_done (fs, subvol);
+    glfs_subvol_done(fs, subvol);
 
-        __GLFS_EXIT_FS;
+    __GLFS_EXIT_FS;
 
 invalid_fs:
-	return ret;
+    return ret;
 }
 
-GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_rmdir, 3.4.0);
-
-
+GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_rename, 3.4.0)
 int
-pub_glfs_rename (struct glfs *fs, const char *oldpath, const char *newpath)
-{
-	int              ret = -1;
-	xlator_t        *subvol = NULL;
-	loc_t            oldloc = {0, };
-	loc_t            newloc = {0, };
-	struct iatt      oldiatt = {0, };
-	struct iatt      newiatt = {0, };
-	int              reval = 0;
-
-        DECLARE_OLD_THIS;
-        __GLFS_ENTRY_VALIDATE_FS (fs, invalid_fs);
-
-	subvol = glfs_active_subvol (fs);
-	if (!subvol) {
-		ret = -1;
-		errno = EIO;
-		goto out;
-	}
+pub_glfs_rename(struct glfs *fs, const char *oldpath, const char *newpath)
+{
+    int ret = -1;
+    xlator_t *subvol = NULL;
+    loc_t oldloc = {
+        0,
+    };
+    loc_t newloc = {
+        0,
+    };
+    struct iatt oldiatt = {
+        0,
+    };
+    struct iatt newiatt = {
+        0,
+    };
+    int reval = 0;
+
+    DECLARE_OLD_THIS;
+    __GLFS_ENTRY_VALIDATE_FS(fs, invalid_fs);
+
+    subvol = glfs_active_subvol(fs);
+    if (!subvol) {
+        ret = -1;
+        errno = EIO;
+        goto out;
+    }
 retry:
-	ret = glfs_lresolve (fs, subvol, oldpath, &oldloc, &oldiatt, reval);
+    ret = glfs_lresolve(fs, subvol, oldpath, &oldloc, &oldiatt, reval);
 
-	ESTALE_RETRY (ret, errno, reval, &oldloc, retry);
+    ESTALE_RETRY(ret, errno, reval, &oldloc, retry);
 
-	if (ret)
-		goto out;
+    if (ret)
+        goto out;
 retrynew:
-	ret = glfs_lresolve (fs, subvol, newpath, &newloc, &newiatt, reval);
-
-	ESTALE_RETRY (ret, errno, reval, &newloc, retrynew);
-
-	if (ret && errno != ENOENT && newloc.parent)
-		goto out;
-
-	if (newiatt.ia_type != IA_INVAL) {
-                if ((oldiatt.ia_type == IA_IFDIR) !=
-                    (newiatt.ia_type == IA_IFDIR)) {
-                       /* Either both old and new must be dirs,
-                        * or both must be non-dirs. Else, fail.
-                        */
-                       ret = -1;
-                       errno = EISDIR;
-                       goto out;
-                }
+    ret = glfs_lresolve(fs, subvol, newpath, &newloc, &newiatt, reval);
+
+    ESTALE_RETRY(ret, errno, reval, &newloc, retrynew);
+
+    if (ret && errno != ENOENT && newloc.parent)
+        goto out;
+
+    if (newiatt.ia_type != IA_INVAL) {
+        if ((oldiatt.ia_type == IA_IFDIR) != (newiatt.ia_type == IA_IFDIR)) {
+            /* Either both old and new must be dirs,
+             * or both must be non-dirs. Else, fail.
+             */
+            ret = -1;
+            errno = EISDIR;
+            goto out;
         }
+    }
 
-	/* TODO: check if new or old is a prefix of the other, and fail EINVAL */
+    /* TODO: - check if new or old is a prefix of the other, and fail EINVAL
+     *       - Add leaseid */
 
-	ret = syncop_rename (subvol, &oldloc, &newloc, NULL, NULL);
-        DECODE_SYNCOP_ERR (ret);
+    ret = syncop_rename(subvol, &oldloc, &newloc, NULL, NULL);
+    DECODE_SYNCOP_ERR(ret);
 
-	if (ret == -1 && errno == ESTALE) {
-		if (reval < DEFAULT_REVAL_COUNT) {
-			reval++;
-			loc_wipe (&oldloc);
-			loc_wipe (&newloc);
-			goto retry;
-		}
-	}
+    if (ret == -1 && errno == ESTALE) {
+        if (reval < DEFAULT_REVAL_COUNT) {
+            reval++;
+            loc_wipe(&oldloc);
+            loc_wipe(&newloc);
+            goto retry;
+        }
+    }
+
+    if (ret == 0) {
+        inode_rename(oldloc.parent->table, oldloc.parent, oldloc.name,
+                     newloc.parent, newloc.name, oldloc.inode, &oldiatt);
 
-	if (ret == 0)
-		inode_rename (oldloc.parent->table, oldloc.parent, oldloc.name,
-			      newloc.parent, newloc.name, oldloc.inode,
-			      &oldiatt);
+        if (newloc.inode && !inode_has_dentry(newloc.inode))
+            inode_forget(newloc.inode, 0);
+    }
 out:
-	loc_wipe (&oldloc);
-	loc_wipe (&newloc);
+    loc_wipe(&oldloc);
+    loc_wipe(&newloc);
 
-	glfs_subvol_done (fs, subvol);
+    glfs_subvol_done(fs, subvol);
 
-        __GLFS_EXIT_FS;
+    __GLFS_EXIT_FS;
 
 invalid_fs:
-	return ret;
+    return ret;
 }
 
-GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_rename, 3.4.0);
-
-
+GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_link, 3.4.0)
 int
-pub_glfs_link (struct glfs *fs, const char *oldpath, const char *newpath)
-{
-	int              ret = -1;
-	xlator_t        *subvol = NULL;
-	loc_t            oldloc = {0, };
-	loc_t            newloc = {0, };
-	struct iatt      oldiatt = {0, };
-	struct iatt      newiatt = {0, };
-	int              reval = 0;
-
-        DECLARE_OLD_THIS;
-        __GLFS_ENTRY_VALIDATE_FS (fs, invalid_fs);
-
-	subvol = glfs_active_subvol (fs);
-	if (!subvol) {
-		ret = -1;
-		errno = EIO;
-		goto out;
-	}
+pub_glfs_link(struct glfs *fs, const char *oldpath, const char *newpath)
+{
+    int ret = -1;
+    xlator_t *subvol = NULL;
+    loc_t oldloc = {
+        0,
+    };
+    loc_t newloc = {
+        0,
+    };
+    struct iatt oldiatt = {
+        0,
+    };
+    struct iatt newiatt = {
+        0,
+    };
+    int reval = 0;
+
+    DECLARE_OLD_THIS;
+    __GLFS_ENTRY_VALIDATE_FS(fs, invalid_fs);
+
+    subvol = glfs_active_subvol(fs);
+    if (!subvol) {
+        ret = -1;
+        errno = EIO;
+        goto out;
+    }
 retry:
-	ret = glfs_lresolve (fs, subvol, oldpath, &oldloc, &oldiatt, reval);
+    ret = glfs_lresolve(fs, subvol, oldpath, &oldloc, &oldiatt, reval);
 
-	ESTALE_RETRY (ret, errno, reval, &oldloc, retry);
+    ESTALE_RETRY(ret, errno, reval, &oldloc, retry);
 
-	if (ret)
-		goto out;
+    if (ret)
+        goto out;
 retrynew:
-	ret = glfs_lresolve (fs, subvol, newpath, &newloc, &newiatt, reval);
+    ret = glfs_lresolve(fs, subvol, newpath, &newloc, &newiatt, reval);
+
+    ESTALE_RETRY(ret, errno, reval, &newloc, retrynew);
+
+    if (ret == 0) {
+        ret = -1;
+        errno = EEXIST;
+        goto out;
+    }
+
+    if (oldiatt.ia_type == IA_IFDIR) {
+        ret = -1;
+        errno = EISDIR;
+        goto out;
+    }
+
+    /* Filling the inode of the hard link to be same as that of the
+       original file
+    */
+    if (newloc.inode) {
+        inode_unref(newloc.inode);
+        newloc.inode = NULL;
+    }
+    newloc.inode = inode_ref(oldloc.inode);
+
+    ret = syncop_link(subvol, &oldloc, &newloc, &newiatt, NULL, NULL);
+    DECODE_SYNCOP_ERR(ret);
+
+    if (ret == -1 && errno == ESTALE) {
+        loc_wipe(&oldloc);
+        loc_wipe(&newloc);
+        if (reval--)
+            goto retry;
+    }
+
+    if (ret == 0)
+        ret = glfs_loc_link(&newloc, &newiatt);
+out:
+    loc_wipe(&oldloc);
+    loc_wipe(&newloc);
 
-	ESTALE_RETRY (ret, errno, reval, &newloc, retrynew);
+    glfs_subvol_done(fs, subvol);
 
-	if (ret == 0) {
-		ret = -1;
-		errno = EEXIST;
-		goto out;
-	}
+    __GLFS_EXIT_FS;
 
-	if (oldiatt.ia_type == IA_IFDIR) {
-		ret = -1;
-		errno = EISDIR;
-		goto out;
-	}
+invalid_fs:
+    return ret;
+}
 
-        /* Filling the inode of the hard link to be same as that of the
-           original file
-        */
-	if (newloc.inode) {
-		inode_unref (newloc.inode);
-		newloc.inode = NULL;
-	}
-        newloc.inode = inode_ref (oldloc.inode);
-
-	ret = syncop_link (subvol, &oldloc, &newloc, &newiatt, NULL, NULL);
-        DECODE_SYNCOP_ERR (ret);
-
-	if (ret == -1 && errno == ESTALE) {
-		loc_wipe (&oldloc);
-		loc_wipe (&newloc);
-		if (reval--)
-			goto retry;
-	}
-
-	if (ret == 0)
-		ret = glfs_loc_link (&newloc, &newiatt);
-out:
-	loc_wipe (&oldloc);
-	loc_wipe (&newloc);
+GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_opendir, 3.4.0)
+struct glfs_fd *
+pub_glfs_opendir(struct glfs *fs, const char *path)
+{
+    int ret = -1;
+    struct glfs_fd *glfd = NULL;
+    xlator_t *subvol = NULL;
+    loc_t loc = {
+        0,
+    };
+    struct iatt iatt = {
+        0,
+    };
+    int reval = 0;
+
+    DECLARE_OLD_THIS;
+    __GLFS_ENTRY_VALIDATE_FS(fs, invalid_fs);
+
+    subvol = glfs_active_subvol(fs);
+    if (!subvol) {
+        ret = -1;
+        errno = EIO;
+        goto out;
+    }
+
+    glfd = glfs_fd_new(fs);
+    if (!glfd)
+        goto out;
+
+    INIT_LIST_HEAD(&glfd->entries);
+retry:
+    ret = glfs_resolve(fs, subvol, path, &loc, &iatt, reval);
 
-	glfs_subvol_done (fs, subvol);
+    ESTALE_RETRY(ret, errno, reval, &loc, retry);
 
-        __GLFS_EXIT_FS;
+    if (ret)
+        goto out;
 
-invalid_fs:
-	return ret;
-}
+    if (!IA_ISDIR(iatt.ia_type)) {
+        ret = -1;
+        errno = ENOTDIR;
+        goto out;
+    }
+
+    if (glfd->fd) {
+        /* Retry. Safe to touch glfd->fd as we
+           still have not glfs_fd_bind() yet.
+        */
+        fd_unref(glfd->fd);
+        glfd->fd = NULL;
+    }
 
-GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_link, 3.4.0);
+    glfd->fd = fd_create(loc.inode, getpid());
+    if (!glfd->fd) {
+        ret = -1;
+        errno = ENOMEM;
+        goto out;
+    }
 
+    ret = syncop_opendir(subvol, &loc, glfd->fd, NULL, NULL);
+    DECODE_SYNCOP_ERR(ret);
 
-struct glfs_fd *
-pub_glfs_opendir (struct glfs *fs, const char *path)
-{
-	int              ret = -1;
-	struct glfs_fd  *glfd = NULL;
-	xlator_t        *subvol = NULL;
-	loc_t            loc = {0, };
-	struct iatt      iatt = {0, };
-	int              reval = 0;
-
-        DECLARE_OLD_THIS;
-        __GLFS_ENTRY_VALIDATE_FS (fs, invalid_fs);
-
-	subvol = glfs_active_subvol (fs);
-	if (!subvol) {
-		ret = -1;
-		errno = EIO;
-		goto out;
-	}
-
-	glfd = glfs_fd_new (fs);
-	if (!glfd)
-		goto out;
-
-	INIT_LIST_HEAD (&glfd->entries);
-retry:
-	ret = glfs_resolve (fs, subvol, path, &loc, &iatt, reval);
-
-	ESTALE_RETRY (ret, errno, reval, &loc, retry);
-
-	if (ret)
-		goto out;
-
-	if (!IA_ISDIR (iatt.ia_type)) {
-		ret = -1;
-		errno = ENOTDIR;
-		goto out;
-	}
-
-	if (glfd->fd) {
-		/* Retry. Safe to touch glfd->fd as we
-		   still have not glfs_fd_bind() yet.
-		*/
-		fd_unref (glfd->fd);
-		glfd->fd = NULL;
-	}
-
-	glfd->fd = fd_create (loc.inode, getpid());
-	if (!glfd->fd) {
-		ret = -1;
-		errno = ENOMEM;
-		goto out;
-	}
-
-	ret = syncop_opendir (subvol, &loc, glfd->fd, NULL, NULL);
-        DECODE_SYNCOP_ERR (ret);
-
-	ESTALE_RETRY (ret, errno, reval, &loc, retry);
+    ESTALE_RETRY(ret, errno, reval, &loc, retry);
 out:
-	loc_wipe (&loc);
+    loc_wipe(&loc);
 
-	if (ret && glfd) {
-		glfs_fd_destroy (glfd);
-		glfd = NULL;
-	} else if (glfd) {
-		fd_bind (glfd->fd);
-		glfs_fd_bind (glfd);
-	}
+    if (ret && glfd) {
+        GF_REF_PUT(glfd);
+        glfd = NULL;
+    } else if (glfd) {
+        glfd_set_state_bind(glfd);
+    }
 
-	glfs_subvol_done (fs, subvol);
+    glfs_subvol_done(fs, subvol);
 
-        __GLFS_EXIT_FS;
+    __GLFS_EXIT_FS;
 
 invalid_fs:
-	return glfd;
+    return glfd;
 }
 
-GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_opendir, 3.4.0);
-
-
+GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_closedir, 3.4.0)
 int
-pub_glfs_closedir (struct glfs_fd *glfd)
+pub_glfs_closedir(struct glfs_fd *glfd)
 {
-        int ret = -1;
+    int ret = -1;
 
-        DECLARE_OLD_THIS;
-	__GLFS_ENTRY_VALIDATE_FD (glfd, invalid_fs);
+    DECLARE_OLD_THIS;
+    __GLFS_ENTRY_VALIDATE_FD(glfd, invalid_fs);
 
-	gf_dirent_free (list_entry (&glfd->entries, gf_dirent_t, list));
+    gf_dirent_free(list_entry(&glfd->entries, gf_dirent_t, list));
 
-	glfs_fd_destroy (glfd);
+    glfs_mark_glfd_for_deletion(glfd);
 
-        __GLFS_EXIT_FS;
+    __GLFS_EXIT_FS;
 
-        ret = 0;
+    ret = 0;
 
 invalid_fs:
-	return ret;
+    return ret;
 }
 
-GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_closedir, 3.4.0);
-
-
+GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_telldir, 3.4.0)
 long
-pub_glfs_telldir (struct glfs_fd *fd)
+pub_glfs_telldir(struct glfs_fd *fd)
 {
-	return fd->offset;
-}
-
-GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_telldir, 3.4.0);
+    if (fd == NULL) {
+        errno = EBADF;
+        return -1;
+    }
 
+    return fd->offset;
+}
 
+GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_seekdir, 3.4.0)
 void
-pub_glfs_seekdir (struct glfs_fd *fd, long offset)
+pub_glfs_seekdir(struct glfs_fd *fd, long offset)
 {
-	gf_dirent_t *entry = NULL;
-	gf_dirent_t *tmp = NULL;
-
-	if (fd->offset == offset)
-		return;
+    gf_dirent_t *entry = NULL;
+    gf_dirent_t *tmp = NULL;
 
-	fd->offset = offset;
-	fd->next = NULL;
+    if (fd == NULL) {
+        errno = EBADF;
+        return;
+    }
 
-	list_for_each_entry_safe (entry, tmp, &fd->entries, list) {
-		if (entry->d_off != offset)
-			continue;
+    if (fd->offset == offset)
+        return;
 
-		if (&tmp->list != &fd->entries) {
-			/* found! */
-			fd->next = tmp;
-			return;
-		}
-	}
-	/* could not find entry at requested offset in the cache.
-	   next readdir_r() will result in glfd_entry_refresh()
-	*/
-}
+    fd->offset = offset;
+    fd->next = NULL;
 
-GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_seekdir, 3.4.0);
+    list_for_each_entry_safe(entry, tmp, &fd->entries, list)
+    {
+        if (entry->d_off != offset)
+            continue;
 
+        if (&tmp->list != &fd->entries) {
+            /* found! */
+            fd->next = tmp;
+            return;
+        }
+    }
+    /* could not find entry at requested offset in the cache.
+       next readdir_r() will result in glfd_entry_refresh()
+    */
+}
 
-int
-pub_glfs_discard_async (struct glfs_fd *glfd, off_t offset, size_t len,
-                        glfs_io_cbk fn, void *data)
+static int
+glfs_discard_async_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                       int32_t op_ret, int32_t op_errno,
+                       struct iatt *preop_stbuf, struct iatt *postop_stbuf,
+                       dict_t *xdata)
 {
-	struct glfs_io *gio = NULL;
-	int             ret = -1;
-
-        DECLARE_OLD_THIS;
-        __GLFS_ENTRY_VALIDATE_FD (glfd, invalid_fs);
-
-	gio = GF_CALLOC (1, sizeof (*gio), glfs_mt_glfs_io_t);
-	if (!gio) {
-		errno = ENOMEM;
-		goto out;
-	}
-
-	gio->op     = GF_FOP_DISCARD;
-	gio->glfd   = glfd;
-	gio->offset = offset;
-	gio->count  = len;
-	gio->fn     = fn;
-	gio->data   = data;
+    glfs_io_async_cbk(op_ret, op_errno, frame, cookie, NULL, 0, preop_stbuf,
+                      postop_stbuf);
 
-	ret = synctask_new (pub_glfs_from_glfd (glfd)->ctx->env,
-			    glfs_io_async_task, glfs_io_async_cbk,
-			    NULL, gio);
-
-	if (ret) {
-		GF_FREE (gio->iov);
-		GF_FREE (gio);
-	}
+    return 0;
+}
 
+static int
+glfs_discard_async_common(struct glfs_fd *glfd, off_t offset, size_t len,
+                          gf_boolean_t oldcb, glfs_io_cbk fn, void *data)
+{
+    struct glfs_io *gio = NULL;
+    int ret = -1;
+    call_frame_t *frame = NULL;
+    xlator_t *subvol = NULL;
+    fd_t *fd = NULL;
+    dict_t *fop_attr = NULL;
+
+    DECLARE_OLD_THIS;
+    __GLFS_ENTRY_VALIDATE_FD(glfd, invalid_fs);
+
+    /* Need to take explicit ref so that the fd
+     * is not destroyed before the fop is complete
+     */
+    GF_REF_GET(glfd);
+
+    subvol = glfs_active_subvol(glfd->fs);
+    if (!subvol) {
+        errno = EIO;
+        goto out;
+    }
+
+    fd = glfs_resolve_fd(glfd->fs, subvol, glfd);
+    if (!fd) {
+        errno = EBADFD;
+        goto out;
+    }
+
+    frame = syncop_create_frame(THIS);
+    if (!frame) {
+        errno = ENOMEM;
+        goto out;
+    }
+
+    gio = GF_CALLOC(1, sizeof(*gio), glfs_mt_glfs_io_t);
+    if (!gio) {
+        errno = ENOMEM;
+        goto out;
+    }
+
+    gio->op = GF_FOP_DISCARD;
+    gio->glfd = glfd;
+    gio->offset = offset;
+    gio->count = len;
+    gio->oldcb = oldcb;
+    gio->fn = fn;
+    gio->data = data;
+
+    frame->local = gio;
+    ret = get_fop_attr_thrd_key(&fop_attr);
+    if (ret)
+        gf_msg_debug("gfapi", 0, "Getting leaseid from thread failed");
+
+    STACK_WIND_COOKIE(frame, glfs_discard_async_cbk, subvol, subvol,
+                      subvol->fops->discard, fd, offset, len, fop_attr);
+
+    ret = 0;
 out:
-        __GLFS_EXIT_FS;
+    if (ret) {
+        if (fd)
+            fd_unref(fd);
+        if (glfd)
+            GF_REF_PUT(glfd);
+        GF_FREE(gio);
+        if (frame)
+            STACK_DESTROY(frame->root);
+        glfs_subvol_done(glfd->fs, subvol);
+    }
+
+    __GLFS_EXIT_FS;
 
 invalid_fs:
-	return ret;
+    return ret;
 }
 
-GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_discard_async, 3.5.0);
-
-
+GFAPI_SYMVER_PUBLIC(glfs_discard_async35, glfs_discard_async, 3.5.0)
 int
-pub_glfs_zerofill_async (struct glfs_fd *glfd, off_t offset, off_t len,
-                         glfs_io_cbk fn, void *data)
+pub_glfs_discard_async35(struct glfs_fd *glfd, off_t offset, size_t len,
+                         glfs_io_cbk34 fn, void *data)
 {
-        struct glfs_io *gio  = NULL;
-        int             ret  = -1;
-
-        DECLARE_OLD_THIS;
-        __GLFS_ENTRY_VALIDATE_FD (glfd, invalid_fs);
-
-        gio = GF_CALLOC (1, sizeof (*gio), glfs_mt_glfs_io_t);
-        if (!gio) {
-                errno = ENOMEM;
-                goto out;
-        }
+    return glfs_discard_async_common(glfd, offset, len, _gf_true, (void *)fn,
+                                     data);
+}
 
-        gio->op     = GF_FOP_ZEROFILL;
-        gio->glfd   = glfd;
-        gio->offset = offset;
-        gio->count  = len;
-        gio->fn     = fn;
-        gio->data   = data;
+GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_discard_async, 6.0)
+int
+pub_glfs_discard_async(struct glfs_fd *glfd, off_t offset, size_t len,
+                       glfs_io_cbk fn, void *data)
+{
+    return glfs_discard_async_common(glfd, offset, len, _gf_false, fn, data);
+}
 
-        ret = synctask_new (pub_glfs_from_glfd (glfd)->ctx->env,
-                            glfs_io_async_task, glfs_io_async_cbk,
-                            NULL, gio);
+static int
+glfs_zerofill_async_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                        int32_t op_ret, int32_t op_errno,
+                        struct iatt *preop_stbuf, struct iatt *postop_stbuf,
+                        dict_t *xdata)
+{
+    glfs_io_async_cbk(op_ret, op_errno, frame, cookie, NULL, 0, preop_stbuf,
+                      postop_stbuf);
 
-        if (ret) {
-                GF_FREE (gio->iov);
-                GF_FREE (gio);
-        }
+    return 0;
+}
 
+static int
+glfs_zerofill_async_common(struct glfs_fd *glfd, off_t offset, off_t len,
+                           gf_boolean_t oldcb, glfs_io_cbk fn, void *data)
+{
+    struct glfs_io *gio = NULL;
+    int ret = -1;
+    call_frame_t *frame = NULL;
+    xlator_t *subvol = NULL;
+    fd_t *fd = NULL;
+    dict_t *fop_attr = NULL;
+
+    DECLARE_OLD_THIS;
+    __GLFS_ENTRY_VALIDATE_FD(glfd, invalid_fs);
+
+    /* Need to take explicit ref so that the fd
+     * is not destroyed before the fop is complete
+     */
+    GF_REF_GET(glfd);
+
+    subvol = glfs_active_subvol(glfd->fs);
+    if (!subvol) {
+        errno = EIO;
+        goto out;
+    }
+
+    fd = glfs_resolve_fd(glfd->fs, subvol, glfd);
+    if (!fd) {
+        errno = EBADFD;
+        goto out;
+    }
+
+    frame = syncop_create_frame(THIS);
+    if (!frame) {
+        errno = ENOMEM;
+        goto out;
+    }
+
+    gio = GF_CALLOC(1, sizeof(*gio), glfs_mt_glfs_io_t);
+    if (!gio) {
+        errno = ENOMEM;
+        goto out;
+    }
+
+    gio->op = GF_FOP_ZEROFILL;
+    gio->glfd = glfd;
+    gio->offset = offset;
+    gio->count = len;
+    gio->oldcb = oldcb;
+    gio->fn = fn;
+    gio->data = data;
+
+    frame->local = gio;
+
+    ret = get_fop_attr_thrd_key(&fop_attr);
+    if (ret)
+        gf_msg_debug("gfapi", 0, "Getting leaseid from thread failed");
+
+    STACK_WIND_COOKIE(frame, glfs_zerofill_async_cbk, subvol, subvol,
+                      subvol->fops->zerofill, fd, offset, len, fop_attr);
+    ret = 0;
 out:
-        __GLFS_EXIT_FS;
+    if (ret) {
+        if (fd)
+            fd_unref(fd);
+        if (glfd)
+            GF_REF_PUT(glfd);
+        GF_FREE(gio);
+        if (frame)
+            STACK_DESTROY(frame->root);
+        glfs_subvol_done(glfd->fs, subvol);
+    }
+    if (fop_attr)
+        dict_unref(fop_attr);
+
+    __GLFS_EXIT_FS;
 
 invalid_fs:
-        return ret;
+    return ret;
 }
 
-GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_zerofill_async, 3.5.0);
+GFAPI_SYMVER_PUBLIC(glfs_zerofill_async35, glfs_zerofill_async, 3.5.0)
+int
+pub_glfs_zerofill_async35(struct glfs_fd *glfd, off_t offset, off_t len,
+                          glfs_io_cbk34 fn, void *data)
+{
+    return glfs_zerofill_async_common(glfd, offset, len, _gf_true, (void *)fn,
+                                      data);
+}
 
+GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_zerofill_async, 6.0)
+int
+pub_glfs_zerofill_async(struct glfs_fd *glfd, off_t offset, off_t len,
+                        glfs_io_cbk fn, void *data)
+{
+    return glfs_zerofill_async_common(glfd, offset, len, _gf_false, fn, data);
+}
 
 void
-gf_dirent_to_dirent (gf_dirent_t *gf_dirent, struct dirent *dirent)
+gf_dirent_to_dirent(gf_dirent_t *gf_dirent, struct dirent *dirent)
 {
-	dirent->d_ino = gf_dirent->d_ino;
+    dirent->d_ino = gf_dirent->d_ino;
 
 #ifdef _DIRENT_HAVE_D_OFF
-	dirent->d_off = gf_dirent->d_off;
+    dirent->d_off = gf_dirent->d_off;
 #endif
 
 #ifdef _DIRENT_HAVE_D_TYPE
-	dirent->d_type = gf_dirent->d_type;
+    dirent->d_type = gf_dirent->d_type;
 #endif
 
 #ifdef _DIRENT_HAVE_D_NAMLEN
-	dirent->d_namlen = strlen (gf_dirent->d_name);
+    dirent->d_namlen = strlen(gf_dirent->d_name);
 #endif
 
-	strncpy (dirent->d_name, gf_dirent->d_name, NAME_MAX);
-	dirent->d_name[NAME_MAX] = 0;
+    snprintf(dirent->d_name, NAME_MAX + 1, "%s", gf_dirent->d_name);
 }
 
-
 int
-glfd_entry_refresh (struct glfs_fd *glfd, int plus)
-{
-	xlator_t        *subvol = NULL;
-	gf_dirent_t      entries;
-	gf_dirent_t      old;
-        gf_dirent_t     *entry = NULL;
-	int              ret = -1;
-	fd_t            *fd = NULL;
-
-	subvol = glfs_active_subvol (glfd->fs);
-	if (!subvol) {
-		ret = -1;
-		errno = EIO;
-		goto out;
-	}
-
-	fd = glfs_resolve_fd (glfd->fs, subvol, glfd);
-	if (!fd) {
-		ret = -1;
-		errno = EBADFD;
-		goto out;
-	}
-
-	if (fd->inode->ia_type != IA_IFDIR) {
-		ret = -1;
-		errno = EBADF;
-		goto out;
-	}
-
-	INIT_LIST_HEAD (&entries.list);
-	INIT_LIST_HEAD (&old.list);
-
-	if (plus)
-		ret = syncop_readdirp (subvol, fd, 131072, glfd->offset,
-				       &entries, NULL, NULL);
-	else
-		ret = syncop_readdir (subvol, fd, 131072, glfd->offset,
-				      &entries, NULL, NULL);
-        DECODE_SYNCOP_ERR (ret);
-	if (ret >= 0) {
-		if (plus) {
-                        /**
-                         * Set inode_needs_lookup flag before linking the
-                         * inode. Doing it later post linkage might lead
-                         * to a race where a fop comes after inode link
-                         * but before setting need_lookup flag.
-                         */
-                        list_for_each_entry (entry, &glfd->entries, list) {
-                                if (entry->inode)
-                                        inode_set_need_lookup (entry->inode, THIS);
-                        }
-
-			gf_link_inodes_from_dirent (THIS, fd->inode, &entries);
+glfd_entry_refresh(struct glfs_fd *glfd, int plus)
+{
+    xlator_t *subvol = NULL;
+    gf_dirent_t entries;
+    gf_dirent_t old;
+    gf_dirent_t *entry = NULL;
+    int ret = -1;
+    fd_t *fd = NULL;
+
+    subvol = glfs_active_subvol(glfd->fs);
+    if (!subvol) {
+        ret = -1;
+        errno = EIO;
+        goto out;
+    }
+
+    fd = glfs_resolve_fd(glfd->fs, subvol, glfd);
+    if (!fd) {
+        ret = -1;
+        errno = EBADFD;
+        goto out;
+    }
+
+    if (fd->inode->ia_type != IA_IFDIR) {
+        ret = -1;
+        errno = EBADF;
+        goto out;
+    }
+
+    INIT_LIST_HEAD(&entries.list);
+    INIT_LIST_HEAD(&old.list);
+
+    if (plus)
+        ret = syncop_readdirp(subvol, fd, 131072, glfd->offset, &entries, NULL,
+                              NULL);
+    else
+        ret = syncop_readdir(subvol, fd, 131072, glfd->offset, &entries, NULL,
+                             NULL);
+    DECODE_SYNCOP_ERR(ret);
+    if (ret >= 0) {
+        if (plus) {
+            list_for_each_entry(entry, &entries.list, list)
+            {
+                if ((!entry->inode && (!IA_ISDIR(entry->d_stat.ia_type))) ||
+                    ((entry->d_stat.ia_ctime == 0) &&
+                     strcmp(entry->d_name, ".") &&
+                     strcmp(entry->d_name, ".."))) {
+                    /* entry->inode for directories will be
+                     * always set to null to force a lookup
+                     * on the dentry. Hence to not degrade
+                     * readdir performance, we skip lookups
+                     * for directory entries. Also we will have
+                     * proper stat if directory present on
+                     * hashed subvolume.
+                     *
+                     * In addition, if the stat is invalid, force
+                     * lookup to fetch proper stat.
+                     */
+                    gf_fill_iatt_for_dirent(entry, fd->inode, subvol);
                 }
+            }
 
-		list_splice_init (&glfd->entries, &old.list);
-		list_splice_init (&entries.list, &glfd->entries);
+            gf_link_inodes_from_dirent(THIS, fd->inode, &entries);
+        }
 
-		/* spurious errno is dangerous for glfd_entry_next() */
-		errno = 0;
-	}
+        list_splice_init(&glfd->entries, &old.list);
+        list_splice_init(&entries.list, &glfd->entries);
 
+        /* spurious errno is dangerous for glfd_entry_next() */
+        errno = 0;
+    }
 
-	if (ret > 0)
-		glfd->next = list_entry (glfd->entries.next, gf_dirent_t, list);
+    if (ret > 0)
+        glfd->next = list_entry(glfd->entries.next, gf_dirent_t, list);
 
-	gf_dirent_free (&old);
+    gf_dirent_free(&old);
 out:
-	if (fd)
-		fd_unref (fd);
+    if (fd)
+        fd_unref(fd);
 
-	glfs_subvol_done (glfd->fs, subvol);
+    glfs_subvol_done(glfd->fs, subvol);
 
-	return ret;
+    return ret;
 }
 
-
 gf_dirent_t *
-glfd_entry_next (struct glfs_fd *glfd, int plus)
+glfd_entry_next(struct glfs_fd *glfd, int plus)
 {
-	gf_dirent_t     *entry = NULL;
-	int              ret = -1;
+    gf_dirent_t *entry = NULL;
+    int ret = -1;
 
-	if (!glfd->offset || !glfd->next) {
-		ret = glfd_entry_refresh (glfd, plus);
-		if (ret < 0)
-			return NULL;
-	}
+    if (!glfd->offset || !glfd->next) {
+        ret = glfd_entry_refresh(glfd, plus);
+        if (ret < 0)
+            return NULL;
+    }
 
-	entry = glfd->next;
-	if (!entry)
-		return NULL;
+    entry = glfd->next;
+    if (!entry)
+        return NULL;
 
-	if (&entry->next->list == &glfd->entries)
-		glfd->next = NULL;
-	else
-		glfd->next = entry->next;
+    if (&entry->next->list == &glfd->entries)
+        glfd->next = NULL;
+    else
+        glfd->next = entry->next;
 
-	glfd->offset = entry->d_off;
+    glfd->offset = entry->d_off;
 
-	return entry;
+    return entry;
 }
 
-
-static struct dirent *
-glfs_readdirbuf_get (struct glfs_fd *glfd)
+struct dirent *
+glfs_readdirbuf_get(struct glfs_fd *glfd)
 {
-        struct dirent *buf = NULL;
-
-        LOCK (&glfd->fd->lock);
-        {
-                buf = glfd->readdirbuf;
-                if (buf) {
-                        memset (buf, 0, READDIRBUF_SIZE);
-                        goto unlock;
-                }
-
-                buf = GF_CALLOC (1, READDIRBUF_SIZE, glfs_mt_readdirbuf_t);
-                if (!buf) {
-                        errno = ENOMEM;
-                        goto unlock;
-                }
+    struct dirent *buf = NULL;
+
+    LOCK(&glfd->fd->lock);
+    {
+        buf = glfd->readdirbuf;
+        if (buf) {
+            memset(buf, 0, READDIRBUF_SIZE);
+            goto unlock;
+        }
 
-                glfd->readdirbuf = buf;
+        buf = GF_CALLOC(1, READDIRBUF_SIZE, glfs_mt_readdirbuf_t);
+        if (!buf) {
+            errno = ENOMEM;
+            goto unlock;
         }
+
+        glfd->readdirbuf = buf;
+    }
 unlock:
-        UNLOCK (&glfd->fd->lock);
+    UNLOCK(&glfd->fd->lock);
 
-        return buf;
+    return buf;
 }
 
-
+GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_readdirplus_r, 3.4.0)
 int
-pub_glfs_readdirplus_r (struct glfs_fd *glfd, struct stat *stat,
-                        struct dirent *ext, struct dirent **res)
+pub_glfs_readdirplus_r(struct glfs_fd *glfd, struct stat *stat,
+                       struct dirent *ext, struct dirent **res)
 {
-	int              ret = 0;
-	gf_dirent_t     *entry = NULL;
-	struct dirent   *buf = NULL;
+    int ret = 0;
+    gf_dirent_t *entry = NULL;
+    struct dirent *buf = NULL;
 
-        DECLARE_OLD_THIS;
-	__GLFS_ENTRY_VALIDATE_FD (glfd, invalid_fs);
+    DECLARE_OLD_THIS;
+    __GLFS_ENTRY_VALIDATE_FD(glfd, invalid_fs);
 
-	errno = 0;
+    GF_REF_GET(glfd);
 
-	if (ext)
-		buf = ext;
-	else
-		buf = glfs_readdirbuf_get (glfd);
+    errno = 0;
 
-	if (!buf) {
-		errno = ENOMEM;
-                ret = -1;
-                goto out;
-	}
+    if (ext)
+        buf = ext;
+    else
+        buf = glfs_readdirbuf_get(glfd);
+
+    if (!buf) {
+        errno = ENOMEM;
+        ret = -1;
+        goto out;
+    }
 
-	entry = glfd_entry_next (glfd, !!stat);
-	if (errno)
-		ret = -1;
+    entry = glfd_entry_next(glfd, !!stat);
+    if (errno)
+        ret = -1;
 
-	if (res) {
-		if (entry)
-			*res = buf;
-		else
-			*res = NULL;
-	}
+    if (res) {
+        if (entry)
+            *res = buf;
+        else
+            *res = NULL;
+    }
 
-	if (entry) {
-		gf_dirent_to_dirent (entry, buf);
-		if (stat)
-			glfs_iatt_to_stat (glfd->fs, &entry->d_stat, stat);
-	}
+    if (entry) {
+        gf_dirent_to_dirent(entry, buf);
+        if (stat)
+            glfs_iatt_to_stat(glfd->fs, &entry->d_stat, stat);
+    }
 
 out:
-        __GLFS_EXIT_FS;
+    if (glfd)
+        GF_REF_PUT(glfd);
 
-	return ret;
+    __GLFS_EXIT_FS;
+
+    return ret;
 
 invalid_fs:
-        return -1;
+    return -1;
 }
 
-GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_readdirplus_r, 3.4.0);
-
-
+GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_readdir_r, 3.4.0)
 int
-pub_glfs_readdir_r (struct glfs_fd *glfd, struct dirent *buf,
-                    struct dirent **res)
+pub_glfs_readdir_r(struct glfs_fd *glfd, struct dirent *buf,
+                   struct dirent **res)
 {
-	return pub_glfs_readdirplus_r (glfd, 0, buf, res);
+    return pub_glfs_readdirplus_r(glfd, 0, buf, res);
 }
 
-GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_readdir_r, 3.4.0);
-
-
+GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_readdirplus, 3.5.0)
 struct dirent *
-pub_glfs_readdirplus (struct glfs_fd *glfd, struct stat *stat)
+pub_glfs_readdirplus(struct glfs_fd *glfd, struct stat *stat)
 {
-        struct dirent *res = NULL;
-        int ret = -1;
+    struct dirent *res = NULL;
+    int ret = -1;
 
-        ret = pub_glfs_readdirplus_r (glfd, stat, NULL, &res);
-        if (ret)
-                return NULL;
+    ret = pub_glfs_readdirplus_r(glfd, stat, NULL, &res);
+    if (ret)
+        return NULL;
 
-        return res;
+    return res;
 }
 
-GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_readdirplus, 3.5.0);
-
-
+GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_readdir, 3.5.0)
 struct dirent *
-pub_glfs_readdir (struct glfs_fd *glfd)
+pub_glfs_readdir(struct glfs_fd *glfd)
 {
-        return pub_glfs_readdirplus (glfd, NULL);
+    return pub_glfs_readdirplus(glfd, NULL);
 }
 
-GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_readdir, 3.5.0);
-
-
+GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_statvfs, 3.4.0)
 int
-pub_glfs_statvfs (struct glfs *fs, const char *path, struct statvfs *buf)
-{
-	int              ret = -1;
-	xlator_t        *subvol = NULL;
-	loc_t            loc = {0, };
-	struct iatt      iatt = {0, };
-	int              reval = 0;
-
-        DECLARE_OLD_THIS;
-        __GLFS_ENTRY_VALIDATE_FS (fs, invalid_fs);
-
-	subvol = glfs_active_subvol (fs);
-	if (!subvol) {
-		ret = -1;
-		errno = EIO;
-		goto out;
-	}
+pub_glfs_statvfs(struct glfs *fs, const char *path, struct statvfs *buf)
+{
+    int ret = -1;
+    xlator_t *subvol = NULL;
+    loc_t loc = {
+        0,
+    };
+    struct iatt iatt = {
+        0,
+    };
+    int reval = 0;
+
+    DECLARE_OLD_THIS;
+    __GLFS_ENTRY_VALIDATE_FS(fs, invalid_fs);
+
+    subvol = glfs_active_subvol(fs);
+    if (!subvol) {
+        ret = -1;
+        errno = EIO;
+        goto out;
+    }
 retry:
-	ret = glfs_resolve (fs, subvol, path, &loc, &iatt, reval);
+    ret = glfs_resolve(fs, subvol, path, &loc, &iatt, reval);
 
-	ESTALE_RETRY (ret, errno, reval, &loc, retry);
+    ESTALE_RETRY(ret, errno, reval, &loc, retry);
 
-	if (ret)
-		goto out;
+    if (ret)
+        goto out;
 
-	ret = syncop_statfs (subvol, &loc, buf, NULL, NULL);
-        DECODE_SYNCOP_ERR (ret);
+    ret = syncop_statfs(subvol, &loc, buf, NULL, NULL);
+    DECODE_SYNCOP_ERR(ret);
 
-	ESTALE_RETRY (ret, errno, reval, &loc, retry);
+    ESTALE_RETRY(ret, errno, reval, &loc, retry);
 out:
-	loc_wipe (&loc);
+    loc_wipe(&loc);
 
-	glfs_subvol_done (fs, subvol);
+    glfs_subvol_done(fs, subvol);
 
-        __GLFS_EXIT_FS;
+    __GLFS_EXIT_FS;
 
 invalid_fs:
-	return ret;
+    return ret;
 }
 
-GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_statvfs, 3.4.0);
-
-
+GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_setattr, 6.0)
 int
-glfs_setattr (struct glfs *fs, const char *path, struct iatt *iatt,
-	      int valid, int follow)
-{
-	int              ret = -1;
-	xlator_t        *subvol = NULL;
-	loc_t            loc = {0, };
-	struct iatt      riatt = {0, };
-	int              reval = 0;
-
-        DECLARE_OLD_THIS;
-        __GLFS_ENTRY_VALIDATE_FS (fs, invalid_fs);
-
-	subvol = glfs_active_subvol (fs);
-	if (!subvol) {
-		ret = -1;
-		errno = EIO;
-		goto out;
-	}
+pub_glfs_setattr(struct glfs *fs, const char *path, struct glfs_stat *stat,
+                 int follow)
+{
+    int ret = -1;
+    int glvalid;
+    xlator_t *subvol = NULL;
+    loc_t loc = {
+        0,
+    };
+    struct iatt riatt = {
+        0,
+    };
+    struct iatt iatt = {
+        0,
+    };
+    int reval = 0;
+
+    DECLARE_OLD_THIS;
+    __GLFS_ENTRY_VALIDATE_FS(fs, invalid_fs);
+
+    GF_VALIDATE_OR_GOTO("glfs_setattr", stat, out);
+
+    subvol = glfs_active_subvol(fs);
+    if (!subvol) {
+        ret = -1;
+        errno = EIO;
+        goto out;
+    }
 retry:
-	if (follow)
-		ret = glfs_resolve (fs, subvol, path, &loc, &riatt, reval);
-	else
-		ret = glfs_lresolve (fs, subvol, path, &loc, &riatt, reval);
+    if (follow)
+        ret = glfs_resolve(fs, subvol, path, &loc, &riatt, reval);
+    else
+        ret = glfs_lresolve(fs, subvol, path, &loc, &riatt, reval);
+
+    ESTALE_RETRY(ret, errno, reval, &loc, retry);
 
-	ESTALE_RETRY (ret, errno, reval, &loc, retry);
+    if (ret)
+        goto out;
 
-	if (ret)
-		goto out;
+    glfs_iatt_from_statx(&iatt, stat);
+    glfsflags_from_gfapiflags(stat, &glvalid);
 
-	ret = syncop_setattr (subvol, &loc, iatt, valid, 0, 0, NULL, NULL);
-        DECODE_SYNCOP_ERR (ret);
+    /* TODO : Add leaseid */
+    ret = syncop_setattr(subvol, &loc, &iatt, glvalid, 0, 0, NULL, NULL);
+    DECODE_SYNCOP_ERR(ret);
 
-	ESTALE_RETRY (ret, errno, reval, &loc, retry);
+    ESTALE_RETRY(ret, errno, reval, &loc, retry);
 out:
-	loc_wipe (&loc);
+    loc_wipe(&loc);
 
-	glfs_subvol_done (fs, subvol);
+    glfs_subvol_done(fs, subvol);
 
-        __GLFS_EXIT_FS;
+    __GLFS_EXIT_FS;
 
 invalid_fs:
-	return ret;
+    return ret;
 }
 
-
+GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_fsetattr, 6.0)
 int
-glfs_fsetattr (struct glfs_fd *glfd, struct iatt *iatt, int valid)
-{
-	int              ret = -1;
-	xlator_t        *subvol = NULL;
-	fd_t            *fd = NULL;
-
-        DECLARE_OLD_THIS;
-	__GLFS_ENTRY_VALIDATE_FD (glfd, invalid_fs);
-
-	subvol = glfs_active_subvol (glfd->fs);
-	if (!subvol) {
-		ret = -1;
-		errno = EIO;
-		goto out;
-	}
-
-	fd = glfs_resolve_fd (glfd->fs, subvol, glfd);
-	if (!fd) {
-		ret = -1;
-		errno = EBADFD;
-		goto out;
-	}
-
-	ret = syncop_fsetattr (subvol, fd, iatt, valid, 0, 0, NULL, NULL);
-        DECODE_SYNCOP_ERR (ret);
+pub_glfs_fsetattr(struct glfs_fd *glfd, struct glfs_stat *stat)
+{
+    int ret = -1;
+    int glvalid;
+    struct iatt iatt = {
+        0,
+    };
+    xlator_t *subvol = NULL;
+    fd_t *fd = NULL;
+
+    DECLARE_OLD_THIS;
+    __GLFS_ENTRY_VALIDATE_FD(glfd, invalid_fs);
+
+    GF_REF_GET(glfd);
+
+    GF_VALIDATE_OR_GOTO("glfs_fsetattr", stat, out);
+
+    subvol = glfs_active_subvol(glfd->fs);
+    if (!subvol) {
+        ret = -1;
+        errno = EIO;
+        goto out;
+    }
+
+    fd = glfs_resolve_fd(glfd->fs, subvol, glfd);
+    if (!fd) {
+        ret = -1;
+        errno = EBADFD;
+        goto out;
+    }
+
+    glfs_iatt_from_statx(&iatt, stat);
+    glfsflags_from_gfapiflags(stat, &glvalid);
+
+    /* TODO : Add leaseid */
+    ret = syncop_fsetattr(subvol, fd, &iatt, glvalid, 0, 0, NULL, NULL);
+    DECODE_SYNCOP_ERR(ret);
 out:
-	if (fd)
-		fd_unref (fd);
+    if (fd)
+        fd_unref(fd);
+    if (glfd)
+        GF_REF_PUT(glfd);
 
-	glfs_subvol_done (glfd->fs, subvol);
+    glfs_subvol_done(glfd->fs, subvol);
 
-        __GLFS_EXIT_FS;
+    __GLFS_EXIT_FS;
 
 invalid_fs:
-	return ret;
+    return ret;
 }
 
-
+GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_chmod, 3.4.0)
 int
-pub_glfs_chmod (struct glfs *fs, const char *path, mode_t mode)
+pub_glfs_chmod(struct glfs *fs, const char *path, mode_t mode)
 {
-	int              ret = -1;
-	struct iatt      iatt = {0, };
-	int              valid = 0;
+    int ret = -1;
+    struct glfs_stat stat = {
+        0,
+    };
 
-	iatt.ia_prot = ia_prot_from_st_mode (mode);
-	valid = GF_SET_ATTR_MODE;
+    stat.glfs_st_mode = mode;
+    stat.glfs_st_mask = GLFS_STAT_MODE;
 
-	ret = glfs_setattr (fs, path, &iatt, valid, 1);
+    ret = glfs_setattr(fs, path, &stat, 1);
 
-	return ret;
+    return ret;
 }
 
-GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_chmod, 3.4.0);
-
-
+GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_fchmod, 3.4.0)
 int
-pub_glfs_fchmod (struct glfs_fd *glfd, mode_t mode)
+pub_glfs_fchmod(struct glfs_fd *glfd, mode_t mode)
 {
-	int              ret = -1;
-	struct iatt      iatt = {0, };
-	int              valid = 0;
+    int ret = -1;
+    struct glfs_stat stat = {
+        0,
+    };
 
-	iatt.ia_prot = ia_prot_from_st_mode (mode);
-	valid = GF_SET_ATTR_MODE;
+    stat.glfs_st_mode = mode;
+    stat.glfs_st_mask = GLFS_STAT_MODE;
 
-	ret = glfs_fsetattr (glfd, &iatt, valid);
+    ret = glfs_fsetattr(glfd, &stat);
 
-	return ret;
+    return ret;
 }
 
-GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_fchmod, 3.4.0);
-
-
+GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_chown, 3.4.0)
 int
-pub_glfs_chown (struct glfs *fs, const char *path, uid_t uid, gid_t gid)
+pub_glfs_chown(struct glfs *fs, const char *path, uid_t uid, gid_t gid)
 {
-	int              ret = 0;
-	int              valid = 0;
-	struct iatt      iatt = {0, };
+    int ret = 0;
+    struct glfs_stat stat = {
+        0,
+    };
 
-        if (uid != (uid_t) -1) {
-                iatt.ia_uid = uid;
-                valid = GF_SET_ATTR_UID;
-        }
+    if (uid != (uid_t)-1) {
+        stat.glfs_st_uid = uid;
+        stat.glfs_st_mask = GLFS_STAT_UID;
+    }
 
-        if (gid != (uid_t) -1) {
-                iatt.ia_gid = gid;
-                valid = valid | GF_SET_ATTR_GID;
-        }
+    if (gid != (uid_t)-1) {
+        stat.glfs_st_gid = gid;
+        stat.glfs_st_mask = stat.glfs_st_mask | GLFS_STAT_GID;
+    }
 
-        if (valid)
-	        ret = glfs_setattr (fs, path, &iatt, valid, 1);
+    if (stat.glfs_st_mask)
+        ret = glfs_setattr(fs, path, &stat, 1);
 
-	return ret;
+    return ret;
 }
 
-GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_chown, 3.4.0);
-
-
+GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_lchown, 3.4.0)
 int
-pub_glfs_lchown (struct glfs *fs, const char *path, uid_t uid, gid_t gid)
+pub_glfs_lchown(struct glfs *fs, const char *path, uid_t uid, gid_t gid)
 {
-	int              ret = 0;
-	int              valid = 0;
-	struct iatt      iatt = {0, };
+    int ret = 0;
+    struct glfs_stat stat = {
+        0,
+    };
 
-        if (uid != (uid_t) -1) {
-	        iatt.ia_uid = uid;
-	        valid = GF_SET_ATTR_UID;
-        }
+    if (uid != (uid_t)-1) {
+        stat.glfs_st_uid = uid;
+        stat.glfs_st_mask = GLFS_STAT_UID;
+    }
 
-        if (gid != (uid_t) -1) {
-                iatt.ia_gid = gid;
-                valid = valid | GF_SET_ATTR_GID;
-        }
+    if (gid != (uid_t)-1) {
+        stat.glfs_st_gid = gid;
+        stat.glfs_st_mask = stat.glfs_st_mask | GLFS_STAT_GID;
+    }
 
-        if (valid)
-	        ret = glfs_setattr (fs, path, &iatt, valid, 0);
+    if (stat.glfs_st_mask)
+        ret = glfs_setattr(fs, path, &stat, 0);
 
-	return ret;
+    return ret;
 }
 
-GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_lchown, 3.4.0);
-
-
+GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_fchown, 3.4.0)
 int
-pub_glfs_fchown (struct glfs_fd *glfd, uid_t uid, gid_t gid)
+pub_glfs_fchown(struct glfs_fd *glfd, uid_t uid, gid_t gid)
 {
-	int              ret = 0;
-	int              valid = 0;
-	struct iatt      iatt = {0, };
+    int ret = 0;
+    struct glfs_stat stat = {
+        0,
+    };
 
-        if (uid != (uid_t) -1) {
-                iatt.ia_uid = uid;
-                valid = GF_SET_ATTR_UID;
-        }
+    if (uid != (uid_t)-1) {
+        stat.glfs_st_uid = uid;
+        stat.glfs_st_mask = GLFS_STAT_UID;
+    }
 
-        if (gid != (uid_t) -1) {
-                iatt.ia_gid = gid;
-                valid = valid | GF_SET_ATTR_GID;
-        }
+    if (gid != (uid_t)-1) {
+        stat.glfs_st_gid = gid;
+        stat.glfs_st_mask = stat.glfs_st_mask | GLFS_STAT_GID;
+    }
 
-        if (valid)
-	        ret = glfs_fsetattr (glfd, &iatt, valid);
+    if (stat.glfs_st_mask)
+        ret = glfs_fsetattr(glfd, &stat);
 
-	return ret;
+    return ret;
 }
 
-GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_fchown, 3.4.0);
-
-
+GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_utimens, 3.4.0)
 int
-pub_glfs_utimens (struct glfs *fs, const char *path, struct timespec times[2])
+pub_glfs_utimens(struct glfs *fs, const char *path,
+                 const struct timespec times[2])
 {
-	int              ret = -1;
-	int              valid = 0;
-	struct iatt      iatt = {0, };
+    int ret = -1;
+    struct glfs_stat stat = {
+        0,
+    };
 
-	iatt.ia_atime = times[0].tv_sec;
-	iatt.ia_atime_nsec = times[0].tv_nsec;
-	iatt.ia_mtime = times[1].tv_sec;
-	iatt.ia_mtime_nsec = times[1].tv_nsec;
+    stat.glfs_st_atime = times[0];
+    stat.glfs_st_mtime = times[1];
 
-	valid = GF_SET_ATTR_ATIME|GF_SET_ATTR_MTIME;
+    stat.glfs_st_mask = GLFS_STAT_ATIME | GLFS_STAT_MTIME;
 
-	ret = glfs_setattr (fs, path, &iatt, valid, 1);
+    ret = glfs_setattr(fs, path, &stat, 1);
 
-	return ret;
+    return ret;
 }
 
-GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_utimens, 3.4.0);
-
-
+GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_lutimens, 3.4.0)
 int
-pub_glfs_lutimens (struct glfs *fs, const char *path, struct timespec times[2])
+pub_glfs_lutimens(struct glfs *fs, const char *path,
+                  const struct timespec times[2])
 {
-	int              ret = -1;
-	int              valid = 0;
-	struct iatt      iatt = {0, };
+    int ret = -1;
+    struct glfs_stat stat = {
+        0,
+    };
 
-	iatt.ia_atime = times[0].tv_sec;
-	iatt.ia_atime_nsec = times[0].tv_nsec;
-	iatt.ia_mtime = times[1].tv_sec;
-	iatt.ia_mtime_nsec = times[1].tv_nsec;
+    stat.glfs_st_atime = times[0];
+    stat.glfs_st_mtime = times[1];
 
-	valid = GF_SET_ATTR_ATIME|GF_SET_ATTR_MTIME;
+    stat.glfs_st_mask = GLFS_STAT_ATIME | GLFS_STAT_MTIME;
 
-	ret = glfs_setattr (fs, path, &iatt, valid, 0);
+    ret = glfs_setattr(fs, path, &stat, 0);
 
-	return ret;
+    return ret;
 }
 
-GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_lutimens, 3.4.0);
-
-
+GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_futimens, 3.4.0)
 int
-pub_glfs_futimens (struct glfs_fd *glfd, struct timespec times[2])
+pub_glfs_futimens(struct glfs_fd *glfd, const struct timespec times[2])
 {
-	int              ret = -1;
-	int              valid = 0;
-	struct iatt      iatt = {0, };
+    int ret = -1;
+    struct glfs_stat stat = {
+        0,
+    };
 
-	iatt.ia_atime = times[0].tv_sec;
-	iatt.ia_atime_nsec = times[0].tv_nsec;
-	iatt.ia_mtime = times[1].tv_sec;
-	iatt.ia_mtime_nsec = times[1].tv_nsec;
+    stat.glfs_st_atime = times[0];
+    stat.glfs_st_mtime = times[1];
 
-	valid = GF_SET_ATTR_ATIME|GF_SET_ATTR_MTIME;
+    stat.glfs_st_mask = GLFS_STAT_ATIME | GLFS_STAT_MTIME;
 
-	ret = glfs_fsetattr (glfd, &iatt, valid);
+    ret = glfs_fsetattr(glfd, &stat);
 
-	return ret;
+    return ret;
 }
 
-GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_futimens, 3.4.0);
-
-
 int
-glfs_getxattr_process (void *value, size_t size, dict_t *xattr,
-		       const char *name)
-{
-	data_t *data = NULL;
-	int     ret = -1;
-
-	data = dict_get (xattr, (char *)name);
-	if (!data) {
-		errno = ENODATA;
-		ret = -1;
-		goto out;
-	}
-
-	ret = data->len;
-	if (!value || !size)
-		goto out;
-
-	if (size < ret) {
-		ret = -1;
-		errno = ERANGE;
-		goto out;
-	}
-
-	memcpy (value, data->data, ret);
+glfs_getxattr_process(void *value, size_t size, dict_t *xattr, const char *name)
+{
+    data_t *data = NULL;
+    int ret = -1;
+
+    data = dict_get(xattr, (char *)name);
+    if (!data) {
+        errno = ENODATA;
+        ret = -1;
+        goto out;
+    }
+
+    ret = data->len;
+    if (!value || !size)
+        goto out;
+
+    if (size < ret) {
+        ret = -1;
+        errno = ERANGE;
+        goto out;
+    }
+
+    memcpy(value, data->data, ret);
 out:
-	if (xattr)
-		dict_unref (xattr);
-	return ret;
+    return ret;
 }
 
-
 ssize_t
-glfs_getxattr_common (struct glfs *fs, const char *path, const char *name,
-		      void *value, size_t size, int follow)
+glfs_getxattr_common(struct glfs *fs, const char *path, const char *name,
+                     void *value, size_t size, int follow)
 {
-	int              ret = -1;
-	xlator_t        *subvol = NULL;
-	loc_t            loc = {0, };
-	struct iatt      iatt = {0, };
-	dict_t          *xattr = NULL;
-	int              reval = 0;
-
-        DECLARE_OLD_THIS;
-        __GLFS_ENTRY_VALIDATE_FS (fs, invalid_fs);
-
-        if (!name || *name == '\0') {
-                ret = -1;
-                errno = EINVAL;
-                goto out;
-        }
-
-        if (strlen(name) > GF_XATTR_NAME_MAX) {
-                ret = -1;
-                errno = ENAMETOOLONG;
-                goto out;
-        }
-
-	subvol = glfs_active_subvol (fs);
-	if (!subvol) {
-		ret = -1;
-		errno = EIO;
-		goto out;
-	}
+    int ret = -1;
+    xlator_t *subvol = NULL;
+    loc_t loc = {
+        0,
+    };
+    struct iatt iatt = {
+        0,
+    };
+    dict_t *xattr = NULL;
+    int reval = 0;
+
+    DECLARE_OLD_THIS;
+    __GLFS_ENTRY_VALIDATE_FS(fs, invalid_fs);
+
+    if (!name || *name == '\0') {
+        ret = -1;
+        errno = EINVAL;
+        goto out;
+    }
+
+    if (strlen(name) > GF_XATTR_NAME_MAX) {
+        ret = -1;
+        errno = ENAMETOOLONG;
+        goto out;
+    }
+
+    subvol = glfs_active_subvol(fs);
+    if (!subvol) {
+        ret = -1;
+        errno = EIO;
+        goto out;
+    }
 
 retry:
-	if (follow)
-		ret = glfs_resolve (fs, subvol, path, &loc, &iatt, reval);
-	else
-		ret = glfs_lresolve (fs, subvol, path, &loc, &iatt, reval);
+    if (follow)
+        ret = glfs_resolve(fs, subvol, path, &loc, &iatt, reval);
+    else
+        ret = glfs_lresolve(fs, subvol, path, &loc, &iatt, reval);
 
-	ESTALE_RETRY (ret, errno, reval, &loc, retry);
+    ESTALE_RETRY(ret, errno, reval, &loc, retry);
 
-	if (ret)
-		goto out;
+    if (ret)
+        goto out;
 
-	ret = syncop_getxattr (subvol, &loc, &xattr, name, NULL, NULL);
-        DECODE_SYNCOP_ERR (ret);
+    ret = syncop_getxattr(subvol, &loc, &xattr, name, NULL, NULL);
+    DECODE_SYNCOP_ERR(ret);
 
-	ESTALE_RETRY (ret, errno, reval, &loc, retry);
+    ESTALE_RETRY(ret, errno, reval, &loc, retry);
 
-	if (ret)
-		goto out;
+    if (ret)
+        goto out;
 
-	ret = glfs_getxattr_process (value, size, xattr, name);
+    ret = glfs_getxattr_process(value, size, xattr, name);
 out:
-	loc_wipe (&loc);
+    loc_wipe(&loc);
+
+    if (xattr)
+        dict_unref(xattr);
 
-	glfs_subvol_done (fs, subvol);
+    glfs_subvol_done(fs, subvol);
 
-        __GLFS_EXIT_FS;
+    __GLFS_EXIT_FS;
 
 invalid_fs:
-	return ret;
+    return ret;
 }
 
-
+GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_getxattr, 3.4.0)
 ssize_t
-pub_glfs_getxattr (struct glfs *fs, const char *path, const char *name,
-                   void *value, size_t size)
+pub_glfs_getxattr(struct glfs *fs, const char *path, const char *name,
+                  void *value, size_t size)
 {
-	return glfs_getxattr_common (fs, path, name, value, size, 1);
+    return glfs_getxattr_common(fs, path, name, value, size, 1);
 }
 
-GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_getxattr, 3.4.0);
-
-
+GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_lgetxattr, 3.4.0)
 ssize_t
-pub_glfs_lgetxattr (struct glfs *fs, const char *path, const char *name,
-                    void *value, size_t size)
+pub_glfs_lgetxattr(struct glfs *fs, const char *path, const char *name,
+                   void *value, size_t size)
 {
-	return glfs_getxattr_common (fs, path, name, value, size, 0);
+    return glfs_getxattr_common(fs, path, name, value, size, 0);
 }
 
-GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_lgetxattr, 3.4.0);
-
-
+GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_fgetxattr, 3.4.0)
 ssize_t
-pub_glfs_fgetxattr (struct glfs_fd *glfd, const char *name, void *value,
-                    size_t size)
+pub_glfs_fgetxattr(struct glfs_fd *glfd, const char *name, void *value,
+                   size_t size)
 {
-	int              ret = -1;
-	xlator_t        *subvol = NULL;
-	dict_t          *xattr = NULL;
-	fd_t            *fd = NULL;
-
-        DECLARE_OLD_THIS;
-	__GLFS_ENTRY_VALIDATE_FD (glfd, invalid_fs);
-
-        if (!name || *name == '\0') {
-                ret = -1;
-                errno = EINVAL;
-                goto out;
-        }
-
-        if (strlen(name) > GF_XATTR_NAME_MAX) {
-                ret = -1;
-                errno = ENAMETOOLONG;
-                goto out;
-        }
-
-	subvol = glfs_active_subvol (glfd->fs);
-	if (!subvol) {
-		ret = -1;
-		errno = EIO;
-		goto out;
-	}
-
-	fd = glfs_resolve_fd (glfd->fs, subvol, glfd);
-	if (!fd) {
-		ret = -1;
-		errno = EBADFD;
-		goto out;
-	}
-
-	ret = syncop_fgetxattr (subvol, fd, &xattr, name, NULL, NULL);
-        DECODE_SYNCOP_ERR (ret);
-	if (ret)
-		goto out;
-
-	ret = glfs_getxattr_process (value, size, xattr, name);
+    int ret = -1;
+    xlator_t *subvol = NULL;
+    dict_t *xattr = NULL;
+    fd_t *fd = NULL;
+
+    DECLARE_OLD_THIS;
+    __GLFS_ENTRY_VALIDATE_FD(glfd, invalid_fs);
+
+    GF_REF_GET(glfd);
+
+    if (!name || *name == '\0') {
+        ret = -1;
+        errno = EINVAL;
+        goto out;
+    }
+
+    if (strlen(name) > GF_XATTR_NAME_MAX) {
+        ret = -1;
+        errno = ENAMETOOLONG;
+        goto out;
+    }
+
+    subvol = glfs_active_subvol(glfd->fs);
+    if (!subvol) {
+        ret = -1;
+        errno = EIO;
+        goto out;
+    }
+
+    fd = glfs_resolve_fd(glfd->fs, subvol, glfd);
+    if (!fd) {
+        ret = -1;
+        errno = EBADFD;
+        goto out;
+    }
+
+    ret = syncop_fgetxattr(subvol, fd, &xattr, name, NULL, NULL);
+    DECODE_SYNCOP_ERR(ret);
+    if (ret)
+        goto out;
+
+    ret = glfs_getxattr_process(value, size, xattr, name);
 out:
-	if (fd)
-		fd_unref (fd);
+    if (fd)
+        fd_unref(fd);
+    if (glfd)
+        GF_REF_PUT(glfd);
+    if (xattr)
+        dict_unref(xattr);
 
-	glfs_subvol_done (glfd->fs, subvol);
+    glfs_subvol_done(glfd->fs, subvol);
 
-        __GLFS_EXIT_FS;
+    __GLFS_EXIT_FS;
 
 invalid_fs:
-	return ret;
+    return ret;
 }
 
-GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_fgetxattr, 3.4.0);
-
-
 int
-glfs_listxattr_process (void *value, size_t size, dict_t *xattr)
+glfs_listxattr_process(void *value, size_t size, dict_t *xattr)
 {
-	int ret = -1;
+    int ret = -1;
 
-	if (!xattr)
-		goto out;
+    if (!xattr)
+        goto out;
 
-	ret = dict_keys_join (NULL, 0, xattr, NULL);
+    ret = dict_keys_join(NULL, 0, xattr, NULL);
 
-        if (!value || !size)
-                goto out;
+    if (!value || !size)
+        goto out;
 
-	if (size < ret) {
-		ret = -1;
-		errno = ERANGE;
-	} else {
-		dict_keys_join (value, size, xattr, NULL);
-	}
+    if (size < ret) {
+        ret = -1;
+        errno = ERANGE;
+    } else {
+        dict_keys_join(value, size, xattr, NULL);
+    }
 
 out:
-        if (xattr)
-                dict_unref (xattr);
-
-	return ret;
+    return ret;
 }
 
-
 ssize_t
-glfs_listxattr_common (struct glfs *fs, const char *path, void *value,
-		       size_t size, int follow)
-{
-	int              ret = -1;
-	xlator_t        *subvol = NULL;
-	loc_t            loc = {0, };
-	struct iatt      iatt = {0, };
-	dict_t          *xattr = NULL;
-	int              reval = 0;
-
-        DECLARE_OLD_THIS;
-        __GLFS_ENTRY_VALIDATE_FS (fs, invalid_fs);
-
-	subvol = glfs_active_subvol (fs);
-	if (!subvol) {
-		ret = -1;
-		errno = EIO;
-		goto out;
-	}
+glfs_listxattr_common(struct glfs *fs, const char *path, void *value,
+                      size_t size, int follow)
+{
+    int ret = -1;
+    xlator_t *subvol = NULL;
+    loc_t loc = {
+        0,
+    };
+    struct iatt iatt = {
+        0,
+    };
+    dict_t *xattr = NULL;
+    int reval = 0;
+
+    DECLARE_OLD_THIS;
+    __GLFS_ENTRY_VALIDATE_FS(fs, invalid_fs);
+
+    subvol = glfs_active_subvol(fs);
+    if (!subvol) {
+        ret = -1;
+        errno = EIO;
+        goto out;
+    }
 
 retry:
-	if (follow)
-		ret = glfs_resolve (fs, subvol, path, &loc, &iatt, reval);
-	else
-		ret = glfs_lresolve (fs, subvol, path, &loc, &iatt, reval);
+    if (follow)
+        ret = glfs_resolve(fs, subvol, path, &loc, &iatt, reval);
+    else
+        ret = glfs_lresolve(fs, subvol, path, &loc, &iatt, reval);
 
-	ESTALE_RETRY (ret, errno, reval, &loc, retry);
+    ESTALE_RETRY(ret, errno, reval, &loc, retry);
 
-	if (ret)
-		goto out;
+    if (ret)
+        goto out;
 
-	ret = syncop_getxattr (subvol, &loc, &xattr, NULL, NULL, NULL);
-        DECODE_SYNCOP_ERR (ret);
+    ret = syncop_getxattr(subvol, &loc, &xattr, NULL, NULL, NULL);
+    DECODE_SYNCOP_ERR(ret);
 
-	ESTALE_RETRY (ret, errno, reval, &loc, retry);
+    ESTALE_RETRY(ret, errno, reval, &loc, retry);
 
-	if (ret)
-		goto out;
+    if (ret)
+        goto out;
 
-	ret = glfs_listxattr_process (value, size, xattr);
+    ret = glfs_listxattr_process(value, size, xattr);
 out:
-	loc_wipe (&loc);
+    loc_wipe(&loc);
 
-	glfs_subvol_done (fs, subvol);
+    if (xattr)
+        dict_unref(xattr);
 
-        __GLFS_EXIT_FS;
+    glfs_subvol_done(fs, subvol);
+
+    __GLFS_EXIT_FS;
 
 invalid_fs:
-	return ret;
+    return ret;
 }
 
-
+GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_listxattr, 3.4.0)
 ssize_t
-pub_glfs_listxattr (struct glfs *fs, const char *path, void *value, size_t size)
+pub_glfs_listxattr(struct glfs *fs, const char *path, void *value, size_t size)
 {
-	return glfs_listxattr_common (fs, path, value, size, 1);
+    return glfs_listxattr_common(fs, path, value, size, 1);
 }
 
-GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_listxattr, 3.4.0);
-
-
+GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_llistxattr, 3.4.0)
 ssize_t
-pub_glfs_llistxattr (struct glfs *fs, const char *path, void *value, size_t size)
+pub_glfs_llistxattr(struct glfs *fs, const char *path, void *value, size_t size)
 {
-	return glfs_listxattr_common (fs, path, value, size, 0);
+    return glfs_listxattr_common(fs, path, value, size, 0);
 }
 
-GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_llistxattr, 3.4.0);
-
-
+GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_flistxattr, 3.4.0)
 ssize_t
-pub_glfs_flistxattr (struct glfs_fd *glfd, void *value, size_t size)
-{
-	int              ret = -1;
-	xlator_t        *subvol = NULL;
-	dict_t          *xattr = NULL;
-	fd_t            *fd = NULL;
-
-        DECLARE_OLD_THIS;
-	__GLFS_ENTRY_VALIDATE_FD (glfd, invalid_fs);
-
-	subvol = glfs_active_subvol (glfd->fs);
-	if (!subvol) {
-		ret = -1;
-		errno = EIO;
-		goto out;
-	}
-
-	fd = glfs_resolve_fd (glfd->fs, subvol, glfd);
-	if (!fd) {
-		ret = -1;
-		errno = EBADFD;
-		goto out;
-	}
-
-	ret = syncop_fgetxattr (subvol, fd, &xattr, NULL, NULL, NULL);
-        DECODE_SYNCOP_ERR (ret);
-	if (ret)
-		goto out;
-
-	ret = glfs_listxattr_process (value, size, xattr);
+pub_glfs_flistxattr(struct glfs_fd *glfd, void *value, size_t size)
+{
+    int ret = -1;
+    xlator_t *subvol = NULL;
+    dict_t *xattr = NULL;
+    fd_t *fd = NULL;
+
+    DECLARE_OLD_THIS;
+    __GLFS_ENTRY_VALIDATE_FD(glfd, invalid_fs);
+
+    GF_REF_GET(glfd);
+
+    subvol = glfs_active_subvol(glfd->fs);
+    if (!subvol) {
+        ret = -1;
+        errno = EIO;
+        goto out;
+    }
+
+    fd = glfs_resolve_fd(glfd->fs, subvol, glfd);
+    if (!fd) {
+        ret = -1;
+        errno = EBADFD;
+        goto out;
+    }
+
+    ret = syncop_fgetxattr(subvol, fd, &xattr, NULL, NULL, NULL);
+    DECODE_SYNCOP_ERR(ret);
+    if (ret)
+        goto out;
+
+    ret = glfs_listxattr_process(value, size, xattr);
 out:
-	if (fd)
-		fd_unref (fd);
+    if (fd)
+        fd_unref(fd);
+    if (glfd)
+        GF_REF_PUT(glfd);
+    if (xattr)
+        dict_unref(xattr);
 
-	glfs_subvol_done (glfd->fs, subvol);
+    glfs_subvol_done(glfd->fs, subvol);
 
-        __GLFS_EXIT_FS;
+    __GLFS_EXIT_FS;
 
 invalid_fs:
-	return ret;
+    return ret;
 }
 
-GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_flistxattr, 3.4.0);
-
 int
-glfs_setxattr_common (struct glfs *fs, const char *path, const char *name,
-		      const void *value, size_t size, int flags, int follow)
+glfs_setxattr_common(struct glfs *fs, const char *path, const char *name,
+                     const void *value, size_t size, int flags, int follow)
 {
-	int              ret = -1;
-	xlator_t        *subvol = NULL;
-	loc_t            loc = {0, };
-	struct iatt      iatt = {0, };
-	dict_t          *xattr = NULL;
-	int              reval = 0;
-
-        DECLARE_OLD_THIS;
-        __GLFS_ENTRY_VALIDATE_FS (fs, invalid_fs);
-
-        if (!name || *name == '\0') {
-                ret = -1;
-                errno = EINVAL;
-                goto out;
-        }
-
-        if (strlen(name) > GF_XATTR_NAME_MAX) {
-                ret = -1;
-                errno = ENAMETOOLONG;
-                goto out;
-        }
-
-	subvol = glfs_active_subvol (fs);
-	if (!subvol) {
-		ret = -1;
-		errno = EIO;
-		goto out;
-	}
+    int ret = -1;
+    xlator_t *subvol = NULL;
+    loc_t loc = {
+        0,
+    };
+    struct iatt iatt = {
+        0,
+    };
+    dict_t *xattr = NULL;
+    int reval = 0;
+    void *value_cp = NULL;
+
+    DECLARE_OLD_THIS;
+    __GLFS_ENTRY_VALIDATE_FS(fs, invalid_fs);
+
+    if (!name || *name == '\0') {
+        ret = -1;
+        errno = EINVAL;
+        goto out;
+    }
+
+    if (strlen(name) > GF_XATTR_NAME_MAX) {
+        ret = -1;
+        errno = ENAMETOOLONG;
+        goto out;
+    }
+
+    subvol = glfs_active_subvol(fs);
+    if (!subvol) {
+        ret = -1;
+        errno = EIO;
+        goto out;
+    }
 
 retry:
-	if (follow)
-		ret = glfs_resolve (fs, subvol, path, &loc, &iatt, reval);
-	else
-		ret = glfs_lresolve (fs, subvol, path, &loc, &iatt, reval);
+    if (follow)
+        ret = glfs_resolve(fs, subvol, path, &loc, &iatt, reval);
+    else
+        ret = glfs_lresolve(fs, subvol, path, &loc, &iatt, reval);
 
-	ESTALE_RETRY (ret, errno, reval, &loc, retry);
+    ESTALE_RETRY(ret, errno, reval, &loc, retry);
 
-	if (ret)
-		goto out;
+    if (ret)
+        goto out;
 
-	xattr = dict_for_key_value (name, value, size);
-	if (!xattr) {
-		ret = -1;
-		errno = ENOMEM;
-		goto out;
-	}
+    value_cp = gf_memdup(value, size);
+    GF_CHECK_ALLOC_AND_LOG(subvol->name, value_cp, ret,
+                           "Failed to"
+                           " duplicate setxattr value",
+                           out);
 
-	ret = syncop_setxattr (subvol, &loc, xattr, flags, NULL, NULL);
-        DECODE_SYNCOP_ERR (ret);
+    xattr = dict_for_key_value(name, value_cp, size, _gf_false);
+    if (!xattr) {
+        GF_FREE(value_cp);
+        ret = -1;
+        errno = ENOMEM;
+        goto out;
+    }
 
-	ESTALE_RETRY (ret, errno, reval, &loc, retry);
+    ret = syncop_setxattr(subvol, &loc, xattr, flags, NULL, NULL);
+    DECODE_SYNCOP_ERR(ret);
 
 out:
-	loc_wipe (&loc);
-	if (xattr)
-		dict_unref (xattr);
+    loc_wipe(&loc);
+    if (xattr)
+        dict_unref(xattr);
 
-	glfs_subvol_done (fs, subvol);
+    glfs_subvol_done(fs, subvol);
 
-        __GLFS_EXIT_FS;
+    __GLFS_EXIT_FS;
 
 invalid_fs:
-	return ret;
+    return ret;
 }
 
+GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_setxattr, 3.4.0)
+int
+pub_glfs_setxattr(struct glfs *fs, const char *path, const char *name,
+                  const void *value, size_t size, int flags)
+{
+    return glfs_setxattr_common(fs, path, name, value, size, flags, 1);
+}
 
+GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_lsetxattr, 3.4.0)
 int
-pub_glfs_setxattr (struct glfs *fs, const char *path, const char *name,
+pub_glfs_lsetxattr(struct glfs *fs, const char *path, const char *name,
                    const void *value, size_t size, int flags)
 {
-	return glfs_setxattr_common (fs, path, name, value, size, flags, 1);
+    return glfs_setxattr_common(fs, path, name, value, size, flags, 0);
 }
 
-GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_setxattr, 3.4.0);
-
-
+GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_fsetxattr, 3.4.0)
 int
-pub_glfs_lsetxattr (struct glfs *fs, const char *path, const char *name,
-                    const void *value, size_t size, int flags)
+pub_glfs_fsetxattr(struct glfs_fd *glfd, const char *name, const void *value,
+                   size_t size, int flags)
 {
-	return glfs_setxattr_common (fs, path, name, value, size, flags, 0);
-}
+    int ret = -1;
+    xlator_t *subvol = NULL;
+    dict_t *xattr = NULL;
+    fd_t *fd = NULL;
+    void *value_cp = NULL;
+
+    DECLARE_OLD_THIS;
+    __GLFS_ENTRY_VALIDATE_FD(glfd, invalid_fs);
+
+    GF_REF_GET(glfd);
+
+    if (!name || *name == '\0') {
+        ret = -1;
+        errno = EINVAL;
+        goto out;
+    }
+
+    if (strlen(name) > GF_XATTR_NAME_MAX) {
+        ret = -1;
+        errno = ENAMETOOLONG;
+        goto out;
+    }
+
+    subvol = glfs_active_subvol(glfd->fs);
+    if (!subvol) {
+        ret = -1;
+        errno = EIO;
+        goto out;
+    }
+
+    fd = glfs_resolve_fd(glfd->fs, subvol, glfd);
+    if (!fd) {
+        ret = -1;
+        errno = EBADFD;
+        goto out;
+    }
+
+    value_cp = gf_memdup(value, size);
+    GF_CHECK_ALLOC_AND_LOG(subvol->name, value_cp, ret,
+                           "Failed to"
+                           " duplicate setxattr value",
+                           out);
+
+    xattr = dict_for_key_value(name, value_cp, size, _gf_false);
+    if (!xattr) {
+        GF_FREE(value_cp);
+        ret = -1;
+        errno = ENOMEM;
+        goto out;
+    }
+
+    ret = syncop_fsetxattr(subvol, fd, xattr, flags, NULL, NULL);
+    DECODE_SYNCOP_ERR(ret);
+out:
+    if (xattr)
+        dict_unref(xattr);
 
-GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_lsetxattr, 3.4.0);
+    if (fd)
+        fd_unref(fd);
+    if (glfd)
+        GF_REF_PUT(glfd);
 
+    glfs_subvol_done(glfd->fs, subvol);
+
+    __GLFS_EXIT_FS;
+
+invalid_fs:
+    return ret;
+}
 
 int
-pub_glfs_fsetxattr (struct glfs_fd *glfd, const char *name, const void *value,
-                    size_t size, int flags)
+glfs_removexattr_common(struct glfs *fs, const char *path, const char *name,
+                        int follow)
 {
-	int              ret = -1;
-	xlator_t        *subvol = NULL;
-	dict_t          *xattr = NULL;
-	fd_t            *fd = NULL;
+    int ret = -1;
+    xlator_t *subvol = NULL;
+    loc_t loc = {
+        0,
+    };
+    struct iatt iatt = {
+        0,
+    };
+    int reval = 0;
+
+    DECLARE_OLD_THIS;
+    __GLFS_ENTRY_VALIDATE_FS(fs, invalid_fs);
+
+    subvol = glfs_active_subvol(fs);
+    if (!subvol) {
+        ret = -1;
+        errno = EIO;
+        goto out;
+    }
+retry:
+    if (follow)
+        ret = glfs_resolve(fs, subvol, path, &loc, &iatt, reval);
+    else
+        ret = glfs_lresolve(fs, subvol, path, &loc, &iatt, reval);
 
-        DECLARE_OLD_THIS;
-	__GLFS_ENTRY_VALIDATE_FD (glfd, invalid_fs);
+    ESTALE_RETRY(ret, errno, reval, &loc, retry);
 
-        if (!name || *name == '\0') {
-                ret = -1;
-                errno = EINVAL;
-                goto out;
-        }
+    if (ret)
+        goto out;
 
-        if (strlen(name) > GF_XATTR_NAME_MAX) {
-                ret = -1;
-                errno = ENAMETOOLONG;
-                goto out;
-        }
+    ret = syncop_removexattr(subvol, &loc, name, NULL, NULL);
+    DECODE_SYNCOP_ERR(ret);
 
-	subvol = glfs_active_subvol (glfd->fs);
-	if (!subvol) {
-		ret = -1;
-		errno = EIO;
-		goto out;
-	}
-
-	fd = glfs_resolve_fd (glfd->fs, subvol, glfd);
-	if (!fd) {
-		ret = -1;
-		errno = EBADFD;
-		goto out;
-	}
-
-	xattr = dict_for_key_value (name, value, size);
-	if (!xattr) {
-		ret = -1;
-		errno = ENOMEM;
-		goto out;
-	}
-
-	ret = syncop_fsetxattr (subvol, fd, xattr, flags, NULL, NULL);
-        DECODE_SYNCOP_ERR (ret);
-out:
-	if (xattr)
-		dict_unref (xattr);
+    ESTALE_RETRY(ret, errno, reval, &loc, retry);
 
-	if (fd)
-		fd_unref (fd);
+out:
+    loc_wipe(&loc);
 
-	glfs_subvol_done (glfd->fs, subvol);
+    glfs_subvol_done(fs, subvol);
 
-        __GLFS_EXIT_FS;
+    __GLFS_EXIT_FS;
 
 invalid_fs:
-	return ret;
+    return ret;
 }
 
-GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_fsetxattr, 3.4.0);
-
+GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_removexattr, 3.4.0)
+int
+pub_glfs_removexattr(struct glfs *fs, const char *path, const char *name)
+{
+    return glfs_removexattr_common(fs, path, name, 1);
+}
 
+GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_lremovexattr, 3.4.0)
 int
-glfs_removexattr_common (struct glfs *fs, const char *path, const char *name,
-			 int follow)
-{
-	int              ret = -1;
-	xlator_t        *subvol = NULL;
-	loc_t            loc = {0, };
-	struct iatt      iatt = {0, };
-	int              reval = 0;
-
-        DECLARE_OLD_THIS;
-        __GLFS_ENTRY_VALIDATE_FS (fs, invalid_fs);
-
-	subvol = glfs_active_subvol (fs);
-	if (!subvol) {
-		ret = -1;
-		errno = EIO;
-		goto out;
-	}
-retry:
-	if (follow)
-		ret = glfs_resolve (fs, subvol, path, &loc, &iatt, reval);
-	else
-		ret = glfs_lresolve (fs, subvol, path, &loc, &iatt, reval);
+pub_glfs_lremovexattr(struct glfs *fs, const char *path, const char *name)
+{
+    return glfs_removexattr_common(fs, path, name, 0);
+}
 
-	ESTALE_RETRY (ret, errno, reval, &loc, retry);
+GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_fremovexattr, 3.4.0)
+int
+pub_glfs_fremovexattr(struct glfs_fd *glfd, const char *name)
+{
+    int ret = -1;
+    xlator_t *subvol = NULL;
+    fd_t *fd = NULL;
+
+    DECLARE_OLD_THIS;
+    __GLFS_ENTRY_VALIDATE_FD(glfd, invalid_fs);
+
+    GF_REF_GET(glfd);
+
+    subvol = glfs_active_subvol(glfd->fs);
+    if (!subvol) {
+        ret = -1;
+        errno = EIO;
+        goto out;
+    }
+
+    fd = glfs_resolve_fd(glfd->fs, subvol, glfd);
+    if (!fd) {
+        ret = -1;
+        errno = EBADFD;
+        goto out;
+    }
+
+    ret = syncop_fremovexattr(subvol, fd, name, NULL, NULL);
+    DECODE_SYNCOP_ERR(ret);
+out:
+    if (fd)
+        fd_unref(fd);
+    if (glfd)
+        GF_REF_PUT(glfd);
 
-	if (ret)
-		goto out;
+    glfs_subvol_done(glfd->fs, subvol);
 
-	ret = syncop_removexattr (subvol, &loc, name, NULL, NULL);
-        DECODE_SYNCOP_ERR (ret);
+    __GLFS_EXIT_FS;
 
-	ESTALE_RETRY (ret, errno, reval, &loc, retry);
+invalid_fs:
+    return ret;
+}
 
+GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_fallocate, 3.5.0)
+int
+pub_glfs_fallocate(struct glfs_fd *glfd, int keep_size, off_t offset,
+                   size_t len)
+{
+    int ret = -1;
+    xlator_t *subvol = NULL;
+    fd_t *fd = NULL;
+    dict_t *fop_attr = NULL;
+
+    DECLARE_OLD_THIS;
+    __GLFS_ENTRY_VALIDATE_FD(glfd, invalid_fs);
+
+    GF_REF_GET(glfd);
+
+    subvol = glfs_active_subvol(glfd->fs);
+    if (!subvol) {
+        ret = -1;
+        errno = EIO;
+        goto out;
+    }
+
+    fd = glfs_resolve_fd(glfd->fs, subvol, glfd);
+    if (!fd) {
+        ret = -1;
+        errno = EBADFD;
+        goto out;
+    }
+
+    ret = get_fop_attr_thrd_key(&fop_attr);
+    if (ret)
+        gf_msg_debug("gfapi", 0, "Getting leaseid from thread failed");
+
+    ret = syncop_fallocate(subvol, fd, keep_size, offset, len, fop_attr, NULL);
+    DECODE_SYNCOP_ERR(ret);
 out:
-	loc_wipe (&loc);
+    if (fd)
+        fd_unref(fd);
+    if (glfd)
+        GF_REF_PUT(glfd);
+    if (fop_attr)
+        dict_unref(fop_attr);
 
-	glfs_subvol_done (fs, subvol);
+    glfs_subvol_done(glfd->fs, subvol);
 
-        __GLFS_EXIT_FS;
+    __GLFS_EXIT_FS;
 
 invalid_fs:
-	return ret;
+    return ret;
 }
 
-
+GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_discard, 3.5.0)
 int
-pub_glfs_removexattr (struct glfs *fs, const char *path, const char *name)
+pub_glfs_discard(struct glfs_fd *glfd, off_t offset, size_t len)
 {
-	return glfs_removexattr_common (fs, path, name, 1);
-}
+    int ret = -1;
+    xlator_t *subvol = NULL;
+    fd_t *fd = NULL;
+    dict_t *fop_attr = NULL;
+
+    DECLARE_OLD_THIS;
+    __GLFS_ENTRY_VALIDATE_FD(glfd, invalid_fs);
+
+    GF_REF_GET(glfd);
+
+    subvol = glfs_active_subvol(glfd->fs);
+    if (!subvol) {
+        ret = -1;
+        errno = EIO;
+        goto out;
+    }
+
+    fd = glfs_resolve_fd(glfd->fs, subvol, glfd);
+    if (!fd) {
+        ret = -1;
+        errno = EBADFD;
+        goto out;
+    }
+
+    ret = get_fop_attr_thrd_key(&fop_attr);
+    if (ret)
+        gf_msg_debug("gfapi", 0, "Getting leaseid from thread failed");
+
+    ret = syncop_discard(subvol, fd, offset, len, fop_attr, NULL);
+    DECODE_SYNCOP_ERR(ret);
+out:
+    if (fd)
+        fd_unref(fd);
+    if (glfd)
+        GF_REF_PUT(glfd);
+    if (fop_attr)
+        dict_unref(fop_attr);
 
-GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_removexattr, 3.4.0);
+    glfs_subvol_done(glfd->fs, subvol);
 
+    __GLFS_EXIT_FS;
 
+invalid_fs:
+    return ret;
+}
+
+GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_zerofill, 3.5.0)
 int
-pub_glfs_lremovexattr (struct glfs *fs, const char *path, const char *name)
+pub_glfs_zerofill(struct glfs_fd *glfd, off_t offset, off_t len)
 {
-	return glfs_removexattr_common (fs, path, name, 0);
-}
+    int ret = -1;
+    xlator_t *subvol = NULL;
+    fd_t *fd = NULL;
+    dict_t *fop_attr = NULL;
+
+    DECLARE_OLD_THIS;
+    __GLFS_ENTRY_VALIDATE_FD(glfd, invalid_fs);
+
+    GF_REF_GET(glfd);
+
+    subvol = glfs_active_subvol(glfd->fs);
+    if (!subvol) {
+        errno = EIO;
+        goto out;
+    }
+
+    fd = glfs_resolve_fd(glfd->fs, subvol, glfd);
+    if (!fd) {
+        errno = EBADFD;
+        goto out;
+    }
+
+    ret = get_fop_attr_thrd_key(&fop_attr);
+    if (ret)
+        gf_msg_debug("gfapi", 0, "Getting leaseid from thread failed");
+
+    ret = syncop_zerofill(subvol, fd, offset, len, fop_attr, NULL);
+    DECODE_SYNCOP_ERR(ret);
+out:
+    if (fd)
+        fd_unref(fd);
+    if (glfd)
+        GF_REF_PUT(glfd);
+    if (fop_attr)
+        dict_unref(fop_attr);
 
-GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_lremovexattr, 3.4.0);
+    glfs_subvol_done(glfd->fs, subvol);
 
+    __GLFS_EXIT_FS;
+
+invalid_fs:
+    return ret;
+}
 
+GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_chdir, 3.4.0)
 int
-pub_glfs_fremovexattr (struct glfs_fd *glfd, const char *name)
-{
-	int              ret = -1;
-	xlator_t        *subvol = NULL;
-	fd_t            *fd = NULL;
-
-        DECLARE_OLD_THIS;
-	__GLFS_ENTRY_VALIDATE_FD (glfd, invalid_fs);
-
-	subvol = glfs_active_subvol (glfd->fs);
-	if (!subvol) {
-		ret = -1;
-		errno = EIO;
-		goto out;
-	}
-
-	fd = glfs_resolve_fd (glfd->fs, subvol, glfd);
-	if (!fd) {
-		ret = -1;
-		errno = EBADFD;
-		goto out;
-	}
-
-	ret = syncop_fremovexattr (subvol, fd, name, NULL, NULL);
-        DECODE_SYNCOP_ERR (ret);
+pub_glfs_chdir(struct glfs *fs, const char *path)
+{
+    int ret = -1;
+    xlator_t *subvol = NULL;
+    loc_t loc = {
+        0,
+    };
+    struct iatt iatt = {
+        0,
+    };
+    int reval = 0;
+
+    DECLARE_OLD_THIS;
+    __GLFS_ENTRY_VALIDATE_FS(fs, invalid_fs);
+
+    subvol = glfs_active_subvol(fs);
+    if (!subvol) {
+        ret = -1;
+        errno = EIO;
+        goto out;
+    }
+retry:
+    ret = glfs_resolve(fs, subvol, path, &loc, &iatt, reval);
+
+    ESTALE_RETRY(ret, errno, reval, &loc, retry);
+
+    if (ret)
+        goto out;
+
+    if (!IA_ISDIR(iatt.ia_type)) {
+        ret = -1;
+        errno = ENOTDIR;
+        goto out;
+    }
+
+    glfs_cwd_set(fs, loc.inode);
+
 out:
-	if (fd)
-		fd_unref (fd);
+    loc_wipe(&loc);
 
-	glfs_subvol_done (glfd->fs, subvol);
+    glfs_subvol_done(fs, subvol);
 
-        __GLFS_EXIT_FS;
+    __GLFS_EXIT_FS;
 
 invalid_fs:
-	return ret;
+    return ret;
 }
 
-GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_fremovexattr, 3.4.0);
-
-
+GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_fchdir, 3.4.0)
 int
-pub_glfs_fallocate (struct glfs_fd *glfd, int keep_size, off_t offset, size_t len)
-{
-	int              ret = -1;
-	xlator_t        *subvol = NULL;
-	fd_t		*fd = NULL;
-
-        DECLARE_OLD_THIS;
-	__GLFS_ENTRY_VALIDATE_FD (glfd, invalid_fs);
-
-	subvol = glfs_active_subvol (glfd->fs);
-	if (!subvol) {
-		ret = -1;
-		errno = EIO;
-		goto out;
-	}
-
-	fd = glfs_resolve_fd (glfd->fs, subvol, glfd);
-	if (!fd) {
-		ret = -1;
-		errno = EBADFD;
-		goto out;
-	}
-
-	ret = syncop_fallocate (subvol, fd, keep_size, offset, len, NULL, NULL);
-        DECODE_SYNCOP_ERR (ret);
+pub_glfs_fchdir(struct glfs_fd *glfd)
+{
+    int ret = -1;
+    inode_t *inode = NULL;
+    xlator_t *subvol = NULL;
+    fd_t *fd = NULL;
+
+    DECLARE_OLD_THIS;
+    __GLFS_ENTRY_VALIDATE_FD(glfd, invalid_fs);
+
+    GF_REF_GET(glfd);
+
+    subvol = glfs_active_subvol(glfd->fs);
+    if (!subvol) {
+        ret = -1;
+        errno = EIO;
+        goto out;
+    }
+
+    fd = glfs_resolve_fd(glfd->fs, subvol, glfd);
+    if (!fd) {
+        ret = -1;
+        errno = EBADFD;
+        goto out;
+    }
+
+    inode = fd->inode;
+
+    if (!IA_ISDIR(inode->ia_type)) {
+        ret = -1;
+        errno = ENOTDIR;
+        goto out;
+    }
+
+    glfs_cwd_set(glfd->fs, inode);
+    ret = 0;
 out:
-	if (fd)
-		fd_unref(fd);
+    if (fd)
+        fd_unref(fd);
+    if (glfd)
+        GF_REF_PUT(glfd);
 
-	glfs_subvol_done (glfd->fs, subvol);
+    glfs_subvol_done(glfd->fs, subvol);
 
-        __GLFS_EXIT_FS;
+    __GLFS_EXIT_FS;
 
 invalid_fs:
-	return ret;
+    return ret;
 }
 
-GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_fallocate, 3.5.0);
+static gf_boolean_t warn_realpath = _gf_true; /* log once */
 
+static char *
+glfs_realpath_common(struct glfs *fs, const char *path, char *resolved_path,
+                     gf_boolean_t warn_deprecated)
+{
+    int ret = -1;
+    char *retpath = NULL;
+    char *allocpath = NULL;
+    xlator_t *subvol = NULL;
+    loc_t loc = {
+        0,
+    };
+    struct iatt iatt = {
+        0,
+    };
+    int reval = 0;
+
+    DECLARE_OLD_THIS;
+    __GLFS_ENTRY_VALIDATE_FS(fs, invalid_fs);
+
+    if (resolved_path)
+        retpath = resolved_path;
+    else if (warn_deprecated) {
+        retpath = allocpath = malloc(PATH_MAX + 1);
+        if (warn_realpath) {
+            warn_realpath = _gf_false;
+            gf_log(THIS->name, GF_LOG_WARNING,
+                   "this application "
+                   "is compiled against an old version of "
+                   "libgfapi, it should use glfs_free() to "
+                   "release the path returned by "
+                   "glfs_realpath()");
+        }
+    } else {
+        retpath = allocpath = GLFS_CALLOC(1, PATH_MAX + 1, NULL,
+                                          glfs_mt_realpath_t);
+    }
+
+    if (!retpath) {
+        ret = -1;
+        errno = ENOMEM;
+        goto out;
+    }
+
+    subvol = glfs_active_subvol(fs);
+    if (!subvol) {
+        ret = -1;
+        errno = EIO;
+        goto out;
+    }
+retry:
+    ret = glfs_resolve(fs, subvol, path, &loc, &iatt, reval);
+
+    ESTALE_RETRY(ret, errno, reval, &loc, retry);
+
+    if (ret)
+        goto out;
+
+    if (loc.path) {
+        snprintf(retpath, PATH_MAX + 1, "%s", loc.path);
+    }
 
-int
-pub_glfs_discard (struct glfs_fd *glfd, off_t offset, size_t len)
-{
-	int              ret = -1;
-	xlator_t        *subvol = NULL;
-	fd_t		*fd = NULL;
-
-        DECLARE_OLD_THIS;
-	__GLFS_ENTRY_VALIDATE_FD (glfd, invalid_fs);
-
-	subvol = glfs_active_subvol (glfd->fs);
-	if (!subvol) {
-		ret = -1;
-		errno = EIO;
-		goto out;
-	}
-
-	fd = glfs_resolve_fd (glfd->fs, subvol, glfd);
-	if (!fd) {
-		ret = -1;
-		errno = EBADFD;
-		goto out;
-	}
-
-	ret = syncop_discard (subvol, fd, offset, len, NULL, NULL);
-        DECODE_SYNCOP_ERR (ret);
 out:
-	if (fd)
-		fd_unref(fd);
+    loc_wipe(&loc);
 
-	glfs_subvol_done (glfd->fs, subvol);
+    if (ret == -1) {
+        if (warn_deprecated && allocpath)
+            free(allocpath);
+        else if (allocpath)
+            GLFS_FREE(allocpath);
+        retpath = NULL;
+    }
 
-        __GLFS_EXIT_FS;
+    glfs_subvol_done(fs, subvol);
+
+    __GLFS_EXIT_FS;
 
 invalid_fs:
-	return ret;
+    return retpath;
 }
 
-GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_discard, 3.5.0);
+GFAPI_SYMVER_PUBLIC(glfs_realpath34, glfs_realpath, 3.4.0)
+char *
+pub_glfs_realpath34(struct glfs *fs, const char *path, char *resolved_path)
+{
+    return glfs_realpath_common(fs, path, resolved_path, _gf_true);
+}
 
+GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_realpath, 3.7.17)
+char *
+pub_glfs_realpath(struct glfs *fs, const char *path, char *resolved_path)
+{
+    return glfs_realpath_common(fs, path, resolved_path, _gf_false);
+}
 
-int
-pub_glfs_zerofill (struct glfs_fd *glfd, off_t offset, off_t len)
+GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_getcwd, 3.4.0)
+char *
+pub_glfs_getcwd(struct glfs *fs, char *buf, size_t n)
 {
-        int               ret             = -1;
-        xlator_t         *subvol          = NULL;
-        fd_t             *fd              = NULL;
+    int ret = -1;
+    inode_t *inode = NULL;
+    char *path = NULL;
 
-        DECLARE_OLD_THIS;
-        __GLFS_ENTRY_VALIDATE_FD (glfd, invalid_fs);
+    DECLARE_OLD_THIS;
+    __GLFS_ENTRY_VALIDATE_FS(fs, invalid_fs);
 
-        subvol = glfs_active_subvol (glfd->fs);
-        if (!subvol) {
-                errno = EIO;
-                goto out;
-        }
+    if (!buf || n < 2) {
+        ret = -1;
+        errno = EINVAL;
+        goto out;
+    }
 
-        fd = glfs_resolve_fd (glfd->fs, subvol, glfd);
-        if (!fd) {
-                errno = EBADFD;
-                goto out;
-        }
+    inode = glfs_cwd_get(fs);
 
-        ret = syncop_zerofill (subvol, fd, offset, len, NULL, NULL);
-        DECODE_SYNCOP_ERR (ret);
+    if (!inode) {
+        strncpy(buf, "/", n);
+        ret = 0;
+        goto out;
+    }
+
+    ret = inode_path(inode, 0, &path);
+    if (n <= ret) {
+        ret = -1;
+        errno = ERANGE;
+        goto out;
+    }
+
+    strncpy(buf, path, n);
+    ret = 0;
 out:
-        if (fd)
-                fd_unref(fd);
+    GF_FREE(path);
 
-        glfs_subvol_done (glfd->fs, subvol);
+    if (inode)
+        inode_unref(inode);
 
-        __GLFS_EXIT_FS;
+    __GLFS_EXIT_FS;
 
 invalid_fs:
-        return ret;
+    if (ret < 0)
+        return NULL;
+
+    return buf;
 }
 
-GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_zerofill, 3.5.0);
+static void
+gf_flock_to_flock(struct gf_flock *gf_flock, struct flock *flock)
+{
+    flock->l_type = gf_flock->l_type;
+    flock->l_whence = gf_flock->l_whence;
+    flock->l_start = gf_flock->l_start;
+    flock->l_len = gf_flock->l_len;
+    flock->l_pid = gf_flock->l_pid;
+}
 
+static void
+gf_flock_from_flock(struct gf_flock *gf_flock, struct flock *flock)
+{
+    gf_flock->l_type = flock->l_type;
+    gf_flock->l_whence = flock->l_whence;
+    gf_flock->l_start = flock->l_start;
+    gf_flock->l_len = flock->l_len;
+    gf_flock->l_pid = flock->l_pid;
+}
 
-int
-pub_glfs_chdir (struct glfs *fs, const char *path)
-{
-	int              ret = -1;
-	xlator_t        *subvol = NULL;
-	loc_t            loc = {0, };
-	struct iatt      iatt = {0, };
-	int              reval = 0;
-
-        DECLARE_OLD_THIS;
-        __GLFS_ENTRY_VALIDATE_FS (fs, invalid_fs);
-
-	subvol = glfs_active_subvol (fs);
-	if (!subvol) {
-		ret = -1;
-		errno = EIO;
-		goto out;
-	}
-retry:
-	ret = glfs_resolve (fs, subvol, path, &loc, &iatt, reval);
+static int
+glfs_lock_common(struct glfs_fd *glfd, int cmd, struct flock *flock,
+                 dict_t *xdata)
+{
+    int ret = -1;
+    xlator_t *subvol = NULL;
+    struct gf_flock gf_flock = {
+        0,
+    };
+    struct gf_flock saved_flock = {
+        0,
+    };
+    fd_t *fd = NULL;
+
+    DECLARE_OLD_THIS;
+    __GLFS_ENTRY_VALIDATE_FD(glfd, invalid_fs);
+
+    if (!flock) {
+        errno = EINVAL;
+        goto out;
+    }
+
+    GF_REF_GET(glfd);
+    subvol = glfs_active_subvol(glfd->fs);
+    if (!subvol) {
+        ret = -1;
+        errno = EIO;
+        goto out;
+    }
+
+    fd = glfs_resolve_fd(glfd->fs, subvol, glfd);
+    if (!fd) {
+        ret = -1;
+        errno = EBADFD;
+        goto out;
+    }
+
+    /* Generate glusterfs flock structure from client flock
+     * structure to be processed by server */
+    gf_flock_from_flock(&gf_flock, flock);
+
+    /* Keep another copy of flock for split/merge of locks
+     * at client side */
+    gf_flock_from_flock(&saved_flock, flock);
+
+    if (glfd->lk_owner.len != 0) {
+        ret = syncopctx_setfslkowner(&glfd->lk_owner);
+
+        if (ret)
+            goto out;
+    }
 
-	ESTALE_RETRY (ret, errno, reval, &loc, retry);
+    ret = get_fop_attr_thrd_key(&xdata);
+    if (ret)
+        gf_msg_debug("gfapi", 0, "Getting leaseid from thread failed");
 
-	if (ret)
-		goto out;
+    ret = syncop_lk(subvol, fd, cmd, &gf_flock, xdata, NULL);
+    DECODE_SYNCOP_ERR(ret);
 
-	if (!IA_ISDIR (iatt.ia_type)) {
-		ret = -1;
-		errno = ENOTDIR;
-		goto out;
-	}
+    /* Convert back from gf_flock to flock as expected by application */
+    gf_flock_to_flock(&gf_flock, flock);
 
-	glfs_cwd_set (fs, loc.inode);
+    if (ret == 0 && (cmd == F_SETLK || cmd == F_SETLKW)) {
+        ret = fd_lk_insert_and_merge(fd, cmd, &saved_flock);
+        if (ret) {
+            gf_smsg(THIS->name, GF_LOG_ERROR, 0,
+                    API_MSG_LOCK_INSERT_MERGE_FAILED, "gfid=%s",
+                    uuid_utoa(fd->inode->gfid), NULL);
+            ret = 0;
+        }
+    }
 
 out:
-	loc_wipe (&loc);
+    if (fd)
+        fd_unref(fd);
+    if (glfd)
+        GF_REF_PUT(glfd);
 
-	glfs_subvol_done (fs, subvol);
+    glfs_subvol_done(glfd->fs, subvol);
 
-        __GLFS_EXIT_FS;
+    __GLFS_EXIT_FS;
 
 invalid_fs:
-	return ret;
+    return ret;
 }
 
-GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_chdir, 3.4.0);
+GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_file_lock, 4.0.0)
+int
+pub_glfs_file_lock(struct glfs_fd *glfd, int cmd, struct flock *flock,
+                   glfs_lock_mode_t lk_mode)
+{
+    int ret = -1;
+    dict_t *xdata_in = NULL;
+
+    if (lk_mode == GLFS_LK_MANDATORY) {
+        /* Create a new dictionary */
+        xdata_in = dict_new();
+        if (xdata_in == NULL) {
+            ret = -1;
+            errno = ENOMEM;
+            goto out;
+        }
 
+        /* Set GF_LK_MANDATORY internally within dictionary to map
+         * GLFS_LK_MANDATORY */
+        ret = dict_set_uint32(xdata_in, GF_LOCK_MODE, GF_LK_MANDATORY);
+        if (ret) {
+            gf_smsg(THIS->name, GF_LOG_ERROR, 0,
+                    API_MSG_SETTING_LOCK_TYPE_FAILED, NULL);
+            ret = -1;
+            errno = ENOMEM;
+            goto out;
+        }
+    }
 
-int
-pub_glfs_fchdir (struct glfs_fd *glfd)
-{
-	int       ret = -1;
-	inode_t  *inode = NULL;
-	xlator_t *subvol = NULL;
-	fd_t     *fd = NULL;
-
-        DECLARE_OLD_THIS;
-	__GLFS_ENTRY_VALIDATE_FD (glfd, invalid_fs);
-
-	subvol = glfs_active_subvol (glfd->fs);
-	if (!subvol) {
-		ret = -1;
-		errno = EIO;
-		goto out;
-	}
-
-	fd = glfs_resolve_fd (glfd->fs, subvol, glfd);
-	if (!fd) {
-		ret = -1;
-		errno = EBADFD;
-		goto out;
-	}
-
-	inode = fd->inode;
-
-	if (!IA_ISDIR (inode->ia_type)) {
-		ret = -1;
-		errno = ENOTDIR;
-		goto out;
-	}
-
-	glfs_cwd_set (glfd->fs, inode);
-	ret = 0;
+    ret = glfs_lock_common(glfd, cmd, flock, xdata_in);
 out:
-	if (fd)
-		fd_unref (fd);
-
-	glfs_subvol_done (glfd->fs, subvol);
+    if (xdata_in)
+        dict_unref(xdata_in);
 
-        __GLFS_EXIT_FS;
+    return ret;
+}
 
-invalid_fs:
-	return ret;
+GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_posix_lock, 3.4.0)
+int
+pub_glfs_posix_lock(struct glfs_fd *glfd, int cmd, struct flock *flock)
+{
+    return glfs_lock_common(glfd, cmd, flock, NULL);
 }
 
-GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_fchdir, 3.4.0);
+GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_fd_set_lkowner, 3.10.7)
+int
+pub_glfs_fd_set_lkowner(struct glfs_fd *glfd, void *data, int len)
+{
+    int ret = -1;
 
+    DECLARE_OLD_THIS;
+    __GLFS_ENTRY_VALIDATE_FD(glfd, invalid_fs);
 
-char *
-pub_glfs_realpath (struct glfs *fs, const char *path, char *resolved_path)
-{
-	int              ret = -1;
-	char            *retpath = NULL;
-	char            *allocpath = NULL;
-	xlator_t        *subvol = NULL;
-	loc_t            loc = {0, };
-	struct iatt      iatt = {0, };
-	int              reval = 0;
-
-        DECLARE_OLD_THIS;
-        __GLFS_ENTRY_VALIDATE_FS (fs, invalid_fs);
-
-	if (resolved_path)
-		retpath = resolved_path;
-	else
-		retpath = allocpath = malloc (PATH_MAX + 1);
-
-	if (!retpath) {
-		ret = -1;
-		errno = ENOMEM;
-		goto out;
-	}
-
-	subvol = glfs_active_subvol (fs);
-	if (!subvol) {
-		ret = -1;
-		errno = EIO;
-		goto out;
-	}
-retry:
-	ret = glfs_resolve (fs, subvol, path, &loc, &iatt, reval);
+    if (!GF_REF_GET(glfd)) {
+        goto invalid_fs;
+    }
+
+    GF_VALIDATE_OR_GOTO(THIS->name, data, out);
 
-	ESTALE_RETRY (ret, errno, reval, &loc, retry);
+    if ((len <= 0) || (len > GFAPI_MAX_LOCK_OWNER_LEN)) {
+        errno = EINVAL;
+        gf_smsg(THIS->name, GF_LOG_ERROR, errno, API_MSG_INVALID_ARG,
+                "lk_owner len=%d", len, NULL);
+        goto out;
+    }
 
-	if (ret)
-		goto out;
+    glfd->lk_owner.len = len;
 
-	if (loc.path) {
-		strncpy (retpath, loc.path, PATH_MAX);
-		retpath[PATH_MAX] = 0;
-	}
+    memcpy(glfd->lk_owner.data, data, len);
 
+    ret = 0;
 out:
-	loc_wipe (&loc);
+    if (glfd)
+        GF_REF_PUT(glfd);
 
-	if (ret == -1) {
-		if (allocpath)
-			free (allocpath);
-		retpath = NULL;
-	}
+    __GLFS_EXIT_FS;
+
+invalid_fs:
+    return ret;
+}
+
+GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_dup, 3.4.0)
+struct glfs_fd *
+pub_glfs_dup(struct glfs_fd *glfd)
+{
+    xlator_t *subvol = NULL;
+    fd_t *fd = NULL;
+    struct glfs_fd *dupfd = NULL;
+    struct glfs *fs = NULL;
+
+    DECLARE_OLD_THIS;
+    __GLFS_ENTRY_VALIDATE_FD(glfd, invalid_fs);
+
+    GF_REF_GET(glfd);
+
+    fs = glfd->fs;
+    subvol = glfs_active_subvol(fs);
+    if (!subvol) {
+        errno = EIO;
+        goto out;
+    }
+
+    fd = glfs_resolve_fd(fs, subvol, glfd);
+    if (!fd) {
+        errno = EBADFD;
+        goto out;
+    }
+
+    dupfd = glfs_fd_new(fs);
+    if (!dupfd) {
+        errno = ENOMEM;
+        goto out;
+    }
+
+    dupfd->fd = fd_ref(fd);
+    dupfd->state = glfd->state;
+out:
+    if (fd)
+        fd_unref(fd);
+    if (dupfd)
+        glfs_fd_bind(dupfd);
+    if (glfd)
+        GF_REF_PUT(glfd);
 
-	glfs_subvol_done (fs, subvol);
+    glfs_subvol_done(fs, subvol);
 
-        __GLFS_EXIT_FS;
+    __GLFS_EXIT_FS;
 
 invalid_fs:
-	return retpath;
+    return dupfd;
 }
 
-GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_realpath, 3.4.0);
+static void
+glfs_enqueue_upcall_data(struct glfs *fs, struct gf_upcall *upcall_data)
+{
+    int ret = -1;
+    upcall_entry *u_list = NULL;
 
+    if (!fs || !upcall_data)
+        goto out;
+
+    u_list = GF_CALLOC(1, sizeof(*u_list), glfs_mt_upcall_entry_t);
+
+    if (!u_list) {
+        gf_smsg(THIS->name, GF_LOG_ERROR, ENOMEM, API_MSG_ALLOC_FAILED, "entry",
+                NULL);
+        goto out;
+    }
+
+    INIT_LIST_HEAD(&u_list->upcall_list);
+
+    gf_uuid_copy(u_list->upcall_data.gfid, upcall_data->gfid);
+    u_list->upcall_data.event_type = upcall_data->event_type;
+
+    switch (upcall_data->event_type) {
+        case GF_UPCALL_CACHE_INVALIDATION:
+            ret = glfs_get_upcall_cache_invalidation(&u_list->upcall_data,
+                                                     upcall_data);
+            break;
+        case GF_UPCALL_RECALL_LEASE:
+            ret = glfs_get_upcall_lease(&u_list->upcall_data, upcall_data);
+            break;
+        default:
+            break;
+    }
+
+    if (ret) {
+        gf_smsg(THIS->name, GF_LOG_ERROR, errno, API_MSG_INVALID_ENTRY, NULL);
+        goto out;
+    }
+
+    pthread_mutex_lock(&fs->upcall_list_mutex);
+    {
+        list_add_tail(&u_list->upcall_list, &fs->upcall_list);
+    }
+    pthread_mutex_unlock(&fs->upcall_list_mutex);
+
+    ret = 0;
 
-char *
-pub_glfs_getcwd (struct glfs *fs, char *buf, size_t n)
-{
-	int              ret = -1;
-	inode_t         *inode = NULL;
-	char            *path = NULL;
-
-        DECLARE_OLD_THIS;
-        __GLFS_ENTRY_VALIDATE_FS (fs, invalid_fs);
-
-	if (!buf || n < 2) {
-		ret = -1;
-		errno = EINVAL;
-		goto out;
-	}
-
-	inode = glfs_cwd_get (fs);
-
-	if (!inode) {
-		strncpy (buf, "/", n);
-		ret = 0;
-		goto out;
-	}
-
-	ret = inode_path (inode, 0, &path);
-	if (n <= ret) {
-		ret = -1;
-		errno = ERANGE;
-		goto out;
-	}
-
-	strncpy (buf, path, n);
-	ret = 0;
 out:
-	GF_FREE (path);
+    if (ret && u_list) {
+        GF_FREE(u_list->upcall_data.data);
+        GF_FREE(u_list);
+    }
+}
 
-	if (inode)
-		inode_unref (inode);
+static void
+glfs_free_upcall_lease(void *to_free)
+{
+    struct glfs_upcall_lease *arg = to_free;
 
-        __GLFS_EXIT_FS;
+    if (!arg)
+        return;
 
-invalid_fs:
-	if (ret < 0)
-		return NULL;
+    if (arg->object)
+        glfs_h_close(arg->object);
 
-	return buf;
+    GF_FREE(arg);
 }
 
-GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_getcwd, 3.4.0);
+int
+glfs_recall_lease_fd(struct glfs *fs, struct gf_upcall *up_data)
+{
+    struct gf_upcall_recall_lease *recall_lease = NULL;
+    xlator_t *subvol = NULL;
+    int ret = 0;
+    inode_t *inode = NULL;
+    struct glfs_fd *glfd = NULL;
+    struct glfs_fd *tmp = NULL;
+    struct list_head glfd_list;
+    fd_t *fd = NULL;
+    uint64_t value = 0;
+    struct glfs_lease lease = {
+        0,
+    };
+
+    GF_VALIDATE_OR_GOTO("gfapi", up_data, out);
+    GF_VALIDATE_OR_GOTO("gfapi", fs, out);
+
+    recall_lease = up_data->data;
+    GF_VALIDATE_OR_GOTO("gfapi", recall_lease, out);
+
+    INIT_LIST_HEAD(&glfd_list);
+
+    subvol = glfs_active_subvol(fs);
+    if (!subvol) {
+        ret = -1;
+        errno = EIO;
+        goto out;
+    }
+
+    gf_msg_debug(THIS->name, 0, "Recall lease received for gfid:%s",
+                 uuid_utoa(up_data->gfid));
+
+    inode = inode_find(subvol->itable, up_data->gfid);
+    if (!inode) {
+        ret = -1;
+        gf_smsg(THIS->name, GF_LOG_ERROR, errno, API_MSG_INODE_FIND_FAILED,
+                "gfid=%s", uuid_utoa(up_data->gfid), "graph_id=%d",
+                subvol->graph->id, NULL);
+        goto out;
+    }
+
+    LOCK(&inode->lock);
+    {
+        list_for_each_entry(fd, &inode->fd_list, inode_list)
+        {
+            ret = fd_ctx_get(fd, subvol, &value);
+            glfd = (struct glfs_fd *)(uintptr_t)value;
+            if (glfd) {
+                gf_msg_trace(THIS->name, 0, "glfd (%p) has held lease", glfd);
+                GF_REF_GET(glfd);
+                list_add_tail(&glfd->list, &glfd_list);
+            }
+        }
+    }
+    UNLOCK(&inode->lock);
 
+    if (!list_empty(&glfd_list)) {
+        list_for_each_entry_safe(glfd, tmp, &glfd_list, list)
+        {
+            LOCK(&glfd->lock);
+            {
+                if (glfd->state != GLFD_CLOSE) {
+                    gf_msg_trace(THIS->name, 0,
+                                 "glfd (%p) has held lease, "
+                                 "calling recall cbk",
+                                 glfd);
+                    glfd->cbk(lease, glfd->cookie);
+                }
+            }
+            UNLOCK(&glfd->lock);
 
-static void
-gf_flock_to_flock (struct gf_flock *gf_flock, struct flock *flock)
+            list_del_init(&glfd->list);
+            GF_REF_PUT(glfd);
+        }
+    }
+
+out:
+    return ret;
+}
+
+static int
+glfs_recall_lease_upcall(struct glfs *fs, struct glfs_upcall *up_arg,
+                         struct gf_upcall *up_data)
 {
-	flock->l_type   = gf_flock->l_type;
-	flock->l_whence = gf_flock->l_whence;
-	flock->l_start  = gf_flock->l_start;
-	flock->l_len    = gf_flock->l_len;
-	flock->l_pid    = gf_flock->l_pid;
+    struct gf_upcall_recall_lease *recall_lease = NULL;
+    struct glfs_object *object = NULL;
+    xlator_t *subvol = NULL;
+    int ret = -1;
+    struct glfs_upcall_lease *up_lease_arg = NULL;
+
+    GF_VALIDATE_OR_GOTO("gfapi", up_data, out);
+    GF_VALIDATE_OR_GOTO("gfapi", fs, out);
+
+    recall_lease = up_data->data;
+    GF_VALIDATE_OR_GOTO("gfapi", recall_lease, out);
+
+    subvol = glfs_active_subvol(fs);
+    if (!subvol) {
+        errno = EIO;
+        goto out;
+    }
+
+    gf_msg_debug(THIS->name, 0, "Recall lease received for gfid:%s",
+                 uuid_utoa(up_data->gfid));
+
+    object = glfs_h_find_handle(fs, up_data->gfid, GFAPI_HANDLE_LENGTH);
+    if (!object) {
+        /* The reason handle creation will fail is because we
+         * couldn't find the inode in the gfapi inode table.
+         *
+         * But since application would have taken inode_ref, the
+         * only case when this can happen is when it has closed
+         * the handle and hence will no more be interested in
+         * the upcall for this particular gfid.
+         */
+        gf_smsg(THIS->name, GF_LOG_DEBUG, errno, API_MSG_CREATE_HANDLE_FAILED,
+                "gfid=%s", uuid_utoa(up_data->gfid), NULL);
+        errno = ESTALE;
+        goto out;
+    }
+
+    up_lease_arg = GF_CALLOC(1, sizeof(struct glfs_upcall_lease),
+                             glfs_mt_upcall_inode_t);
+    up_lease_arg->object = object;
+
+    GF_VALIDATE_OR_GOTO("glfs_recall_lease", up_lease_arg, out);
+
+    up_lease_arg->lease_type = recall_lease->lease_type;
+
+    up_arg->reason = GLFS_UPCALL_RECALL_LEASE;
+    up_arg->event = up_lease_arg;
+    up_arg->free_event = glfs_free_upcall_lease;
+
+    ret = 0;
+
+out:
+    if (ret) {
+        /* Close p_object and oldp_object as well if being referenced.*/
+        if (object)
+            glfs_h_close(object);
+
+        /* Set reason to prevent applications from using ->event */
+        up_arg->reason = GF_UPCALL_EVENT_NULL;
+    }
+    return ret;
 }
 
+static int
+upcall_syncop_args_free(struct upcall_syncop_args *args)
+{
+    dict_t *dict = NULL;
+    struct gf_upcall *upcall_data = NULL;
 
-static void
-gf_flock_from_flock (struct gf_flock *gf_flock, struct flock *flock)
+    if (args) {
+        upcall_data = &args->upcall_data;
+        switch (upcall_data->event_type) {
+            case GF_UPCALL_CACHE_INVALIDATION:
+                dict = ((struct gf_upcall_cache_invalidation *)(upcall_data
+                                                                    ->data))
+                           ->dict;
+                break;
+            case GF_UPCALL_RECALL_LEASE:
+                dict = ((struct gf_upcall_recall_lease *)(upcall_data->data))
+                           ->dict;
+                break;
+        }
+        if (dict)
+            dict_unref(dict);
+
+        GF_FREE(upcall_data->client_uid);
+        GF_FREE(upcall_data->data);
+    }
+    GF_FREE(args);
+    return 0;
+}
+
+static int
+glfs_upcall_syncop_cbk(int ret, call_frame_t *frame, void *opaque)
 {
-	gf_flock->l_type   = flock->l_type;
-	gf_flock->l_whence = flock->l_whence;
-	gf_flock->l_start  = flock->l_start;
-	gf_flock->l_len    = flock->l_len;
-	gf_flock->l_pid    = flock->l_pid;
+    struct upcall_syncop_args *args = opaque;
+
+    (void)upcall_syncop_args_free(args);
+
+    return 0;
 }
 
+static int
+glfs_cbk_upcall_syncop(void *opaque)
+{
+    struct upcall_syncop_args *args = opaque;
+    struct gf_upcall *upcall_data = NULL;
+    struct glfs_upcall *up_arg = NULL;
+    struct glfs *fs;
+    int ret = -1;
+
+    fs = args->fs;
+    upcall_data = &args->upcall_data;
+
+    if (!upcall_data) {
+        goto out;
+    }
+
+    up_arg = GLFS_CALLOC(1, sizeof(struct gf_upcall), glfs_release_upcall,
+                         glfs_mt_upcall_entry_t);
+    if (!up_arg) {
+        gf_smsg(THIS->name, GF_LOG_ERROR, ENOMEM, API_MSG_ALLOC_FAILED, "entry",
+                NULL);
+        goto out;
+    }
+
+    switch (upcall_data->event_type) {
+        case GF_UPCALL_CACHE_INVALIDATION:
+            ret = glfs_h_poll_cache_invalidation(fs, up_arg, upcall_data);
+            break;
+        case GF_UPCALL_RECALL_LEASE:
+            ret = glfs_recall_lease_upcall(fs, up_arg, upcall_data);
+            break;
+        default:
+            errno = EINVAL;
+    }
+
+    /* It could so happen that the file which got
+     * upcall notification may have got deleted by
+     * the same client. In such cases up_arg->reason
+     * is set to GLFS_UPCALL_EVENT_NULL. No need to
+     * send upcall then
+     */
+    if (up_arg->reason == GLFS_UPCALL_EVENT_NULL) {
+        gf_smsg(THIS->name, GF_LOG_DEBUG, errno,
+                API_MSG_UPCALL_EVENT_NULL_RECEIVED, NULL);
+        ret = 0;
+        GLFS_FREE(up_arg);
+        goto out;
+    } else if (ret) {
+        gf_smsg(THIS->name, GF_LOG_ERROR, errno, API_MSG_INVALID_ENTRY, NULL);
+        GLFS_FREE(up_arg);
+        goto out;
+    }
+
+    if (fs->up_cbk && up_arg)
+        (fs->up_cbk)(up_arg, fs->up_data);
+
+    /* application takes care of calling glfs_free on up_arg post
+     * their processing */
 
-int
-pub_glfs_posix_lock (struct glfs_fd *glfd, int cmd, struct flock *flock)
-{
-	int              ret = -1;
-	xlator_t        *subvol = NULL;
-	struct gf_flock  gf_flock = {0, };
-	struct gf_flock  saved_flock = {0, };
-	fd_t            *fd = NULL;
-
-        DECLARE_OLD_THIS;
-	__GLFS_ENTRY_VALIDATE_FD (glfd, invalid_fs);
-
-	subvol = glfs_active_subvol (glfd->fs);
-	if (!subvol) {
-		ret = -1;
-		errno = EIO;
-		goto out;
-	}
-
-	fd = glfs_resolve_fd (glfd->fs, subvol, glfd);
-	if (!fd) {
-		ret = -1;
-		errno = EBADFD;
-		goto out;
-	}
-
-	gf_flock_from_flock (&gf_flock, flock);
-	gf_flock_from_flock (&saved_flock, flock);
-	ret = syncop_lk (subvol, fd, cmd, &gf_flock, NULL, NULL);
-        DECODE_SYNCOP_ERR (ret);
-	gf_flock_to_flock (&gf_flock, flock);
-
-	if (ret == 0 && (cmd == F_SETLK || cmd == F_SETLKW))
-		fd_lk_insert_and_merge (fd, cmd, &saved_flock);
 out:
-	if (fd)
-		fd_unref (fd);
+    return ret;
+}
 
-	glfs_subvol_done (glfd->fs, subvol);
+static struct gf_upcall_cache_invalidation *
+gf_copy_cache_invalidation(struct gf_upcall_cache_invalidation *src)
+{
+    struct gf_upcall_cache_invalidation *dst = NULL;
 
-        __GLFS_EXIT_FS;
+    if (!src)
+        goto out;
 
-invalid_fs:
-	return ret;
-}
+    dst = GF_CALLOC(1, sizeof(struct gf_upcall_cache_invalidation),
+                    glfs_mt_upcall_entry_t);
 
-GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_posix_lock, 3.4.0);
+    if (!dst) {
+        gf_smsg(THIS->name, GF_LOG_ERROR, ENOMEM, API_MSG_ALLOC_FAILED, "entry",
+                NULL);
+        goto out;
+    }
 
+    dst->flags = src->flags;
+    dst->expire_time_attr = src->expire_time_attr;
+    dst->stat = src->stat;
+    dst->p_stat = src->p_stat;
+    dst->oldp_stat = src->oldp_stat;
 
-struct glfs_fd *
-pub_glfs_dup (struct glfs_fd *glfd)
-{
-	xlator_t  *subvol = NULL;
-	fd_t      *fd = NULL;
-	glfs_fd_t *dupfd = NULL;
-	struct glfs *fs = NULL;
-
-        DECLARE_OLD_THIS;
-	__GLFS_ENTRY_VALIDATE_FD (glfd, invalid_fs);
-
-	fs = glfd->fs;
-	subvol = glfs_active_subvol (fs);
-	if (!subvol) {
-		errno = EIO;
-		goto out;
-	}
-
-	fd = glfs_resolve_fd (fs, subvol, glfd);
-	if (!fd) {
-		errno = EBADFD;
-		goto out;
-	}
-
-	dupfd = glfs_fd_new (fs);
-	if (!dupfd) {
-		errno = ENOMEM;
-		goto out;
-	}
-
-	dupfd->fd = fd_ref (fd);
+    if (src->dict)
+        dst->dict = dict_copy_with_ref(src->dict, NULL);
+
+    return dst;
 out:
-	if (fd)
-		fd_unref (fd);
-	if (dupfd)
-		glfs_fd_bind (dupfd);
+    return NULL;
+}
+
+static struct gf_upcall_recall_lease *
+gf_copy_recall_lease(struct gf_upcall_recall_lease *src)
+{
+    struct gf_upcall_recall_lease *dst = NULL;
 
-	glfs_subvol_done (fs, subvol);
+    if (!src)
+        goto out;
 
-        __GLFS_EXIT_FS;
+    dst = GF_CALLOC(1, sizeof(struct gf_upcall_recall_lease),
+                    glfs_mt_upcall_entry_t);
 
-invalid_fs:
-	return dupfd;
+    if (!dst) {
+        gf_smsg(THIS->name, GF_LOG_ERROR, ENOMEM, API_MSG_ALLOC_FAILED, "entry",
+                NULL);
+        goto out;
+    }
+
+    dst->lease_type = src->lease_type;
+    memcpy(dst->tid, src->tid, 16);
+
+    if (src->dict)
+        dst->dict = dict_copy_with_ref(src->dict, NULL);
+
+    return dst;
+out:
+    return NULL;
 }
 
-GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_dup, 3.4.0);
+static struct upcall_syncop_args *
+upcall_syncop_args_init(struct glfs *fs, struct gf_upcall *upcall_data)
+{
+    struct upcall_syncop_args *args = NULL;
+    int ret = -1;
+    struct gf_upcall *t_data = NULL;
+
+    if (!fs || !upcall_data)
+        goto out;
+
+    args = GF_CALLOC(1, sizeof(struct upcall_syncop_args),
+                     glfs_mt_upcall_entry_t);
+    if (!args) {
+        gf_smsg(THIS->name, GF_LOG_ERROR, ENOMEM, API_MSG_ALLOC_FAILED,
+                "syncop args", NULL);
+        goto out;
+    }
+
+    /* Note: we are not taking any ref on fs here.
+     * Ideally applications have to unregister for upcall events
+     * or stop polling for upcall events before performing
+     * glfs_fini. And as for outstanding synctasks created, we wait
+     * for all syncenv threads to finish tasks before cleaning up the
+     * fs->ctx. Hence it seems safe to process these callback
+     * notification without taking any lock/ref.
+     */
+    args->fs = fs;
+    t_data = &(args->upcall_data);
+    t_data->client_uid = gf_strdup(upcall_data->client_uid);
+
+    gf_uuid_copy(t_data->gfid, upcall_data->gfid);
+    t_data->event_type = upcall_data->event_type;
+
+    switch (t_data->event_type) {
+        case GF_UPCALL_CACHE_INVALIDATION:
+            t_data->data = gf_copy_cache_invalidation(
+                (struct gf_upcall_cache_invalidation *)upcall_data->data);
+            break;
+        case GF_UPCALL_RECALL_LEASE:
+            t_data->data = gf_copy_recall_lease(
+                (struct gf_upcall_recall_lease *)upcall_data->data);
+            break;
+    }
+
+    if (!t_data->data)
+        goto out;
+
+    return args;
+out:
+    if (ret) {
+        if (args) {
+            GF_FREE(args->upcall_data.client_uid);
+            GF_FREE(args);
+        }
+    }
+
+    return NULL;
+}
+
+static void
+glfs_cbk_upcall_data(struct glfs *fs, struct gf_upcall *upcall_data)
+{
+    struct upcall_syncop_args *args = NULL;
+    int ret = -1;
+
+    if (!fs || !upcall_data)
+        goto out;
+
+    if (!(fs->upcall_events & upcall_data->event_type)) {
+        /* ignore events which application hasn't registered*/
+        goto out;
+    }
+
+    args = upcall_syncop_args_init(fs, upcall_data);
+
+    if (!args)
+        goto out;
+
+    ret = synctask_new(THIS->ctx->env, glfs_cbk_upcall_syncop,
+                       glfs_upcall_syncop_cbk, NULL, args);
+    /* should we retry incase of failure? */
+    if (ret) {
+        gf_smsg(THIS->name, GF_LOG_ERROR, errno, API_MSG_UPCALL_SYNCOP_FAILED,
+                "event_type=%d", upcall_data->event_type, "gfid=%s",
+                (char *)(upcall_data->gfid), NULL);
+        upcall_syncop_args_free(args);
+    }
+
+out:
+    return;
+}
 
 /*
  * This routine is called in case of any notification received
  * from the server. All the upcall events are queued up in a list
  * to be read by the applications.
  *
- * XXX: Applications may register a cbk function for each 'fs'
- * which then needs to be called by this routine incase of any
- * event received. The cbk fn is responsible for notifying the
+ * In case if the application registers a cbk function, that shall
+ * be called by this routine in case of any event received.
+ * The cbk fn is responsible for notifying the
  * applications the way it desires for each event queued (for eg.,
  * can raise a signal or broadcast a cond variable etc.)
+ *
+ * Otherwise all the upcall events are queued up in a list
+ * to be read/polled by the applications.
  */
+GFAPI_SYMVER_PRIVATE_DEFAULT(glfs_process_upcall_event, 3.7.0)
 void
-priv_glfs_process_upcall_event (struct glfs *fs, void *data)
+priv_glfs_process_upcall_event(struct glfs *fs, void *data)
 {
-        int                ret             = -1;
-        upcall_entry       *u_list         = NULL;
-        glusterfs_ctx_t    *ctx            = NULL;
-        struct gf_upcall   *upcall_data    = NULL;
+    glusterfs_ctx_t *ctx = NULL;
+    struct gf_upcall *upcall_data = NULL;
 
-        gf_msg_debug (THIS->name, 0,
-                      "Upcall gfapi callback is called");
+    DECLARE_OLD_THIS;
 
-        if (!fs || !data)
-                goto out;
+    gf_msg_debug(THIS->name, 0, "Upcall gfapi callback is called");
 
-        /* Unlike in I/O path, "glfs_fini" would not have freed
-         * 'fs' by the time we take lock as it waits for all epoll
-         * threads to exit including this
-         */
-        pthread_mutex_lock (&fs->mutex);
-        {
-                ctx = fs->ctx;
+    __GLFS_ENTRY_VALIDATE_FS(fs, err);
 
-                if (ctx->cleanup_started) {
-                        pthread_mutex_unlock (&fs->mutex);
-                        goto out;
-                }
+    if (!data)
+        goto out;
+
+    /* Unlike in I/O path, "glfs_fini" would not have freed
+     * 'fs' by the time we take lock as it waits for all epoll
+     * threads to exit including this
+     */
+    pthread_mutex_lock(&fs->mutex);
+    {
+        ctx = fs->ctx;
 
-                fs->pin_refcnt++;
+        /* if we're not interested in upcalls (anymore), skip them */
+        if (ctx->cleanup_started || !fs->cache_upcalls) {
+            pthread_mutex_unlock(&fs->mutex);
+            goto out;
         }
-        pthread_mutex_unlock (&fs->mutex);
 
+        fs->pin_refcnt++;
+    }
+    pthread_mutex_unlock(&fs->mutex);
+
+    upcall_data = (struct gf_upcall *)data;
+
+    gf_msg_trace(THIS->name, 0, "Upcall gfapi gfid = %s",
+                 (char *)(upcall_data->gfid));
+
+    /* *
+     * TODO: RECALL LEASE for each glfd
+     *
+     * In case of RECALL_LEASE, we could associate separate
+     * cbk function for each glfd either by
+     * - extending pub_glfs_lease to accept new args (recall_cbk_fn, cookie)
+     * - or by defining new API "glfs_register_recall_cbk_fn (glfd,
+     * recall_cbk_fn, cookie) . In such cases, flag it and instead of calling
+     * below upcall functions, define a new one to go through the glfd list and
+     * invoke each of theirs recall_cbk_fn.
+     * */
+
+    if (fs->up_cbk) { /* upcall cbk registered */
+        (void)glfs_cbk_upcall_data(fs, upcall_data);
+    } else {
+        (void)glfs_enqueue_upcall_data(fs, upcall_data);
+    }
+
+    pthread_mutex_lock(&fs->mutex);
+    {
+        fs->pin_refcnt--;
+    }
+    pthread_mutex_unlock(&fs->mutex);
 
-        upcall_data = (struct gf_upcall *)data;
+out:
+    __GLFS_EXIT_FS;
+err:
+    return;
+}
 
-        gf_msg_trace (THIS->name, 0, "Upcall gfapi gfid = %s"
-                      "ret = %d", (char *)(upcall_data->gfid), ret);
+ssize_t
+glfs_anonymous_pwritev(struct glfs *fs, struct glfs_object *object,
+                       const struct iovec *iovec, int iovcnt, off_t offset,
+                       int flags)
+{
+    xlator_t *subvol = NULL;
+    struct iobref *iobref = NULL;
+    struct iobuf *iobuf = NULL;
+    struct iovec iov = {
+        0,
+    };
+    inode_t *inode = NULL;
+    fd_t *fd = NULL;
+    int ret = -1;
+    size_t size = -1;
+
+    DECLARE_OLD_THIS;
+    __GLFS_ENTRY_VALIDATE_FS(fs, invalid_fs);
+
+    subvol = glfs_active_subvol(fs);
+    if (!subvol) {
+        ret = -1;
+        errno = EIO;
+        goto out;
+    }
+
+    /* get/refresh the in arg objects inode in correlation to the xlator */
+    inode = glfs_resolve_inode(fs, subvol, object);
+    if (!inode) {
+        ret = -1;
+        errno = ESTALE;
+        goto out;
+    }
+
+    fd = fd_anonymous(inode);
+    if (!fd) {
+        ret = -1;
+        gf_smsg("gfapi", GF_LOG_ERROR, ENOMEM, API_MSG_FDCREATE_FAILED, NULL);
+        errno = ENOMEM;
+        goto out;
+    }
+
+    size = iov_length(iovec, iovcnt);
+
+    iobuf = iobuf_get2(subvol->ctx->iobuf_pool, size);
+    if (!iobuf) {
+        ret = -1;
+        errno = ENOMEM;
+        goto out;
+    }
+
+    iobref = iobref_new();
+    if (!iobref) {
+        iobuf_unref(iobuf);
+        errno = ENOMEM;
+        ret = -1;
+        goto out;
+    }
+
+    ret = iobref_add(iobref, iobuf);
+    if (ret) {
+        iobuf_unref(iobuf);
+        iobref_unref(iobref);
+        errno = ENOMEM;
+        ret = -1;
+        goto out;
+    }
+
+    iov_unload(iobuf_ptr(iobuf), iovec, iovcnt);
+
+    iov.iov_base = iobuf_ptr(iobuf);
+    iov.iov_len = size;
+
+    /* TODO : set leaseid */
+    ret = syncop_writev(subvol, fd, &iov, 1, offset, iobref, flags, NULL, NULL,
+                        NULL, NULL);
+    DECODE_SYNCOP_ERR(ret);
+
+    iobuf_unref(iobuf);
+    iobref_unref(iobref);
+
+    if (ret <= 0)
+        goto out;
 
-        u_list = GF_CALLOC (1, sizeof(*u_list),
-                            glfs_mt_upcall_entry_t);
+out:
 
-        if (!u_list) {
-                gf_msg (THIS->name, GF_LOG_ERROR, ENOMEM, API_MSG_ALLOC_FAILED,
-                        "Upcall entry allocation failed.");
-                goto out;
-        }
+    if (fd)
+        fd_unref(fd);
 
-        INIT_LIST_HEAD (&u_list->upcall_list);
+    if (inode)
+        inode_unref(inode);
 
-        gf_uuid_copy (u_list->upcall_data.gfid, upcall_data->gfid);
-        u_list->upcall_data.event_type = upcall_data->event_type;
+    glfs_subvol_done(fs, subvol);
 
-        switch (upcall_data->event_type) {
-        case GF_UPCALL_CACHE_INVALIDATION:
-                ret = glfs_get_upcall_cache_invalidation (&u_list->upcall_data,
-                                                          upcall_data);
-                break;
-        default:
-                goto out;
-        }
+    __GLFS_EXIT_FS;
 
-        if (ret) {
-                gf_msg (THIS->name, GF_LOG_ERROR, errno,
-                        API_MSG_INVALID_ENTRY,
-                        "Upcall entry validation failed.");
-                goto out;
-        }
+invalid_fs:
+    return ret;
+}
 
-        pthread_mutex_lock (&fs->upcall_list_mutex);
-        {
-                list_add_tail (&u_list->upcall_list,
-                               &fs->upcall_list);
-        }
-        pthread_mutex_unlock (&fs->upcall_list_mutex);
+ssize_t
+glfs_anonymous_preadv(struct glfs *fs, struct glfs_object *object,
+                      const struct iovec *iovec, int iovcnt, off_t offset,
+                      int flags)
+{
+    xlator_t *subvol = NULL;
+    struct iovec *iov = NULL;
+    struct iobref *iobref = NULL;
+    inode_t *inode = NULL;
+    fd_t *fd = NULL;
+    int cnt = 0;
+    ssize_t ret = -1;
+    ssize_t size = -1;
+
+    DECLARE_OLD_THIS;
+    __GLFS_ENTRY_VALIDATE_FS(fs, invalid_fs);
+
+    subvol = glfs_active_subvol(fs);
+    if (!subvol) {
+        ret = -1;
+        errno = EIO;
+        goto out;
+    }
+
+    /* get/refresh the in arg objects inode in correlation to the xlator */
+    inode = glfs_resolve_inode(fs, subvol, object);
+    if (!inode) {
+        ret = -1;
+        errno = ESTALE;
+        goto out;
+    }
+
+    fd = fd_anonymous(inode);
+    if (!fd) {
+        ret = -1;
+        gf_smsg("gfapi", GF_LOG_ERROR, ENOMEM, API_MSG_FDCREATE_FAILED, NULL);
+        errno = ENOMEM;
+        goto out;
+    }
+
+    size = iov_length(iovec, iovcnt);
+
+    /* TODO : set leaseid */
+    ret = syncop_readv(subvol, fd, size, offset, flags, &iov, &cnt, &iobref,
+                       NULL, NULL, NULL);
+    DECODE_SYNCOP_ERR(ret);
+    if (ret <= 0)
+        goto out;
+
+    size = iov_copy(iovec, iovcnt, iov, cnt);
+
+    ret = size;
+out:
+    if (iov)
+        GF_FREE(iov);
+    if (iobref)
+        iobref_unref(iobref);
+    if (fd)
+        fd_unref(fd);
 
-        pthread_mutex_lock (&fs->mutex);
-        {
-                fs->pin_refcnt--;
-        }
-        pthread_mutex_unlock (&fs->mutex);
+    if (inode)
+        inode_unref(inode);
 
-        ret = 0;
-out:
-        if (ret && u_list)
-                GF_FREE(u_list);
-        return;
+    glfs_subvol_done(fs, subvol);
+
+    __GLFS_EXIT_FS;
+
+invalid_fs:
+    return ret;
 }
 
-GFAPI_SYMVER_PRIVATE_DEFAULT(glfs_process_upcall_event, 3.7.0);
+static void
+glfs_release_xreaddirp_stat(void *ptr)
+{
+    struct glfs_xreaddirp_stat *to_free = ptr;
 
-ssize_t
-glfs_anonymous_pwritev (struct glfs *fs, struct glfs_object *object,
-                        const struct iovec *iovec, int iovcnt,
-                        off_t offset, int flags)
-{
-        xlator_t        *subvol = NULL;
-        struct iobref   *iobref = NULL;
-        struct iobuf    *iobuf  = NULL;
-        struct iovec    iov     = {0, };
-        inode_t         *inode  = NULL;
-        fd_t            *fd     = NULL;
-        int             ret     = -1;
-        size_t          size    = -1;
-
-        DECLARE_OLD_THIS;
-        __GLFS_ENTRY_VALIDATE_FS (fs, invalid_fs);
-
-        subvol = glfs_active_subvol (fs);
-        if (!subvol) {
-                ret = -1;
-                errno = EIO;
-                goto out;
-        }
+    if (to_free->object)
+        glfs_h_close(to_free->object);
+}
 
-        /* get/refresh the in arg objects inode in correlation to the xlator */
-        inode = glfs_resolve_inode (fs, subvol, object);
-        if (!inode) {
-                ret = -1;
-                errno = ESTALE;
-                goto out;
-        }
+/*
+ * Given glfd of a directory, this function does readdirp and returns
+ * xstat along with dirents.
+ */
+GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_xreaddirplus_r, 3.11.0)
+int
+pub_glfs_xreaddirplus_r(struct glfs_fd *glfd, uint32_t flags,
+                        struct glfs_xreaddirp_stat **xstat_p,
+                        struct dirent *ext, struct dirent **res)
+{
+    int ret = -1;
+    gf_dirent_t *entry = NULL;
+    struct dirent *buf = NULL;
+    struct glfs_xreaddirp_stat *xstat = NULL;
 
-        fd = fd_anonymous (inode);
-        if (!fd) {
-                ret = -1;
-                gf_msg ("gfapi", GF_LOG_ERROR, ENOMEM, API_MSG_FDCREATE_FAILED,
-                        "Allocating anonymous fd failed");
-                errno = ENOMEM;
-                goto out;
-        }
+    DECLARE_OLD_THIS;
+    __GLFS_ENTRY_VALIDATE_FD(glfd, invalid_fs);
 
-        size = iov_length (iovec, iovcnt);
+    GF_REF_GET(glfd);
 
-        iobuf = iobuf_get2 (subvol->ctx->iobuf_pool, size);
-        if (!iobuf) {
-                ret = -1;
-                errno = ENOMEM;
-                goto out;
-        }
+    GF_VALIDATE_OR_GOTO(THIS->name, xstat_p, out);
+    GF_VALIDATE_OR_GOTO(THIS->name, res, out);
 
-        iobref = iobref_new ();
-        if (!iobref) {
-                iobuf_unref (iobuf);
-                errno = ENOMEM;
-                ret = -1;
-                goto out;
-        }
+    errno = 0;
 
-        ret = iobref_add (iobref, iobuf);
-        if (ret) {
-                iobuf_unref (iobuf);
-                iobref_unref (iobref);
-                errno = ENOMEM;
-                ret = -1;
-                goto out;
-        }
+    if (ext)
+        buf = ext;
+    else
+        buf = glfs_readdirbuf_get(glfd);
 
-        iov_unload (iobuf_ptr (iobuf), iovec, iovcnt);
+    if (!buf)
+        goto out;
 
-        iov.iov_base = iobuf_ptr (iobuf);
-        iov.iov_len = size;
+    xstat = GLFS_CALLOC(1, sizeof(struct glfs_xreaddirp_stat),
+                        glfs_release_xreaddirp_stat, glfs_mt_xreaddirp_stat_t);
 
-        ret = syncop_writev (subvol, fd, &iov, 1, offset, iobref, flags,
-                             NULL, NULL);
-        DECODE_SYNCOP_ERR (ret);
+    if (!xstat)
+        goto out;
 
-        iobuf_unref (iobuf);
-        iobref_unref (iobref);
+    /* this is readdirplus operation */
+    entry = glfd_entry_next(glfd, 1);
 
-        if (ret <= 0)
-                goto out;
+    /* XXX: Ideally when we reach EOD, errno should have been
+     * set to ENOENT. But that doesn't seem to be the case.
+     *
+     * The only way to confirm if its EOD at this point is that
+     * errno == 0 and entry == NULL
+     */
+    if (errno)
+        goto out;
+
+    if (!entry) {
+        /* reached EOD, ret = 0  */
+        ret = 0;
+        *res = NULL;
+        *xstat_p = NULL;
+
+        /* free xstat as applications shall not be using it */
+        GLFS_FREE(xstat);
+
+        goto out;
+    }
+
+    *res = buf;
+    gf_dirent_to_dirent(entry, buf);
+
+    if (flags & GFAPI_XREADDIRP_STAT) {
+        glfs_iatt_to_stat(glfd->fs, &entry->d_stat, &xstat->st);
+        xstat->flags_handled |= GFAPI_XREADDIRP_STAT;
+    }
+
+    if ((flags & GFAPI_XREADDIRP_HANDLE) &&
+        /* skip . and .. */
+        strcmp(buf->d_name, ".") && strcmp(buf->d_name, "..")) {
+        /* Now create object.
+         * We can use "glfs_h_find_handle" as well as inodes would have
+         * already got linked as part of 'gf_link_inodes_from_dirent' */
+        xstat->object = glfs_h_create_from_handle(
+            glfd->fs, entry->d_stat.ia_gfid, GFAPI_HANDLE_LENGTH, NULL);
+
+        if (xstat->object) { /* success */
+            /* note: xstat->object->inode->ref is taken
+             * This shall be unref'ed when application does
+             * glfs_free(xstat) */
+            xstat->flags_handled |= GFAPI_XREADDIRP_HANDLE;
+        }
+    }
+
+    ret = xstat->flags_handled;
+    *xstat_p = xstat;
+
+    gf_msg_debug(THIS->name, 0,
+                 "xreaddirp- requested_flags (%x) , processed_flags (%x)",
+                 flags, xstat->flags_handled);
 
 out:
+    GF_REF_PUT(glfd);
 
-        if (fd)
-                fd_unref(fd);
+    if (ret < 0) {
+        gf_smsg(THIS->name, GF_LOG_WARNING, errno, API_MSG_XREADDIRP_R_FAILED,
+                "reason=%s", strerror(errno), NULL);
+
+        if (xstat)
+            GLFS_FREE(xstat);
+    }
 
-        glfs_subvol_done (fs, subvol);
+    __GLFS_EXIT_FS;
 
-        __GLFS_EXIT_FS;
+    return ret;
 
 invalid_fs:
-        return ret;
+    return -1;
 }
 
-ssize_t
-glfs_anonymous_preadv (struct glfs *fs,  struct glfs_object *object,
-                       const struct iovec *iovec, int iovcnt,
-                       off_t offset, int flags)
-{
-        xlator_t        *subvol = NULL;
-        struct iovec    *iov    = NULL;
-        struct iobref   *iobref = NULL;
-        inode_t         *inode  = NULL;
-        fd_t            *fd     = NULL;
-        int             cnt     = 0;
-        ssize_t         ret     = -1;
-        ssize_t         size    = -1;
-
-        DECLARE_OLD_THIS;
-        __GLFS_ENTRY_VALIDATE_FS (fs, invalid_fs);
-
-        subvol = glfs_active_subvol (fs);
-        if (!subvol) {
+GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_xreaddirplus_get_stat, 3.11.0)
+struct stat *
+pub_glfs_xreaddirplus_get_stat(struct glfs_xreaddirp_stat *xstat)
+{
+    GF_VALIDATE_OR_GOTO("glfs_xreaddirplus_get_stat", xstat, out);
+
+    if (!xstat->flags_handled & GFAPI_XREADDIRP_STAT)
+        gf_smsg(THIS->name, GF_LOG_ERROR, errno, API_MSG_FLAGS_HANDLE,
+                "GFAPI_XREADDIRP_STAT"
+                "xstat=%p",
+                xstat, "handles=%x", xstat->flags_handled, NULL);
+    return &xstat->st;
+
+out:
+    return NULL;
+}
+
+void
+gf_lease_to_glfs_lease(struct gf_lease *gf_lease, struct glfs_lease *lease)
+{
+    u_int lease_type = gf_lease->lease_type;
+    lease->cmd = gf_lease->cmd;
+    lease->lease_type = lease_type;
+    memcpy(lease->lease_id, gf_lease->lease_id, LEASE_ID_SIZE);
+}
+
+void
+glfs_lease_to_gf_lease(struct glfs_lease *lease, struct gf_lease *gf_lease)
+{
+    u_int lease_type = lease->lease_type;
+    gf_lease->cmd = lease->cmd;
+    gf_lease->lease_type = lease_type;
+    memcpy(gf_lease->lease_id, lease->lease_id, LEASE_ID_SIZE);
+}
+
+GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_lease, 4.0.0)
+int
+pub_glfs_lease(struct glfs_fd *glfd, struct glfs_lease *lease,
+               glfs_recall_cbk fn, void *data)
+{
+    int ret = -1;
+    loc_t loc = {
+        0,
+    };
+    xlator_t *subvol = NULL;
+    fd_t *fd = NULL;
+    struct gf_lease gf_lease = {
+        0,
+    };
+
+    DECLARE_OLD_THIS;
+    __GLFS_ENTRY_VALIDATE_FD(glfd, invalid_fs);
+
+    GF_REF_GET(glfd);
+
+    if (!is_valid_lease_id(lease->lease_id)) {
+        ret = -1;
+        errno = EINVAL;
+        goto out;
+    }
+
+    subvol = glfs_active_subvol(glfd->fs);
+    if (!subvol) {
+        ret = -1;
+        errno = EIO;
+        goto out;
+    }
+
+    fd = glfs_resolve_fd(glfd->fs, subvol, glfd);
+    if (!fd) {
+        ret = -1;
+        errno = EBADFD;
+        goto out;
+    }
+
+    switch (lease->lease_type) {
+        case GLFS_RD_LEASE:
+            if ((fd->flags != O_RDONLY) && !(fd->flags & O_RDWR)) {
                 ret = -1;
-                errno = EIO;
+                errno = EINVAL;
                 goto out;
-        }
-
-        /* get/refresh the in arg objects inode in correlation to the xlator */
-        inode = glfs_resolve_inode (fs, subvol, object);
-        if (!inode) {
+            }
+            break;
+        case GLFS_RW_LEASE:
+            if (!((fd->flags & O_WRONLY) || (fd->flags & O_RDWR))) {
                 ret = -1;
-                errno = ESTALE;
+                errno = EINVAL;
                 goto out;
-        }
-
-        fd = fd_anonymous (inode);
-        if (!fd) {
+            }
+            break;
+        default:
+            if (lease->cmd != GLFS_GET_LEASE) {
                 ret = -1;
-                gf_msg ("gfapi", GF_LOG_ERROR, ENOMEM, API_MSG_FDCREATE_FAILED,
-                        "Allocating anonymous fd failed");
-                errno = ENOMEM;
+                errno = EINVAL;
                 goto out;
-        }
+            }
+            break;
+    }
 
-        size = iov_length (iovec, iovcnt);
+    /* populate loc */
+    GLFS_LOC_FILL_INODE(fd->inode, loc, out);
 
-	ret = syncop_readv (subvol, fd, size, offset, flags, &iov, &cnt,
-                            &iobref, NULL, NULL);
-        DECODE_SYNCOP_ERR (ret);
-        if (ret <= 0)
-                goto out;
+    glfs_lease_to_gf_lease(lease, &gf_lease);
 
-        size = iov_copy (iovec, iovcnt, iov, cnt);
+    ret = syncop_lease(subvol, &loc, &gf_lease, NULL, NULL);
+    DECODE_SYNCOP_ERR(ret);
+
+    gf_lease_to_glfs_lease(&gf_lease, lease);
+
+    /* TODO: Add leases for client replay
+    if (ret == 0 && (cmd == F_SETLK || cmd == F_SETLKW))
+            fd_lk_insert_and_merge (fd, cmd, &saved_flock);
+    */
+    if (ret == 0) {
+        ret = fd_ctx_set(glfd->fd, subvol, (uint64_t)(long)glfd);
+        if (ret) {
+            gf_smsg(subvol->name, GF_LOG_ERROR, ENOMEM,
+                    API_MSG_FDCTX_SET_FAILED, "fd=%p", glfd->fd, NULL);
+            goto out;
+        }
+        glfd->cbk = fn;
+        glfd->cookie = data;
+    }
 
-        ret = size;
 out:
-        if (iov)
-                GF_FREE (iov);
-        if (iobref)
-                iobref_unref (iobref);
-        if (fd)
-                fd_unref(fd);
 
-        glfs_subvol_done (fs, subvol);
+    if (glfd)
+        GF_REF_PUT(glfd);
+
+    if (subvol)
+        glfs_subvol_done(glfd->fs, subvol);
 
-        __GLFS_EXIT_FS;
+    __GLFS_EXIT_FS;
 
 invalid_fs:
-        return ret;
+    return ret;
 }
diff --git a/api/src/glfs-handleops.c b/api/src/glfs-handleops.c
index e8fbae4074e..53c2ee896f9 100644
--- a/api/src/glfs-handleops.c
+++ b/api/src/glfs-handleops.c
@@ -1,5 +1,5 @@
 /*
- *  Copyright (c) 2013 Red Hat, Inc. <http://www.redhat.com>
+ *  Copyright (c) 2013-2018 Red Hat, Inc. <http://www.redhat.com>
  *  This file is part of GlusterFS.
  *
  *  This file is licensed to you under your choice of the GNU Lesser
@@ -8,2165 +8,2648 @@
  *  cases as published by the Free Software Foundation.
  */
 
-
 #include "glfs-internal.h"
 #include "glfs-mem-types.h"
-#include "syncop.h"
+#include <glusterfs/syncop.h>
 #include "glfs.h"
 #include "glfs-handles.h"
 #include "gfapi-messages.h"
 
 int
-glfs_listxattr_process (void *value, size_t size, dict_t *xattr);
+glfs_listxattr_process(void *value, size_t size, dict_t *xattr);
 
-static void
-glfs_iatt_from_stat (struct stat *stat, int valid, struct iatt *iatt,
-                     int *glvalid)
+void
+glfs_iatt_from_stat(struct stat *stat, int valid, struct iatt *iatt,
+                    int *glvalid)
 {
-        /* validate in args */
-        if ((stat == NULL) || (iatt == NULL) || (glvalid == NULL)) {
-                errno = EINVAL;
-                return;
-        }
-
-        *glvalid = 0;
-
-        if (valid & GFAPI_SET_ATTR_MODE) {
-                iatt->ia_prot = ia_prot_from_st_mode (stat->st_mode);
-                *glvalid |= GF_SET_ATTR_MODE;
-        }
-
-        if (valid & GFAPI_SET_ATTR_UID) {
-                iatt->ia_uid = stat->st_uid;
-                *glvalid |= GF_SET_ATTR_UID;
-        }
-
-        if (valid & GFAPI_SET_ATTR_GID) {
-                iatt->ia_gid = stat->st_gid;
-                *glvalid |= GF_SET_ATTR_GID;
-        }
-
-        if (valid & GFAPI_SET_ATTR_ATIME) {
-                iatt->ia_atime = stat->st_atime;
-                iatt->ia_atime_nsec = ST_ATIM_NSEC (stat);
-                *glvalid |= GF_SET_ATTR_ATIME;
-        }
-
-        if (valid & GFAPI_SET_ATTR_MTIME) {
-                iatt->ia_mtime = stat->st_mtime;
-                iatt->ia_mtime_nsec = ST_MTIM_NSEC (stat);
-                *glvalid |= GF_SET_ATTR_MTIME;
-        }
-
+    /* validate in args */
+    if ((stat == NULL) || (iatt == NULL) || (glvalid == NULL)) {
+        errno = EINVAL;
         return;
+    }
+
+    *glvalid = 0;
+
+    if (valid & GFAPI_SET_ATTR_MODE) {
+        iatt->ia_prot = ia_prot_from_st_mode(stat->st_mode);
+        *glvalid |= GF_SET_ATTR_MODE;
+    }
+
+    if (valid & GFAPI_SET_ATTR_UID) {
+        iatt->ia_uid = stat->st_uid;
+        *glvalid |= GF_SET_ATTR_UID;
+    }
+
+    if (valid & GFAPI_SET_ATTR_GID) {
+        iatt->ia_gid = stat->st_gid;
+        *glvalid |= GF_SET_ATTR_GID;
+    }
+
+    if (valid & GFAPI_SET_ATTR_ATIME) {
+        iatt->ia_atime = stat->st_atime;
+        iatt->ia_atime_nsec = ST_ATIM_NSEC(stat);
+        *glvalid |= GF_SET_ATTR_ATIME;
+    }
+
+    if (valid & GFAPI_SET_ATTR_MTIME) {
+        iatt->ia_mtime = stat->st_mtime;
+        iatt->ia_mtime_nsec = ST_MTIM_NSEC(stat);
+        *glvalid |= GF_SET_ATTR_MTIME;
+    }
+
+    return;
 }
 
+GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_h_lookupat, 3.7.4)
 struct glfs_object *
-pub_glfs_h_lookupat (struct glfs *fs, struct glfs_object *parent,
-                     const char *path, struct stat *stat, int follow)
+pub_glfs_h_lookupat(struct glfs *fs, struct glfs_object *parent,
+                    const char *path, struct stat *stat, int follow)
 {
-        int                      ret = 0;
-        xlator_t                *subvol = NULL;
-        inode_t                 *inode = NULL;
-        struct iatt              iatt = {0, };
-        struct glfs_object      *object = NULL;
-        loc_t                    loc = {0, };
-
-        DECLARE_OLD_THIS;
-
-        /* validate in args */
-        if (path == NULL) {
-                errno = EINVAL;
-                return NULL;
-        }
+    int ret = 0;
+    xlator_t *subvol = NULL;
+    inode_t *inode = NULL;
+    struct iatt iatt = {
+        0,
+    };
+    struct glfs_object *object = NULL;
+    loc_t loc = {
+        0,
+    };
+
+    DECLARE_OLD_THIS;
+
+    /* validate in args */
+    if (path == NULL) {
+        errno = EINVAL;
+        return NULL;
+    }
 
-        __GLFS_ENTRY_VALIDATE_FS (fs, invalid_fs);
+    __GLFS_ENTRY_VALIDATE_FS(fs, invalid_fs);
 
-        /* get the active volume */
-        subvol = glfs_active_subvol (fs);
-        if (!subvol) {
-                errno = EIO;
-                goto out;
-        }
+    /* get the active volume */
+    subvol = glfs_active_subvol(fs);
+    if (!subvol) {
+        errno = EIO;
+        goto out;
+    }
 
-        /* get/refresh the in arg objects inode in correlation to the xlator */
-        if (parent) {
-                inode = glfs_resolve_inode (fs, subvol, parent);
-                if (!inode) {
-                        errno = ESTALE;
-                        goto out;
-                }
+    /* get/refresh the in arg objects inode in correlation to the xlator */
+    if (parent) {
+        inode = glfs_resolve_inode(fs, subvol, parent);
+        if (!inode) {
+            errno = ESTALE;
+            goto out;
         }
+    }
 
-        /* fop/op */
-        ret = glfs_resolve_at (fs, subvol, inode, path, &loc, &iatt,
-                                    follow, 0);
+    /* fop/op */
+    ret = glfs_resolve_at(fs, subvol, inode, path, &loc, &iatt, follow, 0);
 
-        /* populate out args */
-        if (!ret) {
-                if (stat)
-                        glfs_iatt_to_stat (fs, &iatt, stat);
+    /* populate out args */
+    if (!ret) {
+        if (stat)
+            glfs_iatt_to_stat(fs, &iatt, stat);
 
-                ret = glfs_create_object (&loc, &object);
-        }
+        ret = glfs_create_object(&loc, &object);
+    }
 
 out:
-        loc_wipe (&loc);
+    loc_wipe(&loc);
 
-        if (inode)
-                inode_unref (inode);
+    if (inode)
+        inode_unref(inode);
 
-        glfs_subvol_done (fs, subvol);
+    glfs_subvol_done(fs, subvol);
 
-        __GLFS_EXIT_FS;
+    __GLFS_EXIT_FS;
 
 invalid_fs:
-        return object;
+    return object;
 }
 
-GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_h_lookupat, 3.7.4);
-
+GFAPI_SYMVER_PUBLIC(glfs_h_lookupat34, glfs_h_lookupat, 3.4.2)
 struct glfs_object *
-pub_glfs_h_lookupat34 (struct glfs *fs, struct glfs_object *parent,
-                       const char *path, struct stat *stat)
+pub_glfs_h_lookupat34(struct glfs *fs, struct glfs_object *parent,
+                      const char *path, struct stat *stat)
 {
-        return pub_glfs_h_lookupat (fs, parent, path, stat, 0);
+    return pub_glfs_h_lookupat(fs, parent, path, stat, 0);
 }
 
-GFAPI_SYMVER_PUBLIC(glfs_h_lookupat34, glfs_h_lookupat, 3.4.2);
-
+GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_h_statfs, 3.7.0)
 int
-pub_glfs_h_statfs (struct glfs *fs, struct glfs_object *object,
-                   struct statvfs *statvfs)
+pub_glfs_h_statfs(struct glfs *fs, struct glfs_object *object,
+                  struct statvfs *statvfs)
 {
-        int              ret = -1;
-        xlator_t        *subvol = NULL;
-        inode_t         *inode = NULL;
-        loc_t            loc = {0, };
+    int ret = -1;
+    xlator_t *subvol = NULL;
+    inode_t *inode = NULL;
+    loc_t loc = {
+        0,
+    };
+
+    DECLARE_OLD_THIS;
+
+    /* validate in args */
+    if ((fs == NULL) || (object == NULL || statvfs == NULL)) {
+        errno = EINVAL;
+        return -1;
+    }
 
-        DECLARE_OLD_THIS;
+    __GLFS_ENTRY_VALIDATE_FS(fs, invalid_fs);
 
-        /* validate in args */
-        if ((fs == NULL) || (object == NULL || statvfs == NULL)) {
-                errno = EINVAL;
-                return -1;
-        }
+    /* get the active volume */
+    subvol = glfs_active_subvol(fs);
+    if (!subvol) {
+        ret = -1;
+        errno = EIO;
+        goto out;
+    }
 
-        __GLFS_ENTRY_VALIDATE_FS (fs, invalid_fs);
+    /* get/refresh the in arg objects inode in correlation to the xlator */
+    inode = glfs_resolve_inode(fs, subvol, object);
+    if (!inode) {
+        errno = ESTALE;
+        goto out;
+    }
 
-        /* get the active volume */
-        subvol = glfs_active_subvol (fs);
-        if (!subvol) {
-                ret = -1;
-                errno = EIO;
-                goto out;
-        }
-
-        /* get/refresh the in arg objects inode in correlation to the xlator */
-        inode = glfs_resolve_inode (fs, subvol, object);
-        if (!inode) {
-                errno = ESTALE;
-                goto out;
-        }
+    /* populate loc */
+    GLFS_LOC_FILL_INODE(inode, loc, out);
 
-        /* populate loc */
-        GLFS_LOC_FILL_INODE (inode, loc, out);
+    /* fop/op */
+    ret = syncop_statfs(subvol, &loc, statvfs, NULL, NULL);
+    DECODE_SYNCOP_ERR(ret);
 
-        /* fop/op */
-        ret = syncop_statfs (subvol, &loc, statvfs, NULL, NULL);
-        DECODE_SYNCOP_ERR (ret);
-
-        loc_wipe (&loc);
+    loc_wipe(&loc);
 
 out:
-        if (inode)
-                inode_unref (inode);
+    if (inode)
+        inode_unref(inode);
 
-        glfs_subvol_done (fs, subvol);
+    glfs_subvol_done(fs, subvol);
 
-        __GLFS_EXIT_FS;
+    __GLFS_EXIT_FS;
 
 invalid_fs:
-        return ret;
+    return ret;
 }
 
-GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_h_statfs, 3.7.0);
-
+GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_h_stat, 3.4.2)
 int
-pub_glfs_h_stat (struct glfs *fs, struct glfs_object *object, struct stat *stat)
+pub_glfs_h_stat(struct glfs *fs, struct glfs_object *object, struct stat *stat)
 {
-        int              ret = -1;
-        xlator_t        *subvol = NULL;
-        inode_t         *inode = NULL;
-        loc_t            loc = {0, };
-        struct iatt      iatt = {0, };
-
-        DECLARE_OLD_THIS;
-
-        /* validate in args */
-        if ((fs == NULL) || (object == NULL)) {
-                errno = EINVAL;
-                return -1;
-        }
-
-        __GLFS_ENTRY_VALIDATE_FS (fs, invalid_fs);
-
-        /* get the active volume */
-        subvol = glfs_active_subvol (fs);
-        if (!subvol) {
-                ret = -1;
-                errno = EIO;
-                goto out;
-        }
-
-        /* get/refresh the in arg objects inode in correlation to the xlator */
-        inode = glfs_resolve_inode (fs, subvol, object);
-        if (!inode) {
-                errno = ESTALE;
-                goto out;
-        }
-
-        /* populate loc */
-        GLFS_LOC_FILL_INODE (inode, loc, out);
-
-        /* fop/op */
-        ret = syncop_stat (subvol, &loc, &iatt, NULL, NULL);
-        DECODE_SYNCOP_ERR (ret);
-
-        /* populate out args */
-        if (!ret && stat) {
-                glfs_iatt_to_stat (fs, &iatt, stat);
-        }
+    int ret = -1;
+    xlator_t *subvol = NULL;
+    inode_t *inode = NULL;
+    loc_t loc = {
+        0,
+    };
+    struct iatt iatt = {
+        0,
+    };
+
+    DECLARE_OLD_THIS;
+
+    /* validate in args */
+    if ((fs == NULL) || (object == NULL)) {
+        errno = EINVAL;
+        return -1;
+    }
+
+    __GLFS_ENTRY_VALIDATE_FS(fs, invalid_fs);
+
+    /* get the active volume */
+    subvol = glfs_active_subvol(fs);
+    if (!subvol) {
+        ret = -1;
+        errno = EIO;
+        goto out;
+    }
+
+    /* get/refresh the in arg objects inode in correlation to the xlator */
+    inode = glfs_resolve_inode(fs, subvol, object);
+    if (!inode) {
+        errno = ESTALE;
+        goto out;
+    }
+
+    /* populate loc */
+    GLFS_LOC_FILL_INODE(inode, loc, out);
+
+    /* fop/op */
+    ret = syncop_stat(subvol, &loc, &iatt, NULL, NULL);
+    DECODE_SYNCOP_ERR(ret);
+
+    /* populate out args */
+    if (!ret && stat) {
+        glfs_iatt_to_stat(fs, &iatt, stat);
+    }
 out:
-        loc_wipe (&loc);
+    loc_wipe(&loc);
 
-        if (inode)
-                inode_unref (inode);
+    if (inode)
+        inode_unref(inode);
 
-        glfs_subvol_done (fs, subvol);
+    glfs_subvol_done(fs, subvol);
 
-        __GLFS_EXIT_FS;
+    __GLFS_EXIT_FS;
 
 invalid_fs:
-        return ret;
+    return ret;
 }
 
-GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_h_stat, 3.4.2);
-
-
+GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_h_getattrs, 3.4.2)
 int
-pub_glfs_h_getattrs (struct glfs *fs, struct glfs_object *object,
-                     struct stat *stat)
+pub_glfs_h_getattrs(struct glfs *fs, struct glfs_object *object,
+                    struct stat *stat)
 {
-        int                      ret = -1;
-        xlator_t                *subvol = NULL;
-        inode_t                 *inode = NULL;
-        struct iatt              iatt = {0, };
-
-        /* validate in args */
-        if ((fs == NULL) || (object == NULL)) {
-                errno = EINVAL;
-                return -1;
-        }
-
-        DECLARE_OLD_THIS;
-        __GLFS_ENTRY_VALIDATE_FS (fs, invalid_fs);
-
-        /* get the active volume */
-        subvol = glfs_active_subvol (fs);
-        if (!subvol) {
-                ret = -1;
-                errno = EIO;
-                goto out;
-        }
-
-        /* get/refresh the in arg objects inode in correlation to the xlator */
-        inode = glfs_resolve_inode (fs, subvol, object);
-        if (!inode) {
-                ret = 0;
-                errno = ESTALE;
-                goto out;
-        }
+    int ret = -1;
+    xlator_t *subvol = NULL;
+    inode_t *inode = NULL;
+    struct iatt iatt = {
+        0,
+    };
+
+    /* validate in args */
+    if ((fs == NULL) || (object == NULL)) {
+        errno = EINVAL;
+        return -1;
+    }
+
+    DECLARE_OLD_THIS;
+    __GLFS_ENTRY_VALIDATE_FS(fs, invalid_fs);
+
+    /* get the active volume */
+    subvol = glfs_active_subvol(fs);
+    if (!subvol) {
+        ret = -1;
+        errno = EIO;
+        goto out;
+    }
+
+    /* get/refresh the in arg objects inode in correlation to the xlator */
+    inode = glfs_resolve_inode(fs, subvol, object);
+    if (!inode) {
+        ret = 0;
+        errno = ESTALE;
+        goto out;
+    }
 
-        /* fop/op */
-        ret = glfs_resolve_base (fs, subvol, inode, &iatt);
+    /* fop/op */
+    ret = glfs_resolve_base(fs, subvol, inode, &iatt);
 
-        /* populate out args */
-        if (!ret && stat) {
-                glfs_iatt_to_stat (fs, &iatt, stat);
-        }
+    /* populate out args */
+    if (!ret && stat) {
+        glfs_iatt_to_stat(fs, &iatt, stat);
+    }
 
 out:
-        if (inode)
-                inode_unref (inode);
+    if (inode)
+        inode_unref(inode);
 
-        glfs_subvol_done (fs, subvol);
+    glfs_subvol_done(fs, subvol);
 
-        __GLFS_EXIT_FS;
+    __GLFS_EXIT_FS;
 
 invalid_fs:
-        return ret;
+    return ret;
 }
 
-GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_h_getattrs, 3.4.2);
-
-
 int
-glfs_h_getxattrs_common (struct glfs *fs, struct glfs_object *object,
-                         dict_t **xattr, const char *name)
+glfs_h_getxattrs_common(struct glfs *fs, struct glfs_object *object,
+                        dict_t **xattr, const char *name,
+                        gf_boolean_t is_listxattr)
 {
-        int                 ret = 0;
-        xlator_t        *subvol = NULL;
-        inode_t         *inode = NULL;
-        loc_t            loc = {0, };
-
-        /* validate in args */
-        if ((fs == NULL) || (object == NULL)) {
-                errno = EINVAL;
-                return -1;
-        }
+    int ret = 0;
+    xlator_t *subvol = NULL;
+    inode_t *inode = NULL;
+    loc_t loc = {
+        0,
+    };
+
+    /* validate in args */
+    if ((fs == NULL) || (object == NULL)) {
+        errno = EINVAL;
+        return -1;
+    }
 
+    if (!is_listxattr) {
         if (!name || *name == '\0') {
-                errno = EINVAL;
-                return -1;
+            errno = EINVAL;
+            return -1;
         }
 
         if (strlen(name) > GF_XATTR_NAME_MAX) {
-                errno = ENAMETOOLONG;
-                return -1;
-        }
-
-        /* get the active volume */
-        subvol = glfs_active_subvol (fs);
-        if (!subvol) {
-                ret = -1;
-                errno = EIO;
-                goto out;
-        }
-
-        /* get/refresh the in arg objects inode in correlation to the xlator */
-        inode = glfs_resolve_inode (fs, subvol, object);
-        if (!inode) {
-                errno = ESTALE;
-                goto out;
-        }
-
-        /* populate loc */
-        GLFS_LOC_FILL_INODE (inode, loc, out);
-
-        ret = syncop_getxattr (subvol, &loc, xattr, name, NULL, NULL);
-        DECODE_SYNCOP_ERR (ret);
+            errno = ENAMETOOLONG;
+            return -1;
+        }
+    }
+    /* get the active volume */
+    subvol = glfs_active_subvol(fs);
+    if (!subvol) {
+        ret = -1;
+        errno = EIO;
+        goto out;
+    }
+
+    /* get/refresh the in arg objects inode in correlation to the xlator */
+    inode = glfs_resolve_inode(fs, subvol, object);
+    if (!inode) {
+        errno = ESTALE;
+        goto out;
+    }
+
+    /* populate loc */
+    GLFS_LOC_FILL_INODE(inode, loc, out);
+
+    ret = syncop_getxattr(subvol, &loc, xattr, name, NULL, NULL);
+    DECODE_SYNCOP_ERR(ret);
 
 out:
-        loc_wipe (&loc);
+    loc_wipe(&loc);
 
-        if (inode)
-                inode_unref (inode);
+    if (inode)
+        inode_unref(inode);
 
-        glfs_subvol_done (fs, subvol);
+    glfs_subvol_done(fs, subvol);
 
-        return ret;
+    return ret;
 }
 
-
+GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_h_getxattrs, 3.5.1)
 int
-pub_glfs_h_getxattrs (struct glfs *fs, struct glfs_object *object,
-                      const char *name, void *value, size_t size)
+pub_glfs_h_getxattrs(struct glfs *fs, struct glfs_object *object,
+                     const char *name, void *value, size_t size)
 {
-        int                    ret   = -1;
-        dict_t                *xattr = NULL;
+    int ret = -1;
+    dict_t *xattr = NULL;
 
-        /* validate in args */
-        if ((fs == NULL) || (object == NULL)) {
-                errno = EINVAL;
-                return -1;
-        }
+    /* validate in args */
+    if ((fs == NULL) || (object == NULL)) {
+        errno = EINVAL;
+        return -1;
+    }
 
-        DECLARE_OLD_THIS;
-        __GLFS_ENTRY_VALIDATE_FS (fs, invalid_fs);
+    DECLARE_OLD_THIS;
+    __GLFS_ENTRY_VALIDATE_FS(fs, invalid_fs);
 
-        ret = glfs_h_getxattrs_common (fs, object, &xattr, name);
-        if (ret)
-                goto out;
+    ret = glfs_h_getxattrs_common(fs, object, &xattr, name, (name == NULL));
+    if (ret)
+        goto out;
 
-        /* If @name is NULL, means get all the xattrs (i.e listxattr). */
-        if (name)
-                ret = glfs_getxattr_process (value, size, xattr, name);
-        else
-                ret = glfs_listxattr_process (value, size, xattr);
+    /* If @name is NULL, means get all the xattrs (i.e listxattr). */
+    if (name)
+        ret = glfs_getxattr_process(value, size, xattr, name);
+    else
+        ret = glfs_listxattr_process(value, size, xattr);
 
 out:
-        if (xattr)
-                dict_unref (xattr);
+    if (xattr)
+        dict_unref(xattr);
 
-        __GLFS_EXIT_FS;
+    __GLFS_EXIT_FS;
 
 invalid_fs:
-        return ret;
+    return ret;
 }
 
-GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_h_getxattrs, 3.5.1);
-
+GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_h_setattrs, 3.4.2)
 int
-pub_glfs_h_setattrs (struct glfs *fs, struct glfs_object *object,
-                     struct stat *stat, int valid)
+pub_glfs_h_setattrs(struct glfs *fs, struct glfs_object *object,
+                    struct stat *stat, int valid)
 {
-        int              ret = -1;
-        xlator_t        *subvol = NULL;
-        inode_t         *inode = NULL;
-        loc_t            loc = {0, };
-        struct iatt      iatt = {0, };
-        int              glvalid = 0;
-
-        /* validate in args */
-        if ((fs == NULL) || (object == NULL) || (stat == NULL)) {
-                errno = EINVAL;
-                return -1;
-        }
-
-        DECLARE_OLD_THIS;
-        __GLFS_ENTRY_VALIDATE_FS (fs, invalid_fs);
-
-        /* get the active volume */
-        subvol = glfs_active_subvol (fs);
-        if (!subvol) {
-                ret = -1;
-                errno = EIO;
-                goto out;
-        }
-
-        /* get/refresh the in arg objects inode in correlation to the xlator */
-        inode = glfs_resolve_inode (fs, subvol, object);
-        if (!inode) {
-                errno = ESTALE;
-                goto out;
-        }
-
-        /* map valid masks from in args */
-        glfs_iatt_from_stat (stat, valid, &iatt, &glvalid);
-
-        /* populate loc */
-        GLFS_LOC_FILL_INODE (inode, loc, out);
-
-        /* fop/op */
-        ret = syncop_setattr (subvol, &loc, &iatt, glvalid, 0, 0, NULL, NULL);
-        DECODE_SYNCOP_ERR (ret);
+    int ret = -1;
+    xlator_t *subvol = NULL;
+    inode_t *inode = NULL;
+    loc_t loc = {
+        0,
+    };
+    struct iatt iatt = {
+        0,
+    };
+    int glvalid = 0;
+
+    /* validate in args */
+    if ((fs == NULL) || (object == NULL) || (stat == NULL)) {
+        errno = EINVAL;
+        return -1;
+    }
+
+    DECLARE_OLD_THIS;
+    __GLFS_ENTRY_VALIDATE_FS(fs, invalid_fs);
+
+    /* get the active volume */
+    subvol = glfs_active_subvol(fs);
+    if (!subvol) {
+        ret = -1;
+        errno = EIO;
+        goto out;
+    }
+
+    /* get/refresh the in arg objects inode in correlation to the xlator */
+    inode = glfs_resolve_inode(fs, subvol, object);
+    if (!inode) {
+        errno = ESTALE;
+        goto out;
+    }
+
+    /* map valid masks from in args */
+    glfs_iatt_from_stat(stat, valid, &iatt, &glvalid);
+
+    /* populate loc */
+    GLFS_LOC_FILL_INODE(inode, loc, out);
+
+    /* fop/op */
+    ret = syncop_setattr(subvol, &loc, &iatt, glvalid, 0, 0, NULL, NULL);
+    DECODE_SYNCOP_ERR(ret);
 out:
-        loc_wipe (&loc);
+    loc_wipe(&loc);
 
-        if (inode)
-                inode_unref (inode);
+    if (inode)
+        inode_unref(inode);
 
-        glfs_subvol_done (fs, subvol);
+    glfs_subvol_done(fs, subvol);
 
-        __GLFS_EXIT_FS;
+    __GLFS_EXIT_FS;
 
 invalid_fs:
-        return ret;
+    return ret;
 }
 
-GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_h_setattrs, 3.4.2);
-
-
+GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_h_setxattrs, 3.5.0)
 int
-pub_glfs_h_setxattrs (struct glfs *fs, struct glfs_object *object,
-                      const char *name, const void *value, size_t size,
-                      int flags)
+pub_glfs_h_setxattrs(struct glfs *fs, struct glfs_object *object,
+                     const char *name, const void *value, size_t size,
+                     int flags)
 {
-        int              ret = -1;
-        xlator_t        *subvol = NULL;
-        inode_t         *inode = NULL;
-        loc_t            loc = {0, };
-        dict_t          *xattr = NULL;
-
-        /* validate in args */
-        if ((fs == NULL) || (object == NULL) ||
-                 (name == NULL) || (value == NULL)) {
-                errno = EINVAL;
-                return -1;
-        }
-
-        if (!name || *name == '\0') {
-                errno = EINVAL;
-                return -1;
-        }
-
-        if (strlen(name) > GF_XATTR_NAME_MAX) {
-                errno = ENAMETOOLONG;
-                return -1;
-        }
-
-        DECLARE_OLD_THIS;
-        __GLFS_ENTRY_VALIDATE_FS (fs, invalid_fs);
-
-        /* get the active volume */
-        subvol = glfs_active_subvol (fs);
-        if (!subvol) {
-                ret = -1;
-                errno = EIO;
-                goto out;
-        }
-
-        /* get/refresh the in arg objects inode in correlation to the xlator */
-        inode = glfs_resolve_inode (fs, subvol, object);
-        if (!inode) {
-                errno = ESTALE;
-                goto out;
-        }
-
-        xattr = dict_for_key_value (name, value, size);
-        if (!xattr) {
-                ret = -1;
-                errno = ENOMEM;
-                goto out;
-        }
+    int ret = -1;
+    xlator_t *subvol = NULL;
+    inode_t *inode = NULL;
+    loc_t loc = {
+        0,
+    };
+    dict_t *xattr = NULL;
+    void *value_cp = NULL;
+
+    /* validate in args */
+    if ((fs == NULL) || (object == NULL) || (name == NULL) || (value == NULL)) {
+        errno = EINVAL;
+        return -1;
+    }
 
-        /* populate loc */
-        GLFS_LOC_FILL_INODE (inode, loc, out);
+    if (!name || *name == '\0') {
+        errno = EINVAL;
+        return -1;
+    }
 
-        /* fop/op */
-        ret = syncop_setxattr (subvol, &loc, xattr, flags, NULL, NULL);
-        DECODE_SYNCOP_ERR (ret);
+    if (strlen(name) > GF_XATTR_NAME_MAX) {
+        errno = ENAMETOOLONG;
+        return -1;
+    }
+
+    DECLARE_OLD_THIS;
+    __GLFS_ENTRY_VALIDATE_FS(fs, invalid_fs);
+
+    /* get the active volume */
+    subvol = glfs_active_subvol(fs);
+    if (!subvol) {
+        ret = -1;
+        errno = EIO;
+        goto out;
+    }
+
+    /* get/refresh the in arg objects inode in correlation to the xlator */
+    inode = glfs_resolve_inode(fs, subvol, object);
+    if (!inode) {
+        errno = ESTALE;
+        goto out;
+    }
+
+    value_cp = gf_memdup(value, size);
+    GF_CHECK_ALLOC_AND_LOG(subvol->name, value_cp, ret,
+                           "Failed to"
+                           " duplicate setxattr value",
+                           out);
+
+    xattr = dict_for_key_value(name, value_cp, size, _gf_false);
+    if (!xattr) {
+        GF_FREE(value_cp);
+        ret = -1;
+        errno = ENOMEM;
+        goto out;
+    }
+
+    /* populate loc */
+    GLFS_LOC_FILL_INODE(inode, loc, out);
+
+    /* fop/op */
+    ret = syncop_setxattr(subvol, &loc, xattr, flags, NULL, NULL);
+    DECODE_SYNCOP_ERR(ret);
 
 out:
-        loc_wipe (&loc);
+    loc_wipe(&loc);
 
-        if (inode)
-                inode_unref (inode);
+    if (inode)
+        inode_unref(inode);
 
-        if (xattr)
-                dict_unref (xattr);
+    if (xattr)
+        dict_unref(xattr);
 
-        glfs_subvol_done (fs, subvol);
+    glfs_subvol_done(fs, subvol);
 
-        __GLFS_EXIT_FS;
+    __GLFS_EXIT_FS;
 
 invalid_fs:
-        return ret;
+    return ret;
 }
 
-GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_h_setxattrs, 3.5.0);
-
-
+GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_h_removexattrs, 3.5.1)
 int
-pub_glfs_h_removexattrs (struct glfs *fs, struct glfs_object *object,
-                         const char *name)
+pub_glfs_h_removexattrs(struct glfs *fs, struct glfs_object *object,
+                        const char *name)
 {
-        int              ret = -1;
-        xlator_t        *subvol = NULL;
-        inode_t         *inode = NULL;
-        loc_t            loc = {0, };
-
-        /* validate in args */
-        if ((fs == NULL) || (object == NULL) || (name == NULL)) {
-                errno = EINVAL;
-                return -1;
-        }
+    int ret = -1;
+    xlator_t *subvol = NULL;
+    inode_t *inode = NULL;
+    loc_t loc = {
+        0,
+    };
+
+    /* validate in args */
+    if ((fs == NULL) || (object == NULL) || (name == NULL)) {
+        errno = EINVAL;
+        return -1;
+    }
 
-        DECLARE_OLD_THIS;
-        __GLFS_ENTRY_VALIDATE_FS (fs, invalid_fs);
+    DECLARE_OLD_THIS;
+    __GLFS_ENTRY_VALIDATE_FS(fs, invalid_fs);
 
-        /* get the active volume */
-        subvol = glfs_active_subvol (fs);
-        if (!subvol) {
-                ret = -1;
-                errno = EIO;
-                goto out;
-        }
+    /* get the active volume */
+    subvol = glfs_active_subvol(fs);
+    if (!subvol) {
+        ret = -1;
+        errno = EIO;
+        goto out;
+    }
 
-        /* get/refresh the in arg objects inode in correlation to the xlator */
-        inode = glfs_resolve_inode (fs, subvol, object);
-        if (!inode) {
-                errno = ESTALE;
-                goto out;
-        }
+    /* get/refresh the in arg objects inode in correlation to the xlator */
+    inode = glfs_resolve_inode(fs, subvol, object);
+    if (!inode) {
+        errno = ESTALE;
+        goto out;
+    }
 
-        /* populate loc */
-        GLFS_LOC_FILL_INODE (inode, loc, out);
+    /* populate loc */
+    GLFS_LOC_FILL_INODE(inode, loc, out);
 
-        /* fop/op */
-        ret = syncop_removexattr (subvol, &loc, name, NULL, NULL);
-        DECODE_SYNCOP_ERR (ret);
+    /* fop/op */
+    ret = syncop_removexattr(subvol, &loc, name, NULL, NULL);
+    DECODE_SYNCOP_ERR(ret);
 
 out:
-        loc_wipe (&loc);
+    loc_wipe(&loc);
 
-        if (inode)
-                inode_unref (inode);
+    if (inode)
+        inode_unref(inode);
 
-        glfs_subvol_done (fs, subvol);
+    glfs_subvol_done(fs, subvol);
 
-        __GLFS_EXIT_FS;
+    __GLFS_EXIT_FS;
 
 invalid_fs:
-        return ret;
+    return ret;
 }
 
-GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_h_removexattrs, 3.5.1);
-
-
+GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_h_open, 3.4.2)
 struct glfs_fd *
-pub_glfs_h_open (struct glfs *fs, struct glfs_object *object, int flags)
+pub_glfs_h_open(struct glfs *fs, struct glfs_object *object, int flags)
 {
-        int              ret = -1;
-        struct glfs_fd  *glfd = NULL;
-        xlator_t        *subvol = NULL;
-        inode_t         *inode = NULL;
-        loc_t            loc = {0, };
-
-        /* validate in args */
-        if ((fs == NULL) || (object == NULL)) {
-                errno = EINVAL;
-                return NULL;
-        }
-
-        DECLARE_OLD_THIS;
-        __GLFS_ENTRY_VALIDATE_FS (fs, invalid_fs);
-
-        /* get the active volume */
-        subvol = glfs_active_subvol (fs);
-        if (!subvol) {
-                errno = EIO;
-                goto out;
-        }
-
-        /* get/refresh the in arg objects inode in correlation to the xlator */
-        inode = glfs_resolve_inode (fs, subvol, object);
-        if (!inode) {
-                errno = ESTALE;
-                goto out;
-        }
-
-        /* check types to open */
-        if (IA_ISDIR (inode->ia_type)) {
-                ret = -1;
-                errno = EISDIR;
-                goto out;
-        }
-
-        if (!IA_ISREG (inode->ia_type)) {
-                ret = -1;
-                errno = EINVAL;
-                goto out;
-        }
-
-        glfd = glfs_fd_new (fs);
-        if (!glfd) {
-                ret = -1;
-                errno = ENOMEM;
-                goto out;
-        }
-
-        glfd->fd = fd_create (inode, getpid());
-        if (!glfd->fd) {
-                ret = -1;
-                errno = ENOMEM;
-                goto out;
-        }
-        glfd->fd->flags = flags;
-
-        /* populate loc */
-        GLFS_LOC_FILL_INODE (inode, loc, out);
-
-        /* fop/op */
-        ret = syncop_open (subvol, &loc, flags, glfd->fd, NULL, NULL);
-        DECODE_SYNCOP_ERR (ret);
-
-        glfd->fd->flags = flags;
-        fd_bind (glfd->fd);
-        glfs_fd_bind (glfd);
+    int ret = -1;
+    struct glfs_fd *glfd = NULL;
+    xlator_t *subvol = NULL;
+    inode_t *inode = NULL;
+    loc_t loc = {
+        0,
+    };
+    dict_t *fop_attr = NULL;
+
+    /* validate in args */
+    if ((fs == NULL) || (object == NULL)) {
+        errno = EINVAL;
+        return NULL;
+    }
+
+    DECLARE_OLD_THIS;
+    __GLFS_ENTRY_VALIDATE_FS(fs, invalid_fs);
+
+    /* get the active volume */
+    subvol = glfs_active_subvol(fs);
+    if (!subvol) {
+        errno = EIO;
+        goto out;
+    }
+
+    /* get/refresh the in arg objects inode in correlation to the xlator */
+    inode = glfs_resolve_inode(fs, subvol, object);
+    if (!inode) {
+        errno = ESTALE;
+        goto out;
+    }
+
+    /* check types to open */
+    if (IA_ISDIR(inode->ia_type)) {
+        ret = -1;
+        errno = EISDIR;
+        goto out;
+    }
+
+    if (!IA_ISREG(inode->ia_type)) {
+        ret = -1;
+        errno = EINVAL;
+        goto out;
+    }
+
+    glfd = glfs_fd_new(fs);
+    if (!glfd) {
+        ret = -1;
+        errno = ENOMEM;
+        goto out;
+    }
+
+    glfd->fd = fd_create(inode, getpid());
+    if (!glfd->fd) {
+        ret = -1;
+        errno = ENOMEM;
+        goto out;
+    }
+    glfd->fd->flags = flags;
+
+    /* populate loc */
+    GLFS_LOC_FILL_INODE(inode, loc, out);
+
+    /* fop/op */
+    ret = get_fop_attr_thrd_key(&fop_attr);
+    if (ret)
+        gf_msg_debug("gfapi", 0, "Getting leaseid from thread failed");
+
+    ret = syncop_open(subvol, &loc, flags, glfd->fd, fop_attr, NULL);
+    DECODE_SYNCOP_ERR(ret);
+
+    glfd->fd->flags = flags;
 
 out:
-        loc_wipe (&loc);
+    loc_wipe(&loc);
 
-        if (inode)
-                inode_unref (inode);
+    if (inode)
+        inode_unref(inode);
+    if (fop_attr)
+        dict_unref(fop_attr);
 
-        if (ret && glfd) {
-                glfs_fd_destroy (glfd);
-                glfd = NULL;
-        }
+    if (ret && glfd) {
+        GF_REF_PUT(glfd);
+        glfd = NULL;
+    } else if (glfd) {
+        glfd_set_state_bind(glfd);
+    }
 
-        glfs_subvol_done (fs, subvol);
+    glfs_subvol_done(fs, subvol);
 
-        __GLFS_EXIT_FS;
+    __GLFS_EXIT_FS;
 
 invalid_fs:
-        return glfd;
+    return glfd;
 }
 
-GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_h_open, 3.4.2);
-
-
+GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_h_creat, 3.4.2)
 struct glfs_object *
-pub_glfs_h_creat (struct glfs *fs, struct glfs_object *parent, const char *path,
-                  int flags, mode_t mode, struct stat *stat)
+pub_glfs_h_creat(struct glfs *fs, struct glfs_object *parent, const char *path,
+                 int flags, mode_t mode, struct stat *stat)
 {
-        int                 ret = -1;
-        struct glfs_fd     *glfd = NULL;
-        xlator_t           *subvol = NULL;
-        inode_t            *inode = NULL;
-        loc_t               loc = {0, };
-        struct iatt         iatt = {0, };
-        uuid_t              gfid;
-        dict_t             *xattr_req = NULL;
-        struct glfs_object *object = NULL;
-
-        /* validate in args */
-        if ((fs == NULL) || (parent == NULL) || (path == NULL)) {
-                errno = EINVAL;
-                return NULL;
+    int ret = -1;
+    fd_t *fd = NULL;
+    xlator_t *subvol = NULL;
+    inode_t *inode = NULL;
+    loc_t loc = {
+        0,
+    };
+    struct iatt iatt = {
+        0,
+    };
+    uuid_t gfid;
+    dict_t *xattr_req = NULL;
+    struct glfs_object *object = NULL;
+
+    /* validate in args */
+    if ((fs == NULL) || (parent == NULL) || (path == NULL)) {
+        errno = EINVAL;
+        return NULL;
+    }
+
+    DECLARE_OLD_THIS;
+    __GLFS_ENTRY_VALIDATE_FS(fs, invalid_fs);
+
+    /* get the active volume */
+    subvol = glfs_active_subvol(fs);
+    if (!subvol) {
+        ret = -1;
+        errno = EIO;
+        goto out;
+    }
+
+    /* get/refresh the in arg objects inode in correlation to the xlator */
+    inode = glfs_resolve_inode(fs, subvol, parent);
+    if (!inode) {
+        ret = -1;
+        errno = ESTALE;
+        goto out;
+    }
+
+    xattr_req = dict_new();
+    if (!xattr_req) {
+        ret = -1;
+        errno = ENOMEM;
+        goto out;
+    }
+
+    gf_uuid_generate(gfid);
+    ret = dict_set_gfuuid(xattr_req, "gfid-req", gfid, true);
+    if (ret) {
+        ret = -1;
+        errno = ENOMEM;
+        goto out;
+    }
+
+    GLFS_LOC_FILL_PINODE(inode, loc, ret, errno, out, path);
+
+    fd = fd_create(loc.inode, getpid());
+    if (!fd) {
+        ret = -1;
+        errno = ENOMEM;
+        goto out;
+    }
+    fd->flags = flags;
+
+    /* fop/op */
+    ret = syncop_create(subvol, &loc, flags, mode, fd, &iatt, xattr_req, NULL);
+    DECODE_SYNCOP_ERR(ret);
+
+    /* populate out args */
+    if (ret == 0) {
+        ret = glfs_loc_link(&loc, &iatt);
+        if (ret != 0) {
+            goto out;
         }
 
-        DECLARE_OLD_THIS;
-        __GLFS_ENTRY_VALIDATE_FS (fs, invalid_fs);
+        if (stat)
+            glfs_iatt_to_stat(fs, &iatt, stat);
 
-        /* get the active volume */
-        subvol = glfs_active_subvol (fs);
-        if (!subvol) {
-                ret = -1;
-                errno = EIO;
-                goto out;
-        }
+        ret = glfs_create_object(&loc, &object);
+    }
 
-        /* get/refresh the in arg objects inode in correlation to the xlator */
-        inode = glfs_resolve_inode (fs, subvol, parent);
-        if (!inode) {
-                errno = ESTALE;
-                goto out;
-        }
+out:
+    if (ret && object != NULL) {
+        /* Release the held reference */
+        glfs_h_close(object);
+        object = NULL;
+    }
 
-        xattr_req = dict_new ();
-        if (!xattr_req) {
-                ret = -1;
-                errno = ENOMEM;
-                goto out;
-        }
+    loc_wipe(&loc);
 
-        gf_uuid_generate (gfid);
-        ret = dict_set_static_bin (xattr_req, "gfid-req", gfid, 16);
-        if (ret) {
-                ret = -1;
-                errno = ENOMEM;
-                goto out;
-        }
+    if (inode)
+        inode_unref(inode);
 
-        GLFS_LOC_FILL_PINODE (inode, loc, ret, errno, out, path);
+    if (xattr_req)
+        dict_unref(xattr_req);
 
-        glfd = glfs_fd_new (fs);
-        if (!glfd) {
-                 ret = -1;
-                 errno = ENOMEM;
-                 goto out;
-        }
+    if (fd)
+        fd_unref(fd);
 
-        glfd->fd = fd_create (loc.inode, getpid());
-        if (!glfd->fd) {
-                ret = -1;
-                errno = ENOMEM;
-                goto out;
-        }
-        glfd->fd->flags = flags;
+    glfs_subvol_done(fs, subvol);
 
-        /* fop/op */
-        ret = syncop_create (subvol, &loc, flags, mode, glfd->fd,
-                             &iatt, xattr_req, NULL);
-        DECODE_SYNCOP_ERR (ret);
+    __GLFS_EXIT_FS;
 
-        /* populate out args */
-        if (ret == 0) {
-                ret = glfs_loc_link (&loc, &iatt);
-                if (ret != 0) {
-                        goto out;
-                }
+invalid_fs:
+    return object;
+}
 
-                if (stat)
-                        glfs_iatt_to_stat (fs, &iatt, stat);
+GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_h_creat_open, 6.6)
+struct glfs_object *
+pub_glfs_h_creat_open(struct glfs *fs, struct glfs_object *parent,
+                      const char *path, int flags, mode_t mode,
+                      struct stat *stat, struct glfs_fd **out_fd)
+{
+    int ret = -1;
+    struct glfs_fd *glfd = NULL;
+    xlator_t *subvol = NULL;
+    inode_t *inode = NULL;
+    loc_t loc = {
+        0,
+    };
+    struct iatt iatt = {
+        0,
+    };
+    uuid_t gfid;
+    dict_t *xattr_req = NULL;
+    struct glfs_object *object = NULL;
+    dict_t *fop_attr = NULL;
+
+    /* validate in args */
+    if ((fs == NULL) || (parent == NULL) || (path == NULL) ||
+        (out_fd == NULL)) {
+        errno = EINVAL;
+        return NULL;
+    }
+
+    DECLARE_OLD_THIS;
+    __GLFS_ENTRY_VALIDATE_FS(fs, invalid_fs);
+
+    /* get the active volume */
+    subvol = glfs_active_subvol(fs);
+    if (!subvol) {
+        ret = -1;
+        goto out;
+    }
+
+    /* get/refresh the in arg objects inode in correlation to the xlator */
+    inode = glfs_resolve_inode(fs, subvol, parent);
+    if (!inode) {
+        ret = -1;
+        goto out;
+    }
+
+    xattr_req = dict_new();
+    if (!xattr_req) {
+        ret = -1;
+        errno = ENOMEM;
+        goto out;
+    }
+
+    gf_uuid_generate(gfid);
+    ret = dict_set_gfuuid(xattr_req, "gfid-req", gfid, true);
+    if (ret) {
+        ret = -1;
+        errno = ENOMEM;
+        goto out;
+    }
+
+    GLFS_LOC_FILL_PINODE(inode, loc, ret, errno, out, path);
+
+    glfd = glfs_fd_new(fs);
+    if (!glfd) {
+        ret = -1;
+        errno = ENOMEM;
+        goto out;
+    }
+
+    glfd->fd = fd_create(loc.inode, getpid());
+    if (!glfd->fd) {
+        ret = -1;
+        errno = ENOMEM;
+        goto out;
+    }
+    glfd->fd->flags = flags;
+
+    ret = get_fop_attr_thrd_key(&fop_attr);
+    if (ret)
+        gf_msg_debug("gfapi", 0, "Getting leaseid from thread failed");
+
+    /* fop/op */
+    ret = syncop_create(subvol, &loc, flags, mode, glfd->fd, &iatt, xattr_req,
+                        NULL);
+    DECODE_SYNCOP_ERR(ret);
+
+    /* populate out args */
+    if (ret == 0) {
+        glfd->fd->flags = flags;
 
-                ret = glfs_create_object (&loc, &object);
+        ret = glfs_loc_link(&loc, &iatt);
+        if (ret != 0) {
+            goto out;
         }
 
-        glfd->fd->flags = flags;
-        fd_bind (glfd->fd);
-        glfs_fd_bind (glfd);
+        if (stat)
+            glfs_iatt_to_stat(fs, &iatt, stat);
+
+        ret = glfs_create_object(&loc, &object);
+    }
 
 out:
-        if (ret && object != NULL) {
-                /* Release the held reference */
-                glfs_h_close (object);
-                object = NULL;
-        }
+    if (ret && object != NULL) {
+        /* Release the held reference */
+        glfs_h_close(object);
+        object = NULL;
+    }
 
-        loc_wipe(&loc);
+    loc_wipe(&loc);
 
-        if (inode)
-                inode_unref (inode);
+    if (inode)
+        inode_unref(inode);
 
-        if (xattr_req)
-                dict_unref (xattr_req);
+    if (fop_attr)
+        dict_unref(fop_attr);
 
-        if (glfd) {
-                glfs_fd_destroy (glfd);
-                glfd = NULL;
-        }
+    if (xattr_req)
+        dict_unref(xattr_req);
 
-        glfs_subvol_done (fs, subvol);
+    if (ret && glfd) {
+        GF_REF_PUT(glfd);
+    } else if (glfd) {
+        glfd_set_state_bind(glfd);
+        *out_fd = glfd;
+    }
 
-        __GLFS_EXIT_FS;
+    glfs_subvol_done(fs, subvol);
+
+    __GLFS_EXIT_FS;
 
 invalid_fs:
-        return object;
+    return object;
 }
 
-GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_h_creat, 3.4.2);
-
-
+GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_h_mkdir, 3.4.2)
 struct glfs_object *
-pub_glfs_h_mkdir (struct glfs *fs, struct glfs_object *parent, const char *path,
-                  mode_t mode, struct stat *stat)
+pub_glfs_h_mkdir(struct glfs *fs, struct glfs_object *parent, const char *path,
+                 mode_t mode, struct stat *stat)
 {
-        int                 ret = -1;
-        xlator_t           *subvol = NULL;
-        inode_t            *inode = NULL;
-        loc_t               loc = {0, };
-        struct iatt         iatt = {0, };
-        uuid_t              gfid;
-        dict_t             *xattr_req = NULL;
-        struct glfs_object *object = NULL;
-
-        /* validate in args */
-        if ((fs == NULL) || (parent == NULL) || (path == NULL)) {
-                errno = EINVAL;
-                return NULL;
-        }
-
-        DECLARE_OLD_THIS;
-        __GLFS_ENTRY_VALIDATE_FS (fs, invalid_fs);
-
-        /* get the active volume */
-        subvol = glfs_active_subvol (fs);
-        if (!subvol) {
-                ret = -1;
-                errno = EIO;
-                goto out;
-        }
-
-        /* get/refresh the in arg objects inode in correlation to the xlator */
-        inode = glfs_resolve_inode (fs, subvol, parent);
-        if (!inode) {
-                errno = ESTALE;
-                goto out;
-        }
-
-        xattr_req = dict_new ();
-        if (!xattr_req) {
-                ret = -1;
-                errno = ENOMEM;
-                goto out;
-        }
-
-        gf_uuid_generate (gfid);
-        ret = dict_set_static_bin (xattr_req, "gfid-req", gfid, 16);
-        if (ret) {
-                ret = -1;
-                errno = ENOMEM;
-                goto out;
+    int ret = -1;
+    xlator_t *subvol = NULL;
+    inode_t *inode = NULL;
+    loc_t loc = {
+        0,
+    };
+    struct iatt iatt = {
+        0,
+    };
+    uuid_t gfid;
+    dict_t *xattr_req = NULL;
+    struct glfs_object *object = NULL;
+
+    /* validate in args */
+    if ((fs == NULL) || (parent == NULL) || (path == NULL)) {
+        errno = EINVAL;
+        return NULL;
+    }
+
+    DECLARE_OLD_THIS;
+    __GLFS_ENTRY_VALIDATE_FS(fs, invalid_fs);
+
+    /* get the active volume */
+    subvol = glfs_active_subvol(fs);
+    if (!subvol) {
+        ret = -1;
+        errno = EIO;
+        goto out;
+    }
+
+    /* get/refresh the in arg objects inode in correlation to the xlator */
+    inode = glfs_resolve_inode(fs, subvol, parent);
+    if (!inode) {
+        errno = ESTALE;
+        goto out;
+    }
+
+    xattr_req = dict_new();
+    if (!xattr_req) {
+        ret = -1;
+        errno = ENOMEM;
+        goto out;
+    }
+
+    gf_uuid_generate(gfid);
+    ret = dict_set_gfuuid(xattr_req, "gfid-req", gfid, true);
+    if (ret) {
+        ret = -1;
+        errno = ENOMEM;
+        goto out;
+    }
+
+    GLFS_LOC_FILL_PINODE(inode, loc, ret, errno, out, path);
+
+    /* fop/op */
+    ret = syncop_mkdir(subvol, &loc, mode, &iatt, xattr_req, NULL);
+    DECODE_SYNCOP_ERR(ret);
+
+    /* populate out args */
+    if (ret == 0) {
+        ret = glfs_loc_link(&loc, &iatt);
+        if (ret != 0) {
+            goto out;
         }
 
-        GLFS_LOC_FILL_PINODE (inode, loc, ret, errno, out, path);
-
-        /* fop/op */
-        ret = syncop_mkdir (subvol, &loc, mode, &iatt, xattr_req, NULL);
-        DECODE_SYNCOP_ERR (ret);
-
-        /* populate out args */
-        if ( ret == 0 )  {
-                ret = glfs_loc_link (&loc, &iatt);
-                if (ret != 0) {
-                        goto out;
-                }
-
-                if (stat)
-                        glfs_iatt_to_stat (fs, &iatt, stat);
+        if (stat)
+            glfs_iatt_to_stat(fs, &iatt, stat);
 
-                ret = glfs_create_object (&loc, &object);
-        }
+        ret = glfs_create_object(&loc, &object);
+    }
 
 out:
-        if (ret && object != NULL) {
-                glfs_h_close (object);
-                object = NULL;
-        }
+    if (ret && object != NULL) {
+        glfs_h_close(object);
+        object = NULL;
+    }
 
-        loc_wipe(&loc);
+    loc_wipe(&loc);
 
-        if (inode)
-                inode_unref (inode);
+    if (inode)
+        inode_unref(inode);
 
-        if (xattr_req)
-                dict_unref (xattr_req);
+    if (xattr_req)
+        dict_unref(xattr_req);
 
-        glfs_subvol_done (fs, subvol);
+    glfs_subvol_done(fs, subvol);
 
-        __GLFS_EXIT_FS;
+    __GLFS_EXIT_FS;
 
 invalid_fs:
-        return object;
+    return object;
 }
 
-GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_h_mkdir, 3.4.2);
-
-
+GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_h_mknod, 3.4.2)
 struct glfs_object *
-pub_glfs_h_mknod (struct glfs *fs, struct glfs_object *parent, const char *path,
-                  mode_t mode, dev_t dev, struct stat *stat)
+pub_glfs_h_mknod(struct glfs *fs, struct glfs_object *parent, const char *path,
+                 mode_t mode, dev_t dev, struct stat *stat)
 {
-        int                 ret = -1;
-        xlator_t           *subvol = NULL;
-        inode_t            *inode = NULL;
-        loc_t               loc = {0, };
-        struct iatt         iatt = {0, };
-        uuid_t              gfid;
-        dict_t             *xattr_req = NULL;
-        struct glfs_object *object = NULL;
-
-        /* validate in args */
-        if ((fs == NULL) || (parent == NULL) || (path == NULL)) {
-                errno = EINVAL;
-                return NULL;
-        }
-
-        DECLARE_OLD_THIS;
-        __GLFS_ENTRY_VALIDATE_FS (fs, invalid_fs);
-
-        /* get the active volume */
-        subvol = glfs_active_subvol (fs);
-        if (!subvol) {
-                ret = -1;
-                errno = EIO;
-                goto out;
-        }
-
-        /* get/refresh the in arg objects inode in correlation to the xlator */
-        inode = glfs_resolve_inode (fs, subvol, parent);
-        if (!inode) {
-                errno = ESTALE;
-                goto out;
-        }
-
-        xattr_req = dict_new ();
-        if (!xattr_req) {
-                ret = -1;
-                errno = ENOMEM;
-                goto out;
-        }
-
-        gf_uuid_generate (gfid);
-        ret = dict_set_static_bin (xattr_req, "gfid-req", gfid, 16);
-        if (ret) {
-                ret = -1;
-                errno = ENOMEM;
-                goto out;
+    int ret = -1;
+    xlator_t *subvol = NULL;
+    inode_t *inode = NULL;
+    loc_t loc = {
+        0,
+    };
+    struct iatt iatt = {
+        0,
+    };
+    uuid_t gfid;
+    dict_t *xattr_req = NULL;
+    struct glfs_object *object = NULL;
+
+    /* validate in args */
+    if ((fs == NULL) || (parent == NULL) || (path == NULL)) {
+        errno = EINVAL;
+        return NULL;
+    }
+
+    DECLARE_OLD_THIS;
+    __GLFS_ENTRY_VALIDATE_FS(fs, invalid_fs);
+
+    /* get the active volume */
+    subvol = glfs_active_subvol(fs);
+    if (!subvol) {
+        ret = -1;
+        errno = EIO;
+        goto out;
+    }
+
+    /* get/refresh the in arg objects inode in correlation to the xlator */
+    inode = glfs_resolve_inode(fs, subvol, parent);
+    if (!inode) {
+        errno = ESTALE;
+        goto out;
+    }
+
+    xattr_req = dict_new();
+    if (!xattr_req) {
+        ret = -1;
+        errno = ENOMEM;
+        goto out;
+    }
+
+    gf_uuid_generate(gfid);
+    ret = dict_set_gfuuid(xattr_req, "gfid-req", gfid, true);
+    if (ret) {
+        ret = -1;
+        errno = ENOMEM;
+        goto out;
+    }
+
+    GLFS_LOC_FILL_PINODE(inode, loc, ret, errno, out, path);
+
+    /* fop/op */
+    ret = syncop_mknod(subvol, &loc, mode, dev, &iatt, xattr_req, NULL);
+    DECODE_SYNCOP_ERR(ret);
+
+    /* populate out args */
+    if (ret == 0) {
+        ret = glfs_loc_link(&loc, &iatt);
+        if (ret != 0) {
+            goto out;
         }
 
-        GLFS_LOC_FILL_PINODE (inode, loc, ret, errno, out, path);
-
-        /* fop/op */
-        ret = syncop_mknod (subvol, &loc, mode, dev, &iatt, xattr_req, NULL);
-        DECODE_SYNCOP_ERR (ret);
-
-        /* populate out args */
-        if (ret == 0) {
-                ret = glfs_loc_link (&loc, &iatt);
-                if (ret != 0) {
-                        goto out;
-                }
-
-                if (stat)
-                        glfs_iatt_to_stat (fs, &iatt, stat);
+        if (stat)
+            glfs_iatt_to_stat(fs, &iatt, stat);
 
-                ret = glfs_create_object (&loc, &object);
-        }
+        ret = glfs_create_object(&loc, &object);
+    }
 out:
-        if (ret && object != NULL) {
-                glfs_h_close (object);
-                object = NULL;
-        }
+    if (ret && object != NULL) {
+        glfs_h_close(object);
+        object = NULL;
+    }
 
-        loc_wipe(&loc);
+    loc_wipe(&loc);
 
-        if (inode)
-                inode_unref (inode);
+    if (inode)
+        inode_unref(inode);
 
-        if (xattr_req)
-                dict_unref (xattr_req);
+    if (xattr_req)
+        dict_unref(xattr_req);
 
-        glfs_subvol_done (fs, subvol);
+    glfs_subvol_done(fs, subvol);
 
-        __GLFS_EXIT_FS;
+    __GLFS_EXIT_FS;
 
 invalid_fs:
-        return object;
+    return object;
 }
 
-GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_h_mknod, 3.4.2);
-
-
+GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_h_unlink, 3.4.2)
 int
-pub_glfs_h_unlink (struct glfs *fs, struct glfs_object *parent, const char *path)
+pub_glfs_h_unlink(struct glfs *fs, struct glfs_object *parent, const char *path)
 {
-        int                 ret = -1;
-        xlator_t           *subvol = NULL;
-        inode_t            *inode = NULL;
-        loc_t               loc = {0, };
-
-        /* validate in args */
-        if ((fs == NULL) || (parent == NULL) || (path == NULL)) {
-                errno = EINVAL;
-                return -1;
-        }
-
-        DECLARE_OLD_THIS;
-        __GLFS_ENTRY_VALIDATE_FS (fs, invalid_fs);
-
-        /* get the active volume */
-        subvol = glfs_active_subvol (fs);
-        if ( !subvol ) {
-                ret = -1;
-                errno = EIO;
-                goto out;
-        }
-
-        /* get/refresh the in arg objects inode in correlation to the xlator */
-        inode = glfs_resolve_inode (fs, subvol, parent);
-        if (!inode) {
-                errno = ESTALE;
-                goto out;
-        }
-
-        ret = glfs_resolve_at (fs, subvol, inode, path, &loc, NULL, 0 , 0);
+    int ret = -1;
+    xlator_t *subvol = NULL;
+    inode_t *inode = NULL;
+    loc_t loc = {
+        0,
+    };
+
+    /* validate in args */
+    if ((fs == NULL) || (parent == NULL) || (path == NULL)) {
+        errno = EINVAL;
+        return -1;
+    }
+
+    DECLARE_OLD_THIS;
+    __GLFS_ENTRY_VALIDATE_FS(fs, invalid_fs);
+
+    /* get the active volume */
+    subvol = glfs_active_subvol(fs);
+    if (!subvol) {
+        ret = -1;
+        errno = EIO;
+        goto out;
+    }
+
+    /* get/refresh the in arg objects inode in correlation to the xlator */
+    inode = glfs_resolve_inode(fs, subvol, parent);
+    if (!inode) {
+        errno = ESTALE;
+        goto out;
+    }
+
+    ret = glfs_resolve_at(fs, subvol, inode, path, &loc, NULL, 0, 0);
+    if (ret != 0) {
+        goto out;
+    }
+
+    if (!IA_ISDIR(loc.inode->ia_type)) {
+        ret = syncop_unlink(subvol, &loc, NULL, NULL);
+        DECODE_SYNCOP_ERR(ret);
         if (ret != 0) {
-                goto out;
+            goto out;
         }
-
-        if (!IA_ISDIR(loc.inode->ia_type)) {
-                ret = syncop_unlink (subvol, &loc, NULL, NULL);
-                DECODE_SYNCOP_ERR (ret);
-                if (ret != 0) {
-                        goto out;
-                }
-        } else {
-                ret = syncop_rmdir (subvol, &loc, 0, NULL, NULL);
-                DECODE_SYNCOP_ERR (ret);
-                if (ret != 0) {
-                        goto out;
-                }
+    } else {
+        ret = syncop_rmdir(subvol, &loc, 0, NULL, NULL);
+        DECODE_SYNCOP_ERR(ret);
+        if (ret != 0) {
+            goto out;
         }
+    }
 
-        if (ret == 0)
-                ret = glfs_loc_unlink (&loc);
+    if (ret == 0)
+        ret = glfs_loc_unlink(&loc);
 
 out:
-        loc_wipe (&loc);
+    loc_wipe(&loc);
 
-        if (inode)
-                inode_unref (inode);
+    if (inode)
+        inode_unref(inode);
 
-        glfs_subvol_done (fs, subvol);
+    glfs_subvol_done(fs, subvol);
 
-        __GLFS_EXIT_FS;
+    __GLFS_EXIT_FS;
 
 invalid_fs:
-        return ret;
+    return ret;
 }
 
-GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_h_unlink, 3.4.2);
-
-
+GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_h_opendir, 3.4.2)
 struct glfs_fd *
-pub_glfs_h_opendir (struct glfs *fs, struct glfs_object *object)
+pub_glfs_h_opendir(struct glfs *fs, struct glfs_object *object)
 {
-        int              ret = -1;
-        struct glfs_fd  *glfd = NULL;
-        xlator_t        *subvol = NULL;
-        inode_t         *inode = NULL;
-        loc_t            loc = {0, };
-
-        /* validate in args */
-        if ((fs == NULL) || (object == NULL)) {
-                errno = EINVAL;
-                return NULL;
-        }
-
-        DECLARE_OLD_THIS;
-        __GLFS_ENTRY_VALIDATE_FS (fs, invalid_fs);
-
-        /* get the active volume */
-        subvol = glfs_active_subvol (fs);
-        if (!subvol) {
-                ret = -1;
-                errno = EIO;
-                goto out;
-        }
-
-        /* get/refresh the in arg objects inode in correlation to the xlator */
-        inode = glfs_resolve_inode (fs, subvol, object);
-        if (!inode) {
-                errno = ESTALE;
-                goto out;
-        }
-
-        if (!IA_ISDIR (inode->ia_type)) {
-                ret = -1;
-                errno = ENOTDIR;
-                goto out;
-        }
-
-        glfd = glfs_fd_new (fs);
-        if (!glfd)
-                goto out;
-
-        INIT_LIST_HEAD (&glfd->entries);
-
-        glfd->fd = fd_create (inode, getpid());
-        if (!glfd->fd) {
-                ret = -1;
-                errno = ENOMEM;
-                goto out;
-        }
-
-        GLFS_LOC_FILL_INODE (inode, loc, out);
-
-        /* fop/op */
-        ret = syncop_opendir (subvol, &loc, glfd->fd, NULL, NULL);
-        DECODE_SYNCOP_ERR (ret);
+    int ret = -1;
+    struct glfs_fd *glfd = NULL;
+    xlator_t *subvol = NULL;
+    inode_t *inode = NULL;
+    loc_t loc = {
+        0,
+    };
+
+    /* validate in args */
+    if ((fs == NULL) || (object == NULL)) {
+        errno = EINVAL;
+        return NULL;
+    }
+
+    DECLARE_OLD_THIS;
+    __GLFS_ENTRY_VALIDATE_FS(fs, invalid_fs);
+
+    /* get the active volume */
+    subvol = glfs_active_subvol(fs);
+    if (!subvol) {
+        ret = -1;
+        errno = EIO;
+        goto out;
+    }
+
+    /* get/refresh the in arg objects inode in correlation to the xlator */
+    inode = glfs_resolve_inode(fs, subvol, object);
+    if (!inode) {
+        errno = ESTALE;
+        goto out;
+    }
+
+    if (!IA_ISDIR(inode->ia_type)) {
+        ret = -1;
+        errno = ENOTDIR;
+        goto out;
+    }
+
+    glfd = glfs_fd_new(fs);
+    if (!glfd)
+        goto out;
+
+    INIT_LIST_HEAD(&glfd->entries);
+
+    glfd->fd = fd_create(inode, getpid());
+    if (!glfd->fd) {
+        ret = -1;
+        errno = ENOMEM;
+        goto out;
+    }
+
+    GLFS_LOC_FILL_INODE(inode, loc, out);
+
+    /* fop/op */
+    ret = syncop_opendir(subvol, &loc, glfd->fd, NULL, NULL);
+    DECODE_SYNCOP_ERR(ret);
 
 out:
-        loc_wipe (&loc);
+    loc_wipe(&loc);
 
-        if (inode)
-                inode_unref (inode);
+    if (inode)
+        inode_unref(inode);
 
-        if (ret && glfd) {
-                glfs_fd_destroy (glfd);
-                glfd = NULL;
-        } else {
-                fd_bind (glfd->fd);
-                glfs_fd_bind (glfd);
-        }
+    if (ret && glfd) {
+        GF_REF_PUT(glfd);
+        glfd = NULL;
+    } else if (glfd) {
+        glfd_set_state_bind(glfd);
+    }
 
-        glfs_subvol_done (fs, subvol);
+    glfs_subvol_done(fs, subvol);
 
-        __GLFS_EXIT_FS;
+    __GLFS_EXIT_FS;
 
 invalid_fs:
-        return glfd;
+    return glfd;
 }
 
-GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_h_opendir, 3.4.2);
-
-
+GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_h_access, 3.6.0)
 int
-pub_glfs_h_access (struct glfs *fs, struct glfs_object *object, int mask)
+pub_glfs_h_access(struct glfs *fs, struct glfs_object *object, int mask)
 {
-	int              ret = -1;
-	xlator_t        *subvol = NULL;
-	inode_t         *inode = NULL;
-	loc_t            loc = {0, };
-
-	DECLARE_OLD_THIS;
-
-	/* validate in args */
-	if ((fs == NULL) || (object == NULL)) {
-		errno = EINVAL;
-		return ret;
-	}
-
-        __GLFS_ENTRY_VALIDATE_FS (fs, invalid_fs);
+    int ret = -1;
+    xlator_t *subvol = NULL;
+    inode_t *inode = NULL;
+    loc_t loc = {
+        0,
+    };
+
+    DECLARE_OLD_THIS;
+
+    /* validate in args */
+    if ((fs == NULL) || (object == NULL)) {
+        errno = EINVAL;
+        return ret;
+    }
 
-	/* get the active volume */
-	subvol = glfs_active_subvol (fs);
-	if (!subvol) {
-		ret = -1;
-		errno = EIO;
-		goto out;
-	}
+    __GLFS_ENTRY_VALIDATE_FS(fs, invalid_fs);
 
-	/* get/refresh the in arg objects inode in correlation to the xlator */
-	inode = glfs_resolve_inode (fs, subvol, object);
-	if (!inode) {
-		errno = ESTALE;
-		goto out;
-	}
+    /* get the active volume */
+    subvol = glfs_active_subvol(fs);
+    if (!subvol) {
+        ret = -1;
+        errno = EIO;
+        goto out;
+    }
 
+    /* get/refresh the in arg objects inode in correlation to the xlator */
+    inode = glfs_resolve_inode(fs, subvol, object);
+    if (!inode) {
+        errno = ESTALE;
+        goto out;
+    }
 
-	GLFS_LOC_FILL_INODE (inode, loc, out);
+    GLFS_LOC_FILL_INODE(inode, loc, out);
 
-	/* fop/op */
+    /* fop/op */
 
-	ret = syncop_access (subvol, &loc, mask, NULL, NULL);
-        DECODE_SYNCOP_ERR (ret);
+    ret = syncop_access(subvol, &loc, mask, NULL, NULL);
+    DECODE_SYNCOP_ERR(ret);
 
 out:
-	loc_wipe (&loc);
-
-	if (inode)
-		inode_unref (inode);
+    loc_wipe(&loc);
 
+    if (inode)
+        inode_unref(inode);
 
-	glfs_subvol_done (fs, subvol);
+    glfs_subvol_done(fs, subvol);
 
-        __GLFS_EXIT_FS;
+    __GLFS_EXIT_FS;
 
 invalid_fs:
-	return ret;
+    return ret;
 }
 
-GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_h_access, 3.6.0);
-
-
+GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_h_extract_handle, 3.4.2)
 ssize_t
-pub_glfs_h_extract_handle (struct glfs_object *object, unsigned char *handle,
-                           int len)
+pub_glfs_h_extract_handle(struct glfs_object *object, unsigned char *handle,
+                          int len)
 {
-        ssize_t ret = -1;
+    ssize_t ret = -1;
 
-        /* validate in args */
-        if (object == NULL) {
-                errno = EINVAL;
-                goto out;
-        }
+    /* validate in args */
+    if (object == NULL) {
+        errno = EINVAL;
+        goto out;
+    }
 
-        if (!handle || !len) {
-                ret = GFAPI_HANDLE_LENGTH;
-                goto out;
-        }
+    if (!handle || !len) {
+        ret = GFAPI_HANDLE_LENGTH;
+        goto out;
+    }
 
-        if (len < GFAPI_HANDLE_LENGTH)
-        {
-                errno = ERANGE;
-                goto out;
-        }
+    if (len < GFAPI_HANDLE_LENGTH) {
+        errno = ERANGE;
+        goto out;
+    }
 
-        memcpy (handle, object->gfid, GFAPI_HANDLE_LENGTH);
+    memcpy(handle, object->gfid, GFAPI_HANDLE_LENGTH);
 
-        ret = GFAPI_HANDLE_LENGTH;
+    ret = GFAPI_HANDLE_LENGTH;
 
 out:
-        return ret;
+    return ret;
 }
 
-GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_h_extract_handle, 3.4.2);
-
-
+GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_h_create_from_handle, 3.4.2)
 struct glfs_object *
-pub_glfs_h_create_from_handle (struct glfs *fs, unsigned char *handle, int len,
-                               struct stat *stat)
+pub_glfs_h_create_from_handle(struct glfs *fs, unsigned char *handle, int len,
+                              struct stat *stat)
 {
-        loc_t               loc = {0, };
-        int                 ret = -1;
-        struct iatt         iatt = {0, };
-        inode_t            *newinode = NULL;
-        xlator_t           *subvol = NULL;
-        struct glfs_object *object = NULL;
-
-        /* validate in args */
-        if ((fs == NULL) || (handle == NULL) || (len != GFAPI_HANDLE_LENGTH)) {
-                errno = EINVAL;
-                return NULL;
-        }
+    loc_t loc = {
+        0,
+    };
+    int ret = -1;
+    struct iatt iatt = {
+        0,
+    };
+    inode_t *newinode = NULL;
+    xlator_t *subvol = NULL;
+    struct glfs_object *object = NULL;
+    uint64_t ctx_value = LOOKUP_NOT_NEEDED;
+    gf_boolean_t lookup_needed = _gf_false;
+
+    /* validate in args */
+    if ((fs == NULL) || (handle == NULL) || (len != GFAPI_HANDLE_LENGTH)) {
+        errno = EINVAL;
+        return NULL;
+    }
 
-        DECLARE_OLD_THIS;
-        __GLFS_ENTRY_VALIDATE_FS (fs, invalid_fs);
+    DECLARE_OLD_THIS;
+    __GLFS_ENTRY_VALIDATE_FS(fs, invalid_fs);
 
-        /* get the active volume */
-        subvol = glfs_active_subvol (fs);
-        if (!subvol) {
-                errno = EIO;
-                goto out;
-        }
+    /* get the active volume */
+    subvol = glfs_active_subvol(fs);
+    if (!subvol) {
+        errno = EIO;
+        goto out;
+    }
 
-        memcpy (loc.gfid, handle, GFAPI_HANDLE_LENGTH);
+    memcpy(loc.gfid, handle, GFAPI_HANDLE_LENGTH);
 
-        /* make sure the gfid received is valid */
-        GF_VALIDATE_OR_GOTO ("glfs_h_create_from_handle",
-                             !(gf_uuid_is_null (loc.gfid)), out);
+    /* make sure the gfid received is valid */
+    GF_VALIDATE_OR_GOTO("glfs_h_create_from_handle",
+                        !(gf_uuid_is_null(loc.gfid)), out);
 
-        newinode = inode_find (subvol->itable, loc.gfid);
-        if (newinode) {
-                if (!stat) /* No need of lookup */
-                        goto found;
+    newinode = inode_find(subvol->itable, loc.gfid);
+    if (newinode) {
+        if (!stat) /* No need of lookup */
+            goto found;
 
-                loc.inode = newinode;
+        lookup_needed = inode_needs_lookup(newinode, THIS);
+        if (lookup_needed) {
+            loc.inode = newinode;
         } else {
-                loc.inode = inode_new (subvol->itable);
-                if (!loc.inode) {
-                        errno = ENOMEM;
-                        goto out;
-                }
-        }
-
-        ret = syncop_lookup (subvol, &loc, &iatt, 0, 0, 0);
-        DECODE_SYNCOP_ERR (ret);
-        if (ret) {
-                gf_msg (subvol->name, GF_LOG_WARNING, errno,
-                        API_MSG_INODE_REFRESH_FAILED,
-                        "inode refresh of %s failed: %s",
-                        uuid_utoa (loc.gfid), strerror (errno));
-                goto out;
-        }
-
-        newinode = inode_link (loc.inode, 0, 0, &iatt);
-        if (newinode)
-                inode_lookup (newinode);
-        else {
-                gf_msg (subvol->name, GF_LOG_WARNING, EINVAL,
-                        API_MSG_INVALID_ENTRY,
-                        "inode linking of %s failed: %s",
-                        uuid_utoa (loc.gfid), strerror (errno));
-                errno = EINVAL;
-                goto out;
-        }
-
-        /* populate stat */
-        if (stat)
-                glfs_iatt_to_stat (fs, &iatt, stat);
+            /* populate loc */
+            GLFS_LOC_FILL_INODE(newinode, loc, fill_out);
+
+            /* fop/op */
+            ret = syncop_stat(subvol, &loc, &iatt, NULL, NULL);
+            DECODE_SYNCOP_ERR(ret);
+
+            if (ret) {
+            fill_out:
+                /* Drop the reference hold in inode_find */
+                inode_unref(newinode);
+                goto out;
+            }
+
+            glfs_iatt_to_stat(fs, &iatt, stat);
+            goto found;
+        }
+    } else {
+        loc.inode = inode_new(subvol->itable);
+        if (!loc.inode) {
+            errno = ENOMEM;
+            goto out;
+        }
+    }
+
+    ret = syncop_lookup(subvol, &loc, &iatt, 0, 0, 0);
+    DECODE_SYNCOP_ERR(ret);
+    if (ret) {
+        gf_smsg(subvol->name, GF_LOG_WARNING, errno,
+                API_MSG_INODE_REFRESH_FAILED, "gfid=%s", uuid_utoa(loc.gfid),
+                "error=%s", strerror(errno), NULL);
+        goto out;
+    }
+
+    newinode = inode_link(loc.inode, 0, 0, &iatt);
+    if (newinode) {
+        if (newinode == loc.inode) {
+            inode_ctx_set(newinode, THIS, &ctx_value);
+        }
+        inode_lookup(newinode);
+    } else {
+        gf_smsg(subvol->name, GF_LOG_WARNING, errno, API_MSG_INODE_LINK_FAILED,
+                "gfid=%s", uuid_utoa(loc.gfid), NULL);
+        goto out;
+    }
+
+    /* populate stat */
+    if (stat)
+        glfs_iatt_to_stat(fs, &iatt, stat);
 
 found:
-        object = GF_CALLOC (1, sizeof(struct glfs_object),
-                            glfs_mt_glfs_object_t);
-        if (object == NULL) {
-                errno = ENOMEM;
-                ret = -1;
-                goto out;
-        }
+    object = GF_CALLOC(1, sizeof(struct glfs_object), glfs_mt_glfs_object_t);
+    if (object == NULL) {
+        errno = ENOMEM;
+        ret = -1;
+        goto out;
+    }
 
-        /* populate the return object */
-        object->inode = newinode;
-        gf_uuid_copy (object->gfid, object->inode->gfid);
+    /* populate the return object */
+    object->inode = newinode;
+    gf_uuid_copy(object->gfid, object->inode->gfid);
 
 out:
-        /* TODO: Check where the inode ref is being held? */
-        loc_wipe (&loc);
+    /* TODO: Check where the inode ref is being held? */
+    loc_wipe(&loc);
 
-        glfs_subvol_done (fs, subvol);
+    glfs_subvol_done(fs, subvol);
 
-        __GLFS_EXIT_FS;
+    __GLFS_EXIT_FS;
 
 invalid_fs:
-        return object;
+    return object;
 }
 
-GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_h_create_from_handle, 3.4.2);
-
-
+GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_h_close, 3.4.2)
 int
-pub_glfs_h_close (struct glfs_object *object)
+pub_glfs_h_close(struct glfs_object *object)
 {
-        inode_unref (object->inode);
-        GF_FREE (object);
+    /* since glfs_h_* objects hold a reference to inode
+     * it is safe to keep lookup count to '0' */
+    inode_forget(object->inode, 0);
+    inode_unref(object->inode);
+    GF_FREE(object);
 
-        return 0;
+    return 0;
 }
 
-GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_h_close, 3.4.2);
-
-
+GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_h_truncate, 3.4.2)
 int
-pub_glfs_h_truncate (struct glfs *fs, struct glfs_object *object, off_t offset)
+pub_glfs_h_truncate(struct glfs *fs, struct glfs_object *object, off_t offset)
 {
-        loc_t               loc = {0, };
-        int                 ret = -1;
-        xlator_t           *subvol = NULL;
-        inode_t            *inode = NULL;
+    loc_t loc = {
+        0,
+    };
+    int ret = -1;
+    xlator_t *subvol = NULL;
+    inode_t *inode = NULL;
+
+    DECLARE_OLD_THIS;
+
+    /* validate in args */
+    if (object == NULL) {
+        errno = EINVAL;
+        return -1;
+    }
 
-        DECLARE_OLD_THIS;
+    __GLFS_ENTRY_VALIDATE_FS(fs, invalid_fs);
 
-        /* validate in args */
-        if (object == NULL) {
-                errno = EINVAL;
-                return -1;
-        }
+    /* get the active volume */
+    subvol = glfs_active_subvol(fs);
+    if (!subvol) {
+        ret = -1;
+        errno = EIO;
+        goto out;
+    }
 
-        __GLFS_ENTRY_VALIDATE_FS (fs, invalid_fs);
+    /* get/refresh the in arg objects inode in correlation to the xlator */
+    inode = glfs_resolve_inode(fs, subvol, object);
+    if (!inode) {
+        errno = ESTALE;
+        goto out;
+    }
 
-        /* get the active volume */
-        subvol = glfs_active_subvol (fs);
-        if (!subvol) {
-                ret = -1;
-                errno = EIO;
-                goto out;
-        }
+    GLFS_LOC_FILL_INODE(inode, loc, out);
 
-        /* get/refresh the in arg objects inode in correlation to the xlator */
-        inode = glfs_resolve_inode (fs, subvol, object);
-        if (!inode) {
-                errno = ESTALE;
-                goto out;
-        }
+    /* fop/op */
+    ret = syncop_truncate(subvol, &loc, (off_t)offset, NULL, NULL);
+    DECODE_SYNCOP_ERR(ret);
 
-        GLFS_LOC_FILL_INODE (inode, loc, out);
-
-        /* fop/op */
-        ret = syncop_truncate (subvol, &loc, (off_t)offset, NULL, NULL);
-        DECODE_SYNCOP_ERR (ret);
-
-        /* populate out args */
-        if (ret == 0)
-                ret = glfs_loc_unlink (&loc);
+    /* populate out args */
+    if (ret == 0)
+        ret = glfs_loc_unlink(&loc);
 
 out:
-        loc_wipe (&loc);
+    loc_wipe(&loc);
 
-        if (inode)
-                inode_unref (inode);
+    if (inode)
+        inode_unref(inode);
 
-        glfs_subvol_done (fs, subvol);
+    glfs_subvol_done(fs, subvol);
 
-        __GLFS_EXIT_FS;
+    __GLFS_EXIT_FS;
 
 invalid_fs:
-        return ret;
+    return ret;
 }
 
-GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_h_truncate, 3.4.2);
-
-
+GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_h_symlink, 3.4.2)
 struct glfs_object *
-pub_glfs_h_symlink (struct glfs *fs, struct glfs_object *parent,
-                    const char *name, const char *data, struct stat *stat)
+pub_glfs_h_symlink(struct glfs *fs, struct glfs_object *parent,
+                   const char *name, const char *data, struct stat *stat)
 {
-        int                 ret = -1;
-        xlator_t           *subvol = NULL;
-        inode_t            *inode = NULL;
-        loc_t               loc = {0, };
-        struct iatt         iatt = {0, };
-        uuid_t              gfid;
-        dict_t             *xattr_req = NULL;
-        struct glfs_object *object = NULL;
-
-        DECLARE_OLD_THIS;
-
-        /* validate in args */
-        if ((parent == NULL) || (name == NULL) ||
-                (data == NULL)) {
-                errno = EINVAL;
-                return NULL;
-        }
-
-        __GLFS_ENTRY_VALIDATE_FS (fs, invalid_fs);
-
-        /* get the active volume */
-        subvol = glfs_active_subvol (fs);
-        if (!subvol) {
-                ret = -1;
-                errno = EIO;
-                goto out;
-        }
-
-        /* get/refresh the in arg objects inode in correlation to the xlator */
-        inode = glfs_resolve_inode (fs, subvol, parent);
-        if (!inode) {
-                errno = ESTALE;
-                goto out;
-        }
-
-        xattr_req = dict_new ();
-        if (!xattr_req) {
-                ret = -1;
-                errno = ENOMEM;
-                goto out;
-        }
-
-        gf_uuid_generate (gfid);
-        ret = dict_set_static_bin (xattr_req, "gfid-req", gfid, 16);
-        if (ret) {
-                ret = -1;
-                errno = ENOMEM;
-                goto out;
+    int ret = -1;
+    xlator_t *subvol = NULL;
+    inode_t *inode = NULL;
+    loc_t loc = {
+        0,
+    };
+    struct iatt iatt = {
+        0,
+    };
+    uuid_t gfid;
+    dict_t *xattr_req = NULL;
+    struct glfs_object *object = NULL;
+
+    DECLARE_OLD_THIS;
+
+    /* validate in args */
+    if ((parent == NULL) || (name == NULL) || (data == NULL)) {
+        errno = EINVAL;
+        return NULL;
+    }
+
+    __GLFS_ENTRY_VALIDATE_FS(fs, invalid_fs);
+
+    /* get the active volume */
+    subvol = glfs_active_subvol(fs);
+    if (!subvol) {
+        ret = -1;
+        errno = EIO;
+        goto out;
+    }
+
+    /* get/refresh the in arg objects inode in correlation to the xlator */
+    inode = glfs_resolve_inode(fs, subvol, parent);
+    if (!inode) {
+        errno = ESTALE;
+        goto out;
+    }
+
+    xattr_req = dict_new();
+    if (!xattr_req) {
+        ret = -1;
+        errno = ENOMEM;
+        goto out;
+    }
+
+    gf_uuid_generate(gfid);
+    ret = dict_set_gfuuid(xattr_req, "gfid-req", gfid, true);
+    if (ret) {
+        ret = -1;
+        errno = ENOMEM;
+        goto out;
+    }
+
+    GLFS_LOC_FILL_PINODE(inode, loc, ret, errno, out, name);
+
+    /* fop/op */
+    ret = syncop_symlink(subvol, &loc, data, &iatt, xattr_req, NULL);
+    DECODE_SYNCOP_ERR(ret);
+
+    /* populate out args */
+    if (ret == 0) {
+        ret = glfs_loc_link(&loc, &iatt);
+        if (ret != 0) {
+            goto out;
         }
 
-        GLFS_LOC_FILL_PINODE (inode, loc, ret, errno, out, name);
-
-        /* fop/op */
-        ret = syncop_symlink (subvol, &loc, data, &iatt, xattr_req, NULL);
-        DECODE_SYNCOP_ERR (ret);
-
-        /* populate out args */
-        if (ret == 0) {
-                ret = glfs_loc_link (&loc, &iatt);
-                if (ret != 0) {
-                        goto out;
-                }
-
-                if (stat)
-                        glfs_iatt_to_stat (fs, &iatt, stat);
+        if (stat)
+            glfs_iatt_to_stat(fs, &iatt, stat);
 
-                ret = glfs_create_object (&loc, &object);
-        }
+        ret = glfs_create_object(&loc, &object);
+    }
 
 out:
-        if (ret && object != NULL) {
-                pub_glfs_h_close (object);
-                object = NULL;
-        }
+    if (ret && object != NULL) {
+        pub_glfs_h_close(object);
+        object = NULL;
+    }
 
-        loc_wipe(&loc);
+    loc_wipe(&loc);
 
-        if (inode)
-                inode_unref (inode);
+    if (inode)
+        inode_unref(inode);
 
-        if (xattr_req)
-                dict_unref (xattr_req);
+    if (xattr_req)
+        dict_unref(xattr_req);
 
-        glfs_subvol_done (fs, subvol);
+    glfs_subvol_done(fs, subvol);
 
-        __GLFS_EXIT_FS;
+    __GLFS_EXIT_FS;
 
 invalid_fs:
-        return object;
+    return object;
 }
 
-GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_h_symlink, 3.4.2);
-
-
+GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_h_readlink, 3.4.2)
 int
-pub_glfs_h_readlink (struct glfs *fs, struct glfs_object *object, char *buf,
-                     size_t bufsiz)
+pub_glfs_h_readlink(struct glfs *fs, struct glfs_object *object, char *buf,
+                    size_t bufsiz)
 {
-        loc_t               loc = {0, };
-        int                 ret = -1;
-        xlator_t           *subvol = NULL;
-        inode_t            *inode = NULL;
-        char               *linkval = NULL;
-
-        DECLARE_OLD_THIS;
-
-        /* validate in args */
-        if ((object == NULL) || (buf == NULL)) {
-                errno = EINVAL;
-                return -1;
-        }
+    loc_t loc = {
+        0,
+    };
+    int ret = -1;
+    xlator_t *subvol = NULL;
+    inode_t *inode = NULL;
+    char *linkval = NULL;
+
+    DECLARE_OLD_THIS;
+
+    /* validate in args */
+    if ((object == NULL) || (buf == NULL)) {
+        errno = EINVAL;
+        return -1;
+    }
 
-        __GLFS_ENTRY_VALIDATE_FS (fs, invalid_fs);
+    __GLFS_ENTRY_VALIDATE_FS(fs, invalid_fs);
 
-        /* get the active volume */
-        subvol = glfs_active_subvol (fs);
-        if (!subvol) {
-                ret = -1;
-                errno = EIO;
-                goto out;
-        }
+    /* get the active volume */
+    subvol = glfs_active_subvol(fs);
+    if (!subvol) {
+        ret = -1;
+        errno = EIO;
+        goto out;
+    }
 
-        /* get/refresh the in arg objects inode in correlation to the xlator */
-        inode = glfs_resolve_inode (fs, subvol, object);
-        if (!inode) {
-                errno = ESTALE;
-                goto out;
-        }
+    /* get/refresh the in arg objects inode in correlation to the xlator */
+    inode = glfs_resolve_inode(fs, subvol, object);
+    if (!inode) {
+        errno = ESTALE;
+        goto out;
+    }
 
-        GLFS_LOC_FILL_INODE (inode, loc, out);
+    GLFS_LOC_FILL_INODE(inode, loc, out);
 
-        /* fop/op */
-        ret = syncop_readlink (subvol, &loc, &linkval, bufsiz, NULL, NULL);
-        DECODE_SYNCOP_ERR (ret);
+    /* fop/op */
+    ret = syncop_readlink(subvol, &loc, &linkval, bufsiz, NULL, NULL);
+    DECODE_SYNCOP_ERR(ret);
 
-        /* populate out args */
-        if (ret > 0)
-                memcpy (buf, linkval, ret);
+    /* populate out args */
+    if (ret > 0)
+        memcpy(buf, linkval, ret);
 
 out:
-        loc_wipe (&loc);
+    loc_wipe(&loc);
 
-        if (inode)
-                inode_unref (inode);
+    if (inode)
+        inode_unref(inode);
 
-        if (linkval)
-                GF_FREE (linkval);
+    if (linkval)
+        GF_FREE(linkval);
 
-        glfs_subvol_done (fs, subvol);
+    glfs_subvol_done(fs, subvol);
 
-        __GLFS_EXIT_FS;
+    __GLFS_EXIT_FS;
 
 invalid_fs:
-        return ret;
+    return ret;
 }
 
-GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_h_readlink, 3.4.2);
-
-
+GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_h_link, 3.4.2)
 int
-pub_glfs_h_link (struct glfs *fs, struct glfs_object *linksrc,
-             struct glfs_object *parent, const char *name)
+pub_glfs_h_link(struct glfs *fs, struct glfs_object *linksrc,
+                struct glfs_object *parent, const char *name)
 {
-        int                 ret = -1;
-        xlator_t           *subvol = NULL;
-        inode_t            *inode = NULL;
-        inode_t            *pinode = NULL;
-        loc_t               oldloc = {0, };
-        loc_t               newloc = {0, };
-        struct iatt         iatt = {0, };
-
-        DECLARE_OLD_THIS;
-
-        /* validate in args */
-        if ((linksrc == NULL) || (parent == NULL) ||
-                (name == NULL)) {
-                errno = EINVAL;
-                return -1;
-        }
-
-        __GLFS_ENTRY_VALIDATE_FS (fs, invalid_fs);
-
-        /* get the active volume */
-        subvol = glfs_active_subvol (fs);
-        if (!subvol) {
-                ret = -1;
-                errno = EIO;
-                goto out;
-        }
-
-        /* get/refresh the in arg objects inode in correlation to the xlator */
-        inode = glfs_resolve_inode (fs, subvol, linksrc);
-        if (!inode) {
-                errno = ESTALE;
-                goto out;
-        }
+    int ret = -1;
+    xlator_t *subvol = NULL;
+    inode_t *inode = NULL;
+    inode_t *pinode = NULL;
+    loc_t oldloc = {
+        0,
+    };
+    loc_t newloc = {
+        0,
+    };
+    struct iatt iatt = {
+        0,
+    };
+
+    DECLARE_OLD_THIS;
+
+    /* validate in args */
+    if ((linksrc == NULL) || (parent == NULL) || (name == NULL)) {
+        errno = EINVAL;
+        return -1;
+    }
+
+    __GLFS_ENTRY_VALIDATE_FS(fs, invalid_fs);
+
+    /* get the active volume */
+    subvol = glfs_active_subvol(fs);
+    if (!subvol) {
+        ret = -1;
+        errno = EIO;
+        goto out;
+    }
+
+    /* get/refresh the in arg objects inode in correlation to the xlator */
+    inode = glfs_resolve_inode(fs, subvol, linksrc);
+    if (!inode) {
+        errno = ESTALE;
+        goto out;
+    }
+
+    if (inode->ia_type == IA_IFDIR) {
+        ret = -1;
+        errno = EISDIR;
+        goto out;
+    }
+
+    GLFS_LOC_FILL_INODE(inode, oldloc, out);
+
+    /* get/refresh the in arg objects inode in correlation to the xlator */
+    pinode = glfs_resolve_inode(fs, subvol, parent);
+    if (!pinode) {
+        errno = ESTALE;
+        goto out;
+    }
+
+    /* setup newloc based on parent */
+    newloc.parent = inode_ref(pinode);
+    newloc.name = name;
+    ret = glfs_loc_touchup(&newloc);
+    if (ret != 0) {
+        errno = EINVAL;
+        goto out;
+    }
+
+    /* Filling the inode of the hard link to be same as that of the
+     * original file
+     */
+    newloc.inode = inode_ref(inode);
+
+    /* fop/op */
+    ret = syncop_link(subvol, &oldloc, &newloc, &iatt, NULL, NULL);
+    DECODE_SYNCOP_ERR(ret);
+
+    if (ret == 0)
+        ret = glfs_loc_link(&newloc, &iatt);
+out:
+    loc_wipe(&oldloc);
+    loc_wipe(&newloc);
 
-        if (inode->ia_type == IA_IFDIR) {
-                ret = -1;
-                errno = EISDIR;
-                goto out;
-        }
+    if (inode)
+        inode_unref(inode);
 
-        GLFS_LOC_FILL_INODE (inode, oldloc, out);
+    if (pinode)
+        inode_unref(pinode);
 
-        /* get/refresh the in arg objects inode in correlation to the xlator */
-        pinode = glfs_resolve_inode (fs, subvol, parent);
-        if (!pinode) {
-                errno = ESTALE;
-                goto out;
-        }
+    glfs_subvol_done(fs, subvol);
 
-        /* setup newloc based on parent */
-        newloc.parent = inode_ref (pinode);
-        newloc.name = name;
-        ret = glfs_loc_touchup (&newloc);
-        if (ret != 0) {
-                errno = EINVAL;
-                goto out;
-        }
+    __GLFS_EXIT_FS;
 
-        /* Filling the inode of the hard link to be same as that of the
-         * original file
-         */
-        newloc.inode = inode_ref (inode);
+invalid_fs:
+    return ret;
+}
 
-        /* fop/op */
-        ret = syncop_link (subvol, &oldloc, &newloc, &iatt, NULL, NULL);
-        DECODE_SYNCOP_ERR (ret);
+GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_h_rename, 3.4.2)
+int
+pub_glfs_h_rename(struct glfs *fs, struct glfs_object *olddir,
+                  const char *oldname, struct glfs_object *newdir,
+                  const char *newname)
+{
+    int ret = -1;
+    xlator_t *subvol = NULL;
+    inode_t *oldpinode = NULL;
+    inode_t *newpinode = NULL;
+    loc_t oldloc = {
+        0,
+    };
+    loc_t newloc = {
+        0,
+    };
+    struct iatt oldiatt = {
+        0,
+    };
+    struct iatt newiatt = {
+        0,
+    };
+
+    DECLARE_OLD_THIS;
+
+    /* validate in args */
+    if ((olddir == NULL) || (oldname == NULL) || (newdir == NULL) ||
+        (newname == NULL)) {
+        errno = EINVAL;
+        return -1;
+    }
+
+    __GLFS_ENTRY_VALIDATE_FS(fs, invalid_fs);
+
+    /* get the active volume */
+    subvol = glfs_active_subvol(fs);
+    if (!subvol) {
+        ret = -1;
+        errno = EIO;
+        goto out;
+    }
+
+    /* get/refresh the in arg objects inode in correlation to the xlator */
+    oldpinode = glfs_resolve_inode(fs, subvol, olddir);
+    if (!oldpinode) {
+        errno = ESTALE;
+        goto out;
+    }
+
+    ret = glfs_resolve_at(fs, subvol, oldpinode, oldname, &oldloc, &oldiatt, 0,
+                          0);
+    if (ret != 0) {
+        goto out;
+    }
+
+    /* get/refresh the in arg objects inode in correlation to the xlator */
+    newpinode = glfs_resolve_inode(fs, subvol, newdir);
+    if (!newpinode) {
+        errno = ESTALE;
+        goto out;
+    }
+
+    ret = glfs_resolve_at(fs, subvol, newpinode, newname, &newloc, &newiatt, 0,
+                          0);
+
+    if (ret && errno != ENOENT && newloc.parent)
+        goto out;
+
+    if (newiatt.ia_type != IA_INVAL) {
+        if ((oldiatt.ia_type == IA_IFDIR) != (newiatt.ia_type == IA_IFDIR)) {
+            /* Either both old and new must be dirs,
+             * or both must be non-dirs. Else, fail.
+             */
+            ret = -1;
+            errno = EEXIST;
+            goto out;
+        }
+    }
+
+    /* TODO: check if new or old is a prefix of the other, and fail EINVAL */
+
+    ret = syncop_rename(subvol, &oldloc, &newloc, NULL, NULL);
+    DECODE_SYNCOP_ERR(ret);
+
+    if (ret == 0) {
+        inode_rename(oldloc.parent->table, oldloc.parent, oldloc.name,
+                     newloc.parent, newloc.name, oldloc.inode, &oldiatt);
+
+        if (newloc.inode && !inode_has_dentry(newloc.inode))
+            inode_forget(newloc.inode, 0);
+    }
 
-        if (ret == 0)
-                ret = glfs_loc_link (&newloc, &iatt);
 out:
-        loc_wipe (&oldloc);
-        loc_wipe (&newloc);
+    loc_wipe(&oldloc);
+    loc_wipe(&newloc);
 
-        if (inode)
-                inode_unref (inode);
+    if (oldpinode)
+        inode_unref(oldpinode);
 
-        if (pinode)
-                inode_unref (pinode);
+    if (newpinode)
+        inode_unref(newpinode);
 
-        glfs_subvol_done (fs, subvol);
+    glfs_subvol_done(fs, subvol);
 
-        __GLFS_EXIT_FS;
+    __GLFS_EXIT_FS;
 
 invalid_fs:
-        return ret;
+    return ret;
 }
 
-GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_h_link, 3.4.2);
-
-
-int
-pub_glfs_h_rename (struct glfs *fs, struct glfs_object *olddir,
-                   const char *oldname, struct glfs_object *newdir,
-                   const char *newname)
+/*
+ * Given a handle/gfid, find if the corresponding inode is present in
+ * the inode table. If yes create and return the corresponding glfs_object.
+ */
+struct glfs_object *
+glfs_h_find_handle(struct glfs *fs, unsigned char *handle, int len)
 {
-        int                 ret = -1;
-        xlator_t           *subvol = NULL;
-        inode_t            *oldpinode = NULL;
-        inode_t            *newpinode = NULL;
-        loc_t               oldloc = {0, };
-        loc_t               newloc = {0, };
-        struct iatt         oldiatt = {0, };
-        struct iatt         newiatt = {0, };
-
-        DECLARE_OLD_THIS;
-
-        /* validate in args */
-        if ((olddir == NULL) || (oldname == NULL) ||
-                (newdir == NULL) || (newname == NULL)) {
-                errno = EINVAL;
-                return -1;
-        }
-
-        __GLFS_ENTRY_VALIDATE_FS (fs, invalid_fs);
-
-        /* get the active volume */
-        subvol = glfs_active_subvol (fs);
-        if ( !subvol ) {
-                ret = -1;
-                errno = EIO;
-                goto out;
-        }
+    inode_t *newinode = NULL;
+    xlator_t *subvol = NULL;
+    struct glfs_object *object = NULL;
+    uuid_t gfid;
+
+    /* validate in args */
+    if ((fs == NULL) || (handle == NULL) || (len != GFAPI_HANDLE_LENGTH)) {
+        errno = EINVAL;
+        return NULL;
+    }
 
-        /* get/refresh the in arg objects inode in correlation to the xlator */
-        oldpinode = glfs_resolve_inode (fs, subvol, olddir);
-        if (!oldpinode) {
-                errno = ESTALE;
-                goto out;
-        }
+    DECLARE_OLD_THIS;
+    __GLFS_ENTRY_VALIDATE_FS(fs, invalid_fs);
 
-        ret = glfs_resolve_at (fs, subvol, oldpinode, oldname, &oldloc,
-                                    &oldiatt, 0 , 0);
-        if (ret != 0) {
-                goto out;
-        }
+    /* get the active volume */
+    subvol = glfs_active_subvol(fs);
+    if (!subvol) {
+        errno = EIO;
+        goto out;
+    }
 
-        /* get/refresh the in arg objects inode in correlation to the xlator */
-        newpinode = glfs_resolve_inode (fs, subvol, newdir);
-        if (!newpinode) {
-                errno = ESTALE;
-                goto out;
-        }
+    memcpy(gfid, handle, GFAPI_HANDLE_LENGTH);
 
-        ret = glfs_resolve_at (fs, subvol, newpinode, newname, &newloc,
-                                    &newiatt, 0, 0);
+    /* make sure the gfid received is valid */
+    GF_VALIDATE_OR_GOTO("glfs_h_find_handle", !(gf_uuid_is_null(gfid)), out);
 
-        if (ret && errno != ENOENT && newloc.parent)
-                goto out;
+    newinode = inode_find(subvol->itable, gfid);
+    if (!newinode) {
+        goto out;
+    }
 
-        if (newiatt.ia_type != IA_INVAL) {
-                if ((oldiatt.ia_type == IA_IFDIR) !=
-                        (newiatt.ia_type == IA_IFDIR)) {
-                        /* Either both old and new must be dirs,
-                         * or both must be non-dirs. Else, fail.
-                         */
-                        ret = -1;
-                        errno = EEXIST;
-                        goto out;
-                }
-        }
+    object = GF_CALLOC(1, sizeof(struct glfs_object), glfs_mt_glfs_object_t);
+    if (object == NULL) {
+        errno = ENOMEM;
+        goto out;
+    }
 
-        /* TODO: check if new or old is a prefix of the other, and fail EINVAL */
+    /* populate the return object. The ref taken here
+     * is un'refed when the application does glfs_h_close() */
+    object->inode = inode_ref(newinode);
+    gf_uuid_copy(object->gfid, object->inode->gfid);
 
-        ret = syncop_rename (subvol, &oldloc, &newloc, NULL, NULL);
-        DECODE_SYNCOP_ERR (ret);
+out:
+    /* inode_find takes a reference. Unref it. */
+    if (newinode)
+        inode_unref(newinode);
 
-        if (ret == 0)
-                inode_rename (oldloc.parent->table, oldloc.parent, oldloc.name,
-                              newloc.parent, newloc.name, oldloc.inode,
-                              &oldiatt);
+    glfs_subvol_done(fs, subvol);
 
-out:
-        loc_wipe (&oldloc);
-        loc_wipe (&newloc);
+    __GLFS_EXIT_FS;
 
-        if (oldpinode)
-                inode_unref (oldpinode);
+invalid_fs:
+    return object;
+}
 
-        if (newpinode)
-                inode_unref (newpinode);
+static void
+glfs_free_upcall_inode(void *to_free)
+{
+    struct glfs_upcall_inode *arg = to_free;
 
-        glfs_subvol_done (fs, subvol);
+    if (!arg)
+        return;
 
-        __GLFS_EXIT_FS;
+    if (arg->object)
+        glfs_h_close(arg->object);
+    if (arg->p_object)
+        glfs_h_close(arg->p_object);
+    if (arg->oldp_object)
+        glfs_h_close(arg->oldp_object);
 
-invalid_fs:
-        return ret;
+    GF_FREE(arg);
 }
 
-GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_h_rename, 3.4.2);
-
 int
-glfs_h_poll_cache_invalidation (struct glfs *fs,
-                                struct callback_arg *up_arg,
-                                struct gf_upcall *upcall_data)
+glfs_h_poll_cache_invalidation(struct glfs *fs, struct glfs_upcall *up_arg,
+                               struct gf_upcall *upcall_data)
 {
-        int                                 ret           = -1;
-        struct glfs_object                  *p_object     = NULL;
-        struct glfs_object                  *oldp_object  = NULL;
-        struct glfs_object                  *object       = NULL;
-        struct gf_upcall_cache_invalidation *ca_data      = NULL;
-        struct callback_inode_arg           *up_inode_arg = NULL;
-
-        ca_data = upcall_data->data;
-        GF_VALIDATE_OR_GOTO ("glfs_h_poll_cache_invalidation",
-                             ca_data, out);
-
-        object = glfs_h_create_from_handle (fs, upcall_data->gfid,
-                                            GFAPI_HANDLE_LENGTH,
-                                            NULL);
-        GF_VALIDATE_OR_GOTO ("glfs_h_poll_cache_invalidation",
-                             object, out);
-
-        up_inode_arg = calloc (1, sizeof (struct callback_inode_arg));
-        GF_VALIDATE_OR_GOTO ("glfs_h_poll_cache_invalidation",
-                             up_inode_arg, out);
-
-        up_arg->event_arg = up_inode_arg;
-
-        up_inode_arg->object = object;
-        up_inode_arg->flags = ca_data->flags;
-        up_inode_arg->expire_time_attr = ca_data->expire_time_attr;
-
-        /* XXX: Update stat as well incase of UP_*_TIMES.
-         * This will be addressed as part of INODE_UPDATE */
-        if (ca_data->flags & GFAPI_INODE_UPDATE_FLAGS) {
-                glfs_iatt_to_stat (fs, &ca_data->stat, &up_inode_arg->buf);
-        }
-
-        if (ca_data->flags & GFAPI_UP_PARENT_TIMES) {
-                p_object = glfs_h_create_from_handle (fs,
-                                                      ca_data->p_stat.ia_gfid,
-                                                      GFAPI_HANDLE_LENGTH,
-                                                      NULL);
-                GF_VALIDATE_OR_GOTO ("glfs_h_poll_cache_invalidation",
-                                       p_object, out);
+    int ret = -1;
+    struct glfs_object *p_object = NULL;
+    struct glfs_object *oldp_object = NULL;
+    struct glfs_object *object = NULL;
+    struct gf_upcall_cache_invalidation *ca_data = NULL;
+    struct glfs_upcall_inode *up_inode_arg = NULL;
+
+    ca_data = upcall_data->data;
+    GF_VALIDATE_OR_GOTO("glfs_h_poll_cache_invalidation", ca_data, out);
+
+    object = glfs_h_find_handle(fs, upcall_data->gfid, GFAPI_HANDLE_LENGTH);
+    if (!object) {
+        /* The reason handle creation will fail is because we
+         * couldn't find the inode in the gfapi inode table.
+         *
+         * But since application would have taken inode_ref, the
+         * only case when this can happen is when it has closed
+         * the handle and hence will no more be interested in
+         * the upcall for this particular gfid.
+         */
+        gf_smsg(THIS->name, GF_LOG_DEBUG, errno, API_MSG_CREATE_HANDLE_FAILED,
+                "gfid=%s", uuid_utoa(upcall_data->gfid), NULL);
+        errno = ESTALE;
+        goto out;
+    }
+
+    up_inode_arg = GF_CALLOC(1, sizeof(struct glfs_upcall_inode),
+                             glfs_mt_upcall_inode_t);
+    GF_VALIDATE_OR_GOTO("glfs_h_poll_cache_invalidation", up_inode_arg, out);
+
+    up_inode_arg->object = object;
+    up_inode_arg->flags = ca_data->flags;
+    up_inode_arg->expire_time_attr = ca_data->expire_time_attr;
+
+    /* XXX: Update stat as well in case of UP_*_TIMES.
+     * This will be addressed as part of INODE_UPDATE */
+    if (ca_data->flags & GFAPI_INODE_UPDATE_FLAGS) {
+        glfs_iatt_to_stat(fs, &ca_data->stat, &up_inode_arg->buf);
+    }
+
+    if (ca_data->flags & GFAPI_UP_PARENT_TIMES) {
+        p_object = glfs_h_find_handle(fs, ca_data->p_stat.ia_gfid,
+                                      GFAPI_HANDLE_LENGTH);
+        if (!p_object) {
+            gf_smsg(THIS->name, GF_LOG_DEBUG, errno,
+                    API_MSG_CREATE_HANDLE_FAILED, "gfid=%s",
+                    uuid_utoa(ca_data->p_stat.ia_gfid), NULL);
+            errno = ESTALE;
+            goto out;
+        }
+
+        glfs_iatt_to_stat(fs, &ca_data->p_stat, &up_inode_arg->p_buf);
+    }
+    up_inode_arg->p_object = p_object;
+
+    /* In case of RENAME, update old parent as well */
+    if (ca_data->flags & GFAPI_UP_RENAME) {
+        oldp_object = glfs_h_find_handle(fs, ca_data->oldp_stat.ia_gfid,
+                                         GFAPI_HANDLE_LENGTH);
+        if (!oldp_object) {
+            gf_smsg(THIS->name, GF_LOG_DEBUG, errno,
+                    API_MSG_CREATE_HANDLE_FAILED, "gfid=%s",
+                    uuid_utoa(ca_data->oldp_stat.ia_gfid), NULL);
+            errno = ESTALE;
+            /* By the time we receive upcall old parent_dir may
+             * have got removed. We still need to send upcall
+             * for the file/dir and current parent handles. */
+            up_inode_arg->oldp_object = NULL;
+            ret = 0;
+        }
+
+        glfs_iatt_to_stat(fs, &ca_data->oldp_stat, &up_inode_arg->oldp_buf);
+    }
+    up_inode_arg->oldp_object = oldp_object;
+
+    up_arg->reason = GLFS_UPCALL_INODE_INVALIDATE;
+    up_arg->event = up_inode_arg;
+    up_arg->free_event = glfs_free_upcall_inode;
+
+    ret = 0;
 
-                glfs_iatt_to_stat (fs, &ca_data->p_stat, &up_inode_arg->p_buf);
-        }
-        up_inode_arg->p_object = p_object;
-
-        /* In case of RENAME, update old parent as well */
-        if (ca_data->flags & GFAPI_UP_RENAME) {
-                oldp_object = glfs_h_create_from_handle (fs,
-                                                     ca_data->oldp_stat.ia_gfid,
-                                                     GFAPI_HANDLE_LENGTH,
-                                                     NULL);
-                GF_VALIDATE_OR_GOTO ("glfs_h_poll_cache_invalidation",
-                                       oldp_object, out);
-
-                glfs_iatt_to_stat (fs, &ca_data->oldp_stat,
-                                   &up_inode_arg->oldp_buf);
-        }
-        up_inode_arg->oldp_object = oldp_object;
+out:
+    if (ret) {
+        /* Close p_object and oldp_object as well if being referenced.*/
+        if (object)
+            glfs_h_close(object);
+
+        /* Set reason to prevent applications from using ->event */
+        up_arg->reason = GLFS_UPCALL_EVENT_NULL;
+        GF_FREE(up_inode_arg);
+    }
+    return ret;
+}
 
-        ret = 0;
+void
+glfs_release_upcall(void *ptr)
+{
+    struct glfs_upcall *to_free = ptr;
 
-out:
-        return ret;
+    if (to_free->event)
+        to_free->free_event(to_free->event);
 }
 
 /*
- * This API is used to poll for upcall events stored in the
- * upcall list. Current users of this API is NFS-Ganesha.
- * Incase of any event received, it will be mapped appropriately
- * into 'callback_arg' along with the handle object  to be passed
- * to NFS-Ganesha.
- *
- * On success, applications need to check for 'reason' to decide
- * if any upcall event is received.
+ * This API is used to poll for upcall events stored in the upcall list.
+ * Current users of this API is NFS-Ganesha. In case of any event received, it
+ * will be mapped appropriately into 'glfs_upcall' along with the handle object
+ * to be passed to NFS-Ganesha.
  *
- * Current supported upcall_events -
- *      GFAPI_INODE_INVALIDATE -
- *              'arg - callback_inode_arg
+ * On success, applications need to check if up_arg is not-NULL or errno is not
+ * ENOENT. glfs_upcall_get_reason() can be used to decide what kind of event
+ * has been received.
  *
- * After processing the event, applications need to free 'event_arg'.
+ * Current supported upcall_events:
+ *      GLFS_UPCALL_INODE_INVALIDATE
  *
- * Incase of INODE_INVALIDATE, applications need to free "object",
- * "p_object" and "oldp_object" using glfs_h_close(..).
+ * After processing the event, applications need to free 'up_arg' by calling
+ * glfs_free().
  *
- * Also similar to I/Os, the application should ideally stop polling
- * before calling glfs_fini(..). Hence making an assumption that
- * 'fs' & ctx structures cannot be freed while in this routine.
+ * Also similar to I/Os, the application should ideally stop polling before
+ * calling glfs_fini(..). Hence making an assumption that 'fs' & ctx structures
+ * cannot be freed while in this routine.
  */
+GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_h_poll_upcall, 3.7.16)
 int
-pub_glfs_h_poll_upcall (struct glfs *fs, struct callback_arg *up_arg)
+pub_glfs_h_poll_upcall(struct glfs *fs, struct glfs_upcall **up_arg)
 {
-        upcall_entry                        *u_list         = NULL;
-        upcall_entry                        *tmp            = NULL;
-        xlator_t                            *subvol         = NULL;
-        int                                 found           = 0;
-        int                                 reason          = 0;
-        glusterfs_ctx_t                     *ctx            = NULL;
-        int                                 ret             = -1;
-        struct gf_upcall                    *upcall_data    = NULL;
-
-        DECLARE_OLD_THIS;
-
-        if (!up_arg) {
-                errno = EINVAL;
-                goto err;
-        }
+    upcall_entry *u_list = NULL;
+    upcall_entry *tmp = NULL;
+    xlator_t *subvol = NULL;
+    glusterfs_ctx_t *ctx = NULL;
+    int ret = -1;
+    struct gf_upcall *upcall_data = NULL;
+
+    DECLARE_OLD_THIS;
+
+    if (!up_arg) {
+        errno = EINVAL;
+        goto err;
+    }
+
+    __GLFS_ENTRY_VALIDATE_FS(fs, err);
+
+    /* get the active volume */
+    subvol = glfs_active_subvol(fs);
+    if (!subvol) {
+        errno = EIO;
+        goto restore;
+    }
+
+    /* Ideally applications should stop polling before calling
+     * 'glfs_fini'. Yet cross check if cleanup has started. */
+    pthread_mutex_lock(&fs->mutex);
+    {
+        ctx = fs->ctx;
+
+        if (ctx->cleanup_started) {
+            pthread_mutex_unlock(&fs->mutex);
+            goto out;
+        }
+
+        fs->pin_refcnt++;
+
+        /* once we call this function, the applications seems to be
+         * interested in events, enable caching them */
+        fs->cache_upcalls = _gf_true;
+    }
+    pthread_mutex_unlock(&fs->mutex);
+
+    pthread_mutex_lock(&fs->upcall_list_mutex);
+    {
+        list_for_each_entry_safe(u_list, tmp, &fs->upcall_list, upcall_list)
+        {
+            list_del_init(&u_list->upcall_list);
+            upcall_data = &u_list->upcall_data;
+            break;
+        }
+    }
+    /* No other thread can delete this entry. So unlock it */
+    pthread_mutex_unlock(&fs->upcall_list_mutex);
+
+    if (upcall_data) {
+        switch (upcall_data->event_type) {
+            case GF_UPCALL_CACHE_INVALIDATION:
+                *up_arg = GLFS_CALLOC(1, sizeof(struct gf_upcall),
+                                      glfs_release_upcall,
+                                      glfs_mt_upcall_entry_t);
+                if (!*up_arg) {
+                    errno = ENOMEM;
+                    break; /* goto free u_list */
+                }
 
-        __GLFS_ENTRY_VALIDATE_FS (fs, err);
+                /* XXX: Need to revisit this to support
+                 * GLFS_UPCALL_INODE_UPDATE if required. */
+                ret = glfs_h_poll_cache_invalidation(fs, *up_arg, upcall_data);
+                if (ret || (*up_arg)->reason == GLFS_UPCALL_EVENT_NULL) {
+                    /* It could so happen that the file which got
+                     * upcall notification may have got deleted by
+                     * the same client. Irrespective of the error,
+                     * return with an error or success+ENOENT. */
+                    if ((*up_arg)->reason == GLFS_UPCALL_EVENT_NULL)
+                        errno = ENOENT;
+
+                    GLFS_FREE(*up_arg);
+                    *up_arg = NULL;
+                }
+                break;
+            case GF_UPCALL_RECALL_LEASE:
+                gf_log("glfs_h_poll_upcall", GF_LOG_DEBUG,
+                       "UPCALL_RECALL_LEASE is not implemented yet");
+            /* fallthrough till we support leases */
+            case GF_UPCALL_EVENT_NULL:
+                /* no 'default:' label, to force handling all upcall events */
+                errno = ENOENT;
+                break;
+        }
+
+        GF_FREE(u_list->upcall_data.data);
+        GF_FREE(u_list);
+    } else {
+        /* fs->upcall_list was empty, no upcall events cached */
+        errno = ENOENT;
+    }
+
+    ret = 0;
 
-        /* get the active volume */
-        subvol = glfs_active_subvol (fs);
+out:
+    pthread_mutex_lock(&fs->mutex);
+    {
+        fs->pin_refcnt--;
+    }
+    pthread_mutex_unlock(&fs->mutex);
 
-        if (!subvol) {
-                errno = EIO;
-                goto restore;
-        }
+    glfs_subvol_done(fs, subvol);
 
-        /* Ideally applications should stop polling before calling
-         * 'glfs_fini'. Yet cross check if cleanup has started
-         */
-        pthread_mutex_lock (&fs->mutex);
-        {
-                ctx = fs->ctx;
+restore:
+    __GLFS_EXIT_FS;
+err:
+    return ret;
+}
 
-                if (ctx->cleanup_started) {
-                        pthread_mutex_unlock (&fs->mutex);
-                        goto out;
-                }
+static gf_boolean_t log_upcall370 = _gf_true; /* log once */
 
-                fs->pin_refcnt++;
-        }
-        pthread_mutex_unlock (&fs->mutex);
+/* The old glfs_h_poll_upcall interface requires intimate knowledge of the
+ * structures that are returned to the calling application. This is not
+ * recommended, as the returned structures need to returned correctly (handles
+ * closed, memory free'd with the unavailable GF_FREE(), and possibly more.)
+ *
+ * To the best of our knowledge, only NFS-Ganesha uses the upcall events
+ * through gfapi. We keep this backwards compatibility function around so that
+ * applications using the existing implementation do not break.
+ *
+ * WARNING: this function will be removed in the future.
+ */
+GFAPI_SYMVER_PUBLIC(glfs_h_poll_upcall370, glfs_h_poll_upcall, 3.7.0)
+int
+pub_glfs_h_poll_upcall370(struct glfs *fs, struct glfs_callback_arg *up_arg)
+{
+    struct glfs_upcall *upcall = NULL;
+    int ret = -1;
+
+    if (log_upcall370) {
+        log_upcall370 = _gf_false;
+        gf_log(THIS->name, GF_LOG_WARNING,
+               "this application is "
+               "compiled against an old version of libgfapi, it "
+               "should use glfs_free() to release the structure "
+               "returned by glfs_h_poll_upcall() - for more details, "
+               "see http://review.gluster.org/14701");
+    }
+
+    ret = pub_glfs_h_poll_upcall(fs, &upcall);
+    if (ret == 0) {
+        up_arg->fs = fs;
+        if ((errno == ENOENT) || !upcall || !upcall->event) {
+            up_arg->reason = GLFS_UPCALL_EVENT_NULL;
+            goto out;
+        }
+
+        up_arg->reason = upcall->reason;
+
+        if (upcall->reason == GLFS_UPCALL_INODE_INVALIDATE) {
+            struct glfs_callback_inode_arg *cb_inode = NULL;
+            struct glfs_upcall_inode *up_inode = NULL;
+
+            cb_inode = GF_CALLOC(1, sizeof(struct glfs_callback_inode_arg),
+                                 glfs_mt_upcall_inode_t);
+            if (!cb_inode) {
+                errno = ENOMEM;
+                ret = -1;
+                goto out;
+            }
 
-        pthread_mutex_lock (&fs->upcall_list_mutex);
-        {
-                list_for_each_entry_safe (u_list, tmp,
-                                          &fs->upcall_list,
-                                          upcall_list) {
-                        found = 1;
-                        break;
-                }
-        }
-        /* No other thread can delete this entry. So unlock it */
-        pthread_mutex_unlock (&fs->upcall_list_mutex);
-
-        if (found) {
-                upcall_data = &u_list->upcall_data;
-
-                switch (upcall_data->event_type) {
-                case GF_UPCALL_CACHE_INVALIDATION:
-                        /* XXX: Need to revisit this to support
-                         * GFAPI_INODE_UPDATE if required.
-                         */
-                        reason = GFAPI_INODE_INVALIDATE;
-                        ret = glfs_h_poll_cache_invalidation (fs,
-                                                              up_arg,
-                                                              upcall_data);
-                        if (!ret) {
-                                break;
-                        }
-                        /* It could so happen that the file which got
-                         * upcall notification may have got deleted
-                         * by other thread. Irrespective of the error,
-                         * log it and return with CBK_NULL reason.
-                         *
-                         * Applications will ignore this notification
-                         * as up_arg->object will be NULL */
-                        gf_msg (subvol->name, GF_LOG_WARNING, errno,
-                                API_MSG_CREATE_HANDLE_FAILED,
-                                "handle creation of %s failed",
-                                uuid_utoa (upcall_data->gfid));
-
-                        reason = GFAPI_CBK_EVENT_NULL;
-                        break;
-                default:
-                        break;
-                }
+            up_inode = upcall->event;
 
-                up_arg->reason = reason;
+            /* copy attributes one by one, the memory layout might
+             * be different between the old glfs_callback_inode_arg
+             * and new glfs_upcall_inode */
+            cb_inode->object = up_inode->object;
+            cb_inode->flags = up_inode->flags;
+            memcpy(&cb_inode->buf, &up_inode->buf, sizeof(struct stat));
+            cb_inode->expire_time_attr = up_inode->expire_time_attr;
+            cb_inode->p_object = up_inode->p_object;
+            memcpy(&cb_inode->p_buf, &up_inode->p_buf, sizeof(struct stat));
+            cb_inode->oldp_object = up_inode->oldp_object;
+            memcpy(&cb_inode->oldp_buf, &up_inode->oldp_buf,
+                   sizeof(struct stat));
 
-                list_del_init (&u_list->upcall_list);
-                GF_FREE (u_list->upcall_data.data);
-                GF_FREE (u_list);
+            up_arg->event_arg = cb_inode;
         }
-
-        ret = 0;
+    }
 
 out:
-        pthread_mutex_lock (&fs->mutex);
-        {
-                fs->pin_refcnt--;
-        }
-        pthread_mutex_unlock (&fs->mutex);
-
-        glfs_subvol_done (fs, subvol);
+    if (upcall) {
+        /* we can not use glfs_free() here, objects need to stay */
+        GF_FREE(upcall->event);
+        GF_FREE(upcall);
+    }
 
-restore:
-        __GLFS_EXIT_FS;
-err:
-        return ret;
+    return ret;
 }
 
-GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_h_poll_upcall, 3.7.0);
-
 #ifdef HAVE_ACL_LIBACL_H
-#include "glusterfs-acl.h"
+#include <glusterfs/glusterfs-acl.h>
 #include <acl/libacl.h>
 
+GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_h_acl_set, 3.7.0)
 int
-pub_glfs_h_acl_set (struct glfs *fs, struct glfs_object *object,
-                    const acl_type_t type, const acl_t acl)
+pub_glfs_h_acl_set(struct glfs *fs, struct glfs_object *object,
+                   const acl_type_t type, const acl_t acl)
 {
-        int ret = -1;
-        char *acl_s = NULL;
-        const char *acl_key = NULL;
-        struct glfs_object *new_object = NULL;
+    int ret = -1;
+    char *acl_s = NULL;
+    const char *acl_key = NULL;
+    struct glfs_object *new_object = NULL;
 
-        DECLARE_OLD_THIS;
+    DECLARE_OLD_THIS;
 
-        if (!object || !acl) {
-                errno = EINVAL;
-                return ret;
-        }
+    if (!object || !acl) {
+        errno = EINVAL;
+        return ret;
+    }
 
-        __GLFS_ENTRY_VALIDATE_FS (fs, invalid_fs);
+    __GLFS_ENTRY_VALIDATE_FS(fs, invalid_fs);
 
-        acl_key = gf_posix_acl_get_key (type);
-        if (!acl_key)
-                goto out;
+    acl_key = gf_posix_acl_get_key(type);
+    if (!acl_key)
+        goto out;
 
-        acl_s = acl_to_any_text (acl, NULL, ',',
-                                 TEXT_ABBREVIATE | TEXT_NUMERIC_IDS);
-        if (!acl_s)
-                goto out;
+    acl_s = acl_to_any_text(acl, NULL, ',', TEXT_ABBREVIATE | TEXT_NUMERIC_IDS);
+    if (!acl_s)
+        goto out;
 
-        if (IA_ISLNK (object->inode->ia_type)) {
-                new_object = glfs_h_resolve_symlink (fs, object);
-                if (new_object == NULL)
-                        goto out;
-        } else
-                new_object = object;
+    if (IA_ISLNK(object->inode->ia_type)) {
+        new_object = glfs_h_resolve_symlink(fs, object);
+        if (new_object == NULL)
+            goto out;
+    } else
+        new_object = object;
 
-        ret = pub_glfs_h_setxattrs (fs, new_object, acl_key, acl_s,
-                                    strlen (acl_s) + 1, 0);
+    ret = pub_glfs_h_setxattrs(fs, new_object, acl_key, acl_s,
+                               strlen(acl_s) + 1, 0);
 
-        acl_free (acl_s);
+    acl_free(acl_s);
 
 out:
-        if (IA_ISLNK (object->inode->ia_type) && new_object)
-                glfs_h_close (new_object);
+    if (IA_ISLNK(object->inode->ia_type) && new_object)
+        glfs_h_close(new_object);
 
-        __GLFS_EXIT_FS;
+    __GLFS_EXIT_FS;
 
 invalid_fs:
-        return ret;
+    return ret;
 }
 
+GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_h_acl_get, 3.7.0)
 acl_t
-pub_glfs_h_acl_get (struct glfs *fs, struct glfs_object *object,
-                    const acl_type_t type)
+pub_glfs_h_acl_get(struct glfs *fs, struct glfs_object *object,
+                   const acl_type_t type)
 {
-        int                 ret = 0;
-        acl_t acl = NULL;
-        char *acl_s = NULL;
-        dict_t *xattr = NULL;
-        const char *acl_key = NULL;
-        struct glfs_object *new_object = NULL;
-
-        DECLARE_OLD_THIS;
-
-        if (!object) {
-                errno = EINVAL;
-                return NULL;
-        }
+    int ret = 0;
+    acl_t acl = NULL;
+    char *acl_s = NULL;
+    dict_t *xattr = NULL;
+    const char *acl_key = NULL;
+    struct glfs_object *new_object = NULL;
 
-        __GLFS_ENTRY_VALIDATE_FS (fs, invalid_fs);
+    DECLARE_OLD_THIS;
 
-        acl_key = gf_posix_acl_get_key (type);
-        if (!acl_key)
-                goto out;
+    if (!object) {
+        errno = EINVAL;
+        return NULL;
+    }
 
-        if (IA_ISLNK (object->inode->ia_type)) {
-                new_object = glfs_h_resolve_symlink (fs, object);
-                if (new_object == NULL)
-                        goto out;
-        } else
-                new_object = object;
+    __GLFS_ENTRY_VALIDATE_FS(fs, invalid_fs);
 
-        ret = glfs_h_getxattrs_common (fs, new_object, &xattr, acl_key);
-        if (ret)
-                goto out;
+    acl_key = gf_posix_acl_get_key(type);
+    if (!acl_key)
+        goto out;
 
-        ret = dict_get_str (xattr, (char *)acl_key, &acl_s);
-        if (ret == -1)
-                goto out;
+    if (IA_ISLNK(object->inode->ia_type)) {
+        new_object = glfs_h_resolve_symlink(fs, object);
+        if (new_object == NULL)
+            goto out;
+    } else
+        new_object = object;
 
-        acl = acl_from_text (acl_s);
+    ret = glfs_h_getxattrs_common(fs, new_object, &xattr, acl_key, _gf_false);
+    if (ret)
+        goto out;
+
+    ret = dict_get_str(xattr, (char *)acl_key, &acl_s);
+    if (ret)
+        goto out;
+
+    acl = acl_from_text(acl_s);
 
 out:
-        GF_FREE (acl_s);
-        if (IA_ISLNK (object->inode->ia_type) && new_object)
-                glfs_h_close (new_object);
+    if (xattr)
+        dict_unref(xattr);
 
-        __GLFS_EXIT_FS;
+    if (IA_ISLNK(object->inode->ia_type) && new_object)
+        glfs_h_close(new_object);
+
+    __GLFS_EXIT_FS;
 
 invalid_fs:
-        return acl;
+    return acl;
 }
 #else /* !HAVE_ACL_LIBACL_H */
+GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_h_acl_get, 3.7.0)
 acl_t
-pub_glfs_h_acl_get (struct glfs *fs, struct glfs_object *object,
-                    const acl_type_t type)
+pub_glfs_h_acl_get(struct glfs *fs, struct glfs_object *object,
+                   const acl_type_t type)
 {
-        errno = ENOTSUP;
-        return NULL;
+    errno = ENOTSUP;
+    return NULL;
 }
 
+GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_h_acl_set, 3.7.0)
 int
-pub_glfs_h_acl_set (struct glfs *fs, struct glfs_object *object,
-                    const acl_type_t type, const acl_t acl)
+pub_glfs_h_acl_set(struct glfs *fs, struct glfs_object *object,
+                   const acl_type_t type, const acl_t acl)
 {
-        errno = ENOTSUP;
-        return -1;
+    errno = ENOTSUP;
+    return -1;
 }
 #endif
-GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_h_acl_set, 3.7.0);
-GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_h_acl_get, 3.7.0);
 
 /* The API to perform read using anonymous fd */
+GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_h_anonymous_read, 3.7.0)
 ssize_t
-pub_glfs_h_anonymous_read (struct glfs *fs, struct glfs_object *object,
-                           const void *buf, size_t count, off_t offset)
+pub_glfs_h_anonymous_read(struct glfs *fs, struct glfs_object *object,
+                          const void *buf, size_t count, off_t offset)
 {
-        struct iovec    iov     = {0, };
-        ssize_t         ret     = 0;
-
-        /* validate in args */
-        if ((fs == NULL) || (object == NULL)) {
-                errno = EINVAL;
-                return -1;
-        }
+    struct iovec iov = {
+        0,
+    };
+    ssize_t ret = 0;
+
+    /* validate in args */
+    if ((fs == NULL) || (object == NULL)) {
+        errno = EINVAL;
+        return -1;
+    }
 
-        iov.iov_base = (void *) buf;
-        iov.iov_len = count;
+    iov.iov_base = (void *)buf;
+    iov.iov_len = count;
 
-        ret = glfs_anonymous_preadv (fs, object, &iov, 1, offset, 0);
+    ret = glfs_anonymous_preadv(fs, object, &iov, 1, offset, 0);
 
-        return ret;
+    return ret;
 }
 
-GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_h_anonymous_read, 3.7.0);
-
 /* The API to perform write using anonymous fd */
+GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_h_anonymous_write, 3.7.0)
 ssize_t
-pub_glfs_h_anonymous_write (struct glfs *fs, struct glfs_object *object,
-                            const void *buf, size_t count, off_t offset)
+pub_glfs_h_anonymous_write(struct glfs *fs, struct glfs_object *object,
+                           const void *buf, size_t count, off_t offset)
+{
+    struct iovec iov = {
+        0,
+    };
+    ssize_t ret = 0;
+
+    /* validate in args */
+    if ((fs == NULL) || (object == NULL)) {
+        errno = EINVAL;
+        return -1;
+    }
+
+    iov.iov_base = (void *)buf;
+    iov.iov_len = count;
+
+    ret = glfs_anonymous_pwritev(fs, object, &iov, 1, offset, 0);
+
+    return ret;
+}
+
+GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_object_copy, 3.11.0)
+struct glfs_object *
+pub_glfs_object_copy(struct glfs_object *src)
 {
-        struct iovec iov        = {0, };
-        ssize_t      ret        = 0;
+    struct glfs_object *object = NULL;
 
-        /* validate in args */
-        if ((fs == NULL) || (object == NULL)) {
-                errno = EINVAL;
-                return -1;
-        }
+    GF_VALIDATE_OR_GOTO("glfs_dup_object", src, out);
 
-        iov.iov_base = (void *) buf;
-        iov.iov_len = count;
+    object = GF_CALLOC(1, sizeof(struct glfs_object), glfs_mt_glfs_object_t);
+    if (object == NULL) {
+        errno = ENOMEM;
+        gf_smsg(THIS->name, GF_LOG_WARNING, errno, API_MSG_CREATE_HANDLE_FAILED,
+                "glfs_dup_object gfid=%s", uuid_utoa(src->inode->gfid), NULL);
+        return NULL;
+    }
 
-        ret = glfs_anonymous_pwritev (fs, object, &iov, 1, offset, 0);
+    object->inode = inode_ref(src->inode);
+    gf_uuid_copy(object->gfid, src->inode->gfid);
 
-        return ret;
+out:
+    return object;
+}
+
+GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_xreaddirplus_get_object, 3.11.0)
+struct glfs_object *
+pub_glfs_xreaddirplus_get_object(struct glfs_xreaddirp_stat *xstat)
+{
+    GF_VALIDATE_OR_GOTO("glfs_xreaddirplus_get_object", xstat, out);
+
+    if (!(xstat->flags_handled & GFAPI_XREADDIRP_HANDLE))
+        gf_smsg(THIS->name, GF_LOG_ERROR, errno, API_MSG_HANDLE_NOT_SET,
+                "GFAPI_XREADDIRP_HANDLE xstat=%p", xstat, "handle=%x",
+                xstat->flags_handled, NULL);
+
+    return xstat->object;
+
+out:
+    return NULL;
 }
 
-GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_h_anonymous_write, 3.7.0);
+GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_h_lease, 4.0.0)
+int
+pub_glfs_h_lease(struct glfs *fs, struct glfs_object *object,
+                 struct glfs_lease *lease)
+{
+    int ret = -1;
+    xlator_t *subvol = NULL;
+    inode_t *inode = NULL;
+    loc_t loc = {
+        0,
+    };
+    struct gf_lease gf_lease = {
+        0,
+    };
+
+    /* validate in args */
+    if ((fs == NULL) || (object == NULL)) {
+        errno = EINVAL;
+        return -1;
+    }
+
+    DECLARE_OLD_THIS;
+    __GLFS_ENTRY_VALIDATE_FS(fs, invalid_fs);
+
+    /* get the active volume */
+    subvol = glfs_active_subvol(fs);
+    if (!subvol) {
+        ret = -1;
+        errno = EIO;
+        goto out;
+    }
+
+    /* get/refresh the in arg objects inode in correlation to the xlator */
+    inode = glfs_resolve_inode(fs, subvol, object);
+    if (!inode) {
+        errno = ESTALE;
+        goto out;
+    }
+
+    /* populate loc */
+    GLFS_LOC_FILL_INODE(inode, loc, out);
+
+    glfs_lease_to_gf_lease(lease, &gf_lease);
+
+    ret = syncop_lease(subvol, &loc, &gf_lease, NULL, NULL);
+    DECODE_SYNCOP_ERR(ret);
+
+    gf_lease_to_glfs_lease(&gf_lease, lease);
+
+out:
+    loc_wipe(&loc);
+
+    if (inode)
+        inode_unref(inode);
+
+    glfs_subvol_done(fs, subvol);
+
+    __GLFS_EXIT_FS;
+
+invalid_fs:
+    return ret;
+}
diff --git a/api/src/glfs-handles.h b/api/src/glfs-handles.h
index 6446cc30c7d..4d039b9c76b 100644
--- a/api/src/glfs-handles.h
+++ b/api/src/glfs-handles.h
@@ -1,5 +1,5 @@
 /*
-  Copyright (c) 2013 Red Hat, Inc. <http://www.redhat.com>
+  Copyright (c) 2013-2018 Red Hat, Inc. <http://www.redhat.com>
   This file is part of GlusterFS.
 
   This file is licensed to you under your choice of the GNU Lesser
@@ -46,46 +46,39 @@
  *
  */
 
-/* Values for valid falgs to be used when using XXXsetattr, to set multiple
- attribute values passed via the related stat structure.
- */
-#define GFAPI_SET_ATTR_MODE  0x1
-#define GFAPI_SET_ATTR_UID   0x2
-#define GFAPI_SET_ATTR_GID   0x4
-#define GFAPI_SET_ATTR_SIZE  0x8
-#define GFAPI_SET_ATTR_ATIME 0x10
-#define GFAPI_SET_ATTR_MTIME 0x20
-
 /* Handle length for object handles returned from glfs_h_extract_handle or
  * glfs_h_create_from_handle */
 #define GFAPI_HANDLE_LENGTH 16
 
 /* These flags should be in sync to the ones defined in upcall.h */
-#define GFAPI_UP_NLINK   0x00000001   /* update nlink */
-#define GFAPI_UP_MODE    0x00000002   /* update mode and ctime */
-#define GFAPI_UP_OWN     0x00000004   /* update mode,uid,gid and ctime */
-#define GFAPI_UP_SIZE    0x00000008   /* update fsize */
-#define GFAPI_UP_TIMES   0x00000010   /* update all times */
-#define GFAPI_UP_ATIME   0x00000020   /* update atime only */
-#define GFAPI_UP_PERM    0x00000040   /* update fields needed for
-                                         permission checking */
-#define GFAPI_UP_RENAME  0x00000080   /* this is a rename op -
-                                         delete the cache entry */
-#define GFAPI_UP_FORGET  0x00000100   /* inode_forget on server side -
-                                         invalidate the cache entry */
-#define GFAPI_UP_PARENT_TIMES   0x00000200   /* update parent dir times */
-
-#define GFAPI_INODE_UPDATE_FLAGS (GFAPI_UP_NLINK | GFAPI_UP_MODE | \
-                                  GFAPI_UP_OWN | GFAPI_UP_SIZE | \
-                                  GFAPI_UP_TIMES | GFAPI_UP_ATIME)
+#define GFAPI_UP_NLINK 0x00000001 /* update nlink */
+#define GFAPI_UP_MODE 0x00000002  /* update mode and ctime */
+#define GFAPI_UP_OWN 0x00000004   /* update mode,uid,gid and ctime */
+#define GFAPI_UP_SIZE 0x00000008  /* update fsize */
+#define GFAPI_UP_TIMES 0x00000010 /* update all times */
+#define GFAPI_UP_ATIME 0x00000020 /* update atime only */
+#define GFAPI_UP_PERM                                                          \
+    0x00000040 /* update fields needed for                                     \
+                  permission checking */
+#define GFAPI_UP_RENAME                                                        \
+    0x00000080 /* this is a rename op -                                        \
+                  delete the cache entry */
+#define GFAPI_UP_FORGET                                                        \
+    0x00000100                           /* inode_forget on server side -      \
+                                            invalidate the cache entry */
+#define GFAPI_UP_PARENT_TIMES 0x00000200 /* update parent dir times */
+
+#define GFAPI_INODE_UPDATE_FLAGS                                               \
+    (GFAPI_UP_NLINK | GFAPI_UP_MODE | GFAPI_UP_OWN | GFAPI_UP_SIZE |           \
+     GFAPI_UP_TIMES | GFAPI_UP_ATIME)
 
 /* Portability non glibc c++ build systems */
 #ifndef __THROW
-# if defined __cplusplus
-#  define __THROW       throw ()
-# else
-#  define __THROW
-# endif
+#if defined __cplusplus
+#define __THROW throw()
+#else
+#define __THROW
+#endif
 #endif
 
 __BEGIN_DECLS
@@ -101,168 +94,167 @@ __BEGIN_DECLS
 struct glfs_object;
 typedef struct glfs_object glfs_object_t;
 
-/*
- * Applications (currently NFS-Ganesha) can make use of this
- * structure to read upcall notifications sent by server.
- *
- * On success, applications need to check for 'reason' to decide
- * if any upcall event is received.
- *
- * Currently supported upcall_events -
- *      GFAPI_INODE_INVALIDATE -
- *              'event_arg' - callback_inode_arg
+/* Functions for getting details about the glfs_upcall_inode
  *
- * After processing the event, applications need to free 'event_arg'.
+ * None of the pointers returned by the below functions should be free()'d,
+ * glfs_free()'d or glfs_h_close()'d by the application.
  *
- * Also similar to I/Os, the application should ideally stop polling
- * before calling glfs_fini(..). Hence making an assumption that
- * 'fs' & ctx structures cannot be freed while in this routine.
+ * Releasing of the structures is done by passing the glfs_upcall pointer
+ * to glfs_free().
  */
-struct callback_arg {
-        struct glfs             *fs; /* glfs object */
-        int                     reason;  /* Upcall event type */
-        void                    *event_arg; /* changes based in the event type */
-};
+struct glfs_upcall_inode;
+typedef struct glfs_upcall_inode glfs_upcall_inode_t;
 
-/*
- * After processing upcall event, they need to free "object" , "p_object",
- * "oldp_object" using glfs_h_close(..).
- */
-struct callback_inode_arg {
-        struct glfs_object      *object; /* Object which need to be acted upon */
-        int                     flags; /* Cache UPDATE/INVALIDATE flags */
-        struct stat             buf; /* Latest stat of this entry */
-        unsigned int            expire_time_attr; /* the amount of time for which
-                                                   * the application need to cache
-                                                   * this entry
-                                                   */
-        struct glfs_object      *p_object; /* parent Object to be updated */
-        struct stat             p_buf; /* Latest stat of parent dir handle */
-        struct glfs_object      *oldp_object; /* Old parent Object
-                                               * to be updated */
-        struct stat             oldp_buf; /* Latest stat of old parent
-                                           * dir handle */
-};
-
-/* reason list in callback_arg */
-enum gfapi_callback_type {
-        GFAPI_CBK_EVENT_NULL,
-        GFAPI_INODE_INVALIDATE, /* invalidate cache entry */
-};
+glfs_object_t *
+glfs_upcall_inode_get_object(glfs_upcall_inode_t *arg) __THROW
+    GFAPI_PUBLIC(glfs_upcall_inode_get_object, 3.7.16);
+
+uint64_t
+glfs_upcall_inode_get_flags(glfs_upcall_inode_t *arg) __THROW
+    GFAPI_PUBLIC(glfs_upcall_inode_get_flags, 3.7.16);
+
+struct stat *
+glfs_upcall_inode_get_stat(glfs_upcall_inode_t *arg) __THROW
+    GFAPI_PUBLIC(glfs_upcall_inode_get_stat, 3.7.16);
+
+uint64_t
+glfs_upcall_inode_get_expire(glfs_upcall_inode_t *arg) __THROW
+    GFAPI_PUBLIC(glfs_upcall_inode_get_expire, 3.7.16);
+
+glfs_object_t *
+glfs_upcall_inode_get_pobject(glfs_upcall_inode_t *arg) __THROW
+    GFAPI_PUBLIC(glfs_upcall_inode_get_pobject, 3.7.16);
+
+struct stat *
+glfs_upcall_inode_get_pstat(glfs_upcall_inode_t *arg) __THROW
+    GFAPI_PUBLIC(glfs_upcall_inode_get_pstat, 3.7.16);
+
+glfs_object_t *
+glfs_upcall_inode_get_oldpobject(glfs_upcall_inode_t *arg) __THROW
+    GFAPI_PUBLIC(glfs_upcall_inode_get_oldpobject, 3.7.16);
+
+struct stat *
+glfs_upcall_inode_get_oldpstat(glfs_upcall_inode_t *arg) __THROW
+    GFAPI_PUBLIC(glfs_upcall_inode_get_oldpstat, 3.7.16);
 
 /* Handle based operations */
 /* Operations that generate handles */
-struct glfs_object *glfs_h_lookupat (struct glfs *fs,
-                                     struct glfs_object *parent,
-                                     const char *path,
-                                     struct stat *stat, int follow) __THROW
-        GFAPI_PUBLIC(glfs_h_lookupat, 3.7.4);
-
-struct glfs_object *glfs_h_creat (struct glfs *fs, struct glfs_object *parent,
-				  const char *path, int flags, mode_t mode,
-				  struct stat *sb) __THROW
-        GFAPI_PUBLIC(glfs_h_create, 3.4.2);
-
-struct glfs_object *glfs_h_mkdir (struct glfs *fs, struct glfs_object *parent,
-				  const char *path, mode_t flags,
-				  struct stat *sb) __THROW
-        GFAPI_PUBLIC(glfs_h_mkdir, 3.4.2);
-
-struct glfs_object *glfs_h_mknod (struct glfs *fs, struct glfs_object *parent,
-				  const char *path, mode_t mode, dev_t dev,
-				  struct stat *sb) __THROW
-        GFAPI_PUBLIC(glfs_h_mknod, 3.4.2);
-
-struct glfs_object *glfs_h_symlink (struct glfs *fs, struct glfs_object *parent,
-				    const char *name, const char *data,
-				    struct stat *stat) __THROW
-        GFAPI_PUBLIC(glfs_h_symlink, 3.4.2);
+glfs_object_t *
+glfs_h_lookupat(glfs_t *fs, glfs_object_t *parent, const char *path,
+                struct stat *stat, int follow) __THROW
+    GFAPI_PUBLIC(glfs_h_lookupat, 3.7.4);
+
+glfs_object_t *
+glfs_h_creat(glfs_t *fs, glfs_object_t *parent, const char *path, int flags,
+             mode_t mode, struct stat *sb) __THROW
+    GFAPI_PUBLIC(glfs_h_creat, 3.4.2);
+
+glfs_object_t *
+glfs_h_mkdir(glfs_t *fs, glfs_object_t *parent, const char *path, mode_t flags,
+             struct stat *sb) __THROW GFAPI_PUBLIC(glfs_h_mkdir, 3.4.2);
+
+glfs_object_t *
+glfs_h_mknod(glfs_t *fs, glfs_object_t *parent, const char *path, mode_t mode,
+             dev_t dev, struct stat *sb) __THROW
+    GFAPI_PUBLIC(glfs_h_mknod, 3.4.2);
+
+glfs_object_t *
+glfs_h_symlink(glfs_t *fs, glfs_object_t *parent, const char *name,
+               const char *data, struct stat *stat) __THROW
+    GFAPI_PUBLIC(glfs_h_symlink, 3.4.2);
 
 /* Operations on the actual objects */
-int glfs_h_unlink (struct glfs *fs, struct glfs_object *parent,
-		   const char *path) __THROW
-        GFAPI_PUBLIC(glfs_h_unlink, 3.4.2);
+int
+glfs_h_unlink(glfs_t *fs, glfs_object_t *parent, const char *path) __THROW
+    GFAPI_PUBLIC(glfs_h_unlink, 3.4.2);
 
-int glfs_h_close (struct glfs_object *object) __THROW
-        GFAPI_PUBLIC(glfs_h_close, 3.4.2);
+int
+glfs_h_close(glfs_object_t *object) __THROW GFAPI_PUBLIC(glfs_h_close, 3.4.2);
 
-int glfs_caller_specific_init (void *uid_caller_key, void *gid_caller_key,
-			       void *future) __THROW
-        GFAPI_PUBLIC(glfs_caller_specific_init, 3.5.0);
+int
+glfs_caller_specific_init(void *uid_caller_key, void *gid_caller_key,
+                          void *future) __THROW
+    GFAPI_PUBLIC(glfs_caller_specific_init, 3.5.0);
 
-int glfs_h_truncate (struct glfs *fs, struct glfs_object *object,
-                     off_t offset) __THROW
-        GFAPI_PUBLIC(glfs_h_truncate, 3.4.2);
+int
+glfs_h_truncate(glfs_t *fs, glfs_object_t *object, off_t offset) __THROW
+    GFAPI_PUBLIC(glfs_h_truncate, 3.4.2);
 
-int glfs_h_stat(struct glfs *fs, struct glfs_object *object,
-                struct stat *stat) __THROW
-        GFAPI_PUBLIC(glfs_h_stat, 3.4.2);
+int
+glfs_h_stat(glfs_t *fs, glfs_object_t *object, struct stat *stat) __THROW
+    GFAPI_PUBLIC(glfs_h_stat, 3.4.2);
 
-int glfs_h_statfs(struct glfs *fs, struct glfs_object *object,
-                struct statvfs *stat) __THROW
-        GFAPI_PUBLIC(glfs_h_statfs, 3.7.0);
+int
+glfs_h_statfs(glfs_t *fs, glfs_object_t *object, struct statvfs *stat) __THROW
+    GFAPI_PUBLIC(glfs_h_statfs, 3.7.0);
 
-int glfs_h_getattrs (struct glfs *fs, struct glfs_object *object,
-		     struct stat *stat) __THROW
-        GFAPI_PUBLIC(glfs_h_getattrs, 3.4.2);
+int
+glfs_h_getattrs(glfs_t *fs, glfs_object_t *object, struct stat *stat) __THROW
+    GFAPI_PUBLIC(glfs_h_getattrs, 3.4.2);
 
-int glfs_h_getxattrs (struct glfs *fs, struct glfs_object *object,
-		      const char *name, void *value,
-		      size_t size) __THROW
-        GFAPI_PUBLIC(glfs_h_getxattrs, 3.5.1);
+int
+glfs_h_getxattrs(glfs_t *fs, glfs_object_t *object, const char *name,
+                 void *value, size_t size) __THROW
+    GFAPI_PUBLIC(glfs_h_getxattrs, 3.5.1);
 
-int glfs_h_setattrs (struct glfs *fs, struct glfs_object *object,
-		     struct stat *sb, int valid) __THROW
-        GFAPI_PUBLIC(glfs_h_setattrs, 3.4.2);
+int
+glfs_h_setattrs(glfs_t *fs, glfs_object_t *object, struct stat *sb,
+                int valid) __THROW GFAPI_PUBLIC(glfs_h_setattrs, 3.4.2);
 
-int glfs_h_setxattrs (struct glfs *fs, struct glfs_object *object,
-		      const char *name, const void *value,
-		      size_t size, int flags) __THROW
-        GFAPI_PUBLIC(glfs_h_setxattrs, 3.5.0);
+int
+glfs_h_setxattrs(glfs_t *fs, glfs_object_t *object, const char *name,
+                 const void *value, size_t size, int flags) __THROW
+    GFAPI_PUBLIC(glfs_h_setxattrs, 3.5.0);
 
-int glfs_h_readlink (struct glfs *fs, struct glfs_object *object, char *buf,
-		     size_t bufsiz) __THROW
-        GFAPI_PUBLIC(glfs_h_readlink, 3.4.2);
+int
+glfs_h_readlink(glfs_t *fs, glfs_object_t *object, char *buf,
+                size_t bufsiz) __THROW GFAPI_PUBLIC(glfs_h_readlink, 3.4.2);
 
-int glfs_h_link (struct glfs *fs, struct glfs_object *linktgt,
-		 struct glfs_object *parent, const char *name) __THROW
-        GFAPI_PUBLIC(glfs_h_link, 3.4.2);
+int
+glfs_h_link(glfs_t *fs, glfs_object_t *linktgt, glfs_object_t *parent,
+            const char *name) __THROW GFAPI_PUBLIC(glfs_h_link, 3.4.2);
 
-int glfs_h_rename (struct glfs *fs, struct glfs_object *olddir,
-		   const char *oldname, struct glfs_object *newdir,
-		   const char *newname) __THROW
-        GFAPI_PUBLIC(glfs_h_rename, 3.4.2);
+int
+glfs_h_rename(glfs_t *fs, glfs_object_t *olddir, const char *oldname,
+              glfs_object_t *newdir, const char *newname) __THROW
+    GFAPI_PUBLIC(glfs_h_rename, 3.4.2);
 
-int glfs_h_removexattrs (struct glfs *fs, struct glfs_object *object,
-			 const char *name) __THROW
-        GFAPI_PUBLIC(glfs_h_removexattrs, 3.5.1);
+int
+glfs_h_removexattrs(glfs_t *fs, glfs_object_t *object, const char *name) __THROW
+    GFAPI_PUBLIC(glfs_h_removexattrs, 3.5.1);
 
 /* Operations enabling opaque invariant handle to object transitions */
-ssize_t glfs_h_extract_handle (struct glfs_object *object,
-			       unsigned char *handle, int len) __THROW
-        GFAPI_PUBLIC(glfs_h_extract_handle, 3.4.2);
+ssize_t
+glfs_h_extract_handle(glfs_object_t *object, unsigned char *handle,
+                      int len) __THROW
+    GFAPI_PUBLIC(glfs_h_extract_handle, 3.4.2);
 
 /* Given a handle, looks up the inode and creates glfs_object.
  * In addition, if provided 'stat', copies the inode attributes
  */
-struct glfs_object *glfs_h_create_from_handle (struct glfs *fs,
-					       unsigned char *handle, int len,
-					       struct stat *stat) __THROW
-        GFAPI_PUBLIC(glfs_h_create_from_handle, 3.4.2);
+glfs_object_t *
+glfs_h_create_from_handle(glfs_t *fs, unsigned char *handle, int len,
+                          struct stat *stat) __THROW
+    GFAPI_PUBLIC(glfs_h_create_from_handle, 3.4.2);
 
 /* Operations enabling object handles to fd transitions */
-struct glfs_fd *glfs_h_opendir (struct glfs *fs,
-                                struct glfs_object *object) __THROW
-        GFAPI_PUBLIC(glfs_h_opendir, 3.4.2);
+glfs_fd_t *
+glfs_h_opendir(glfs_t *fs, glfs_object_t *object) __THROW
+    GFAPI_PUBLIC(glfs_h_opendir, 3.4.2);
 
-struct glfs_fd *glfs_h_open (struct glfs *fs, struct glfs_object *object,
-			     int flags) __THROW
-        GFAPI_PUBLIC(glfs_h_open, 3.4.2);
+glfs_fd_t *
+glfs_h_open(glfs_t *fs, glfs_object_t *object, int flags) __THROW
+    GFAPI_PUBLIC(glfs_h_open, 3.4.2);
 
 int
-glfs_h_access (struct glfs *fs, struct glfs_object *object, int mask) __THROW
-        GFAPI_PUBLIC(glfs_h_access, 3.6.0);
-
+glfs_h_access(glfs_t *fs, glfs_object_t *object, int mask) __THROW
+    GFAPI_PUBLIC(glfs_h_access, 3.6.0);
+
+struct glfs_object *
+glfs_h_creat_open(struct glfs *fs, struct glfs_object *parent, const char *path,
+                  int flags, mode_t mode, struct stat *stat,
+                  struct glfs_fd **out_fd) __THROW
+    GFAPI_PUBLIC(glfs_h_creat_open, 6.6);
 /*
   SYNOPSIS
 
@@ -272,8 +264,8 @@ glfs_h_access (struct glfs *fs, struct glfs_object *object, int mask) __THROW
 
   This API is used to poll for upcall events stored in the
   upcall list. Current users of this API is NFS-Ganesha.
-  Incase of any event received, it will be mapped appropriately
-  into 'callback_arg' along with the handle('glfs_object') to be
+  In case of any event received, it will be mapped appropriately
+  into 'glfs_upcall' along with the handle('glfs_object') to be
   passed to NFS-Ganesha.
 
   In case of success, applications need to check the value of
@@ -283,11 +275,8 @@ glfs_h_access (struct glfs *fs, struct glfs_object *object, int mask) __THROW
   PARAMETERS
 
   @fs: glfs object to poll the upcall events for
-  @cbk: Structure to store upcall events as desired by the application.
-        Application is responsible for allocating and passing the
-        references of all the pointers of this structure except for
-        "handle". In case of any events received, it needs to free
-        "handle"
+  @cbk: Pointer that will contain an upcall event for use by the application.
+        Application is responsible for free'ing the structure with glfs_free().
 
   RETURN VALUES
 
@@ -297,28 +286,69 @@ glfs_h_access (struct glfs *fs, struct glfs_object *object, int mask) __THROW
 */
 
 int
-glfs_h_poll_upcall (struct glfs *fs, struct callback_arg *cbk) __THROW
-        GFAPI_PUBLIC(glfs_h_poll_upcall, 3.7.0);
+glfs_h_poll_upcall(glfs_t *fs, glfs_upcall_t **cbk) __THROW
+    GFAPI_PUBLIC(glfs_h_poll_upcall, 3.7.16);
 
 int
-glfs_h_acl_set (struct glfs *fs, struct glfs_object *object,
-                const acl_type_t type, const acl_t acl) __THROW
-        GFAPI_PUBLIC(glfs_h_acl_set, 3.7.0);
+glfs_h_acl_set(glfs_t *fs, glfs_object_t *object, const acl_type_t type,
+               const acl_t acl) __THROW GFAPI_PUBLIC(glfs_h_acl_set, 3.7.0);
 
 acl_t
-glfs_h_acl_get (struct glfs *fs, struct glfs_object *object,
-                const acl_type_t type) __THROW
-        GFAPI_PUBLIC(glfs_h_acl_get, 3.7.0);
+glfs_h_acl_get(glfs_t *fs, glfs_object_t *object, const acl_type_t type) __THROW
+    GFAPI_PUBLIC(glfs_h_acl_get, 3.7.0);
 
 size_t
-glfs_h_anonymous_write (struct glfs *fs, struct glfs_object *object,
-                        const void *buf, size_t count, off_t offset) __THROW
-        GFAPI_PUBLIC(glfs_h_anonymous_write, 3.7.0);
+glfs_h_anonymous_write(glfs_t *fs, glfs_object_t *object, const void *buf,
+                       size_t count, off_t offset) __THROW
+    GFAPI_PUBLIC(glfs_h_anonymous_write, 3.7.0);
 
 ssize_t
-glfs_h_anonymous_read (struct glfs *fs, struct glfs_object *object,
-                      const void *buf, size_t count, off_t offset) __THROW
-        GFAPI_PUBLIC(glfs_h_anonymous_read, 3.7.0);
+glfs_h_anonymous_read(glfs_t *fs, glfs_object_t *object, const void *buf,
+                      size_t count, off_t offset) __THROW
+    GFAPI_PUBLIC(glfs_h_anonymous_read, 3.7.0);
+
+/*
+ * Caution: The object returned by this object gets freed as part
+ * of 'glfs_free(xstat)'. Make sure to have a copy using 'glfs_object_copy()'
+ * to use post that.
+ */
+glfs_object_t *
+glfs_xreaddirplus_get_object(struct glfs_xreaddirp_stat *xstat) __THROW
+    GFAPI_PUBLIC(glfs_xreaddirplus_get_object, 3.11.0);
+
+/* Applications should close the object returned by this routine
+ * explicitly using 'glfs_h_close()'
+ */
+glfs_object_t *
+glfs_object_copy(glfs_object_t *src) __THROW
+    GFAPI_PUBLIC(glfs_object_copy, 3.11.0);
+
+int
+glfs_h_lease(glfs_t *fs, glfs_object_t *object, glfs_lease_t *lease) __THROW
+    GFAPI_PUBLIC(glfs_h_lease, 4.0.0);
+
+glfs_object_t *
+glfs_h_find_handle(glfs_t *fs, unsigned char *handle, int len) __THROW
+    GFAPI_PUBLIC(glfs_h_lease, 4.0.0);
+
+/* Functions for getting details about the glfs_upcall_lease
+ *
+ * None of the pointers returned by the below functions should be free()'d,
+ * glfs_free()'d or glfs_h_close()'d by the application.
+ *
+ * Releasing of the structures is done by passing the glfs_upcall pointer
+ * to glfs_free().
+ */
+struct glfs_upcall_lease;
+typedef struct glfs_upcall_lease glfs_upcall_lease_t;
+
+glfs_object_t *
+glfs_upcall_lease_get_object(glfs_upcall_lease_t *arg) __THROW
+    GFAPI_PUBLIC(glfs_upcall_lease_get_object, 4.1.6);
+
+uint32_t
+glfs_upcall_lease_get_lease_type(glfs_upcall_lease_t *arg) __THROW
+    GFAPI_PUBLIC(glfs_upcall_lease_get_lease_type, 4.1.6);
 
 __END_DECLS
 
diff --git a/api/src/glfs-internal.h b/api/src/glfs-internal.h
index ab08cb5cfc3..7cc3b18a104 100644
--- a/api/src/glfs-internal.h
+++ b/api/src/glfs-internal.h
@@ -1,5 +1,5 @@
 /*
-  Copyright (c) 2012 Red Hat, Inc. <http://www.redhat.com>
+  Copyright (c) 2012-2018 Red Hat, Inc. <http://www.redhat.com>
   This file is part of GlusterFS.
 
   This file is licensed to you under your choice of the GNU Lesser
@@ -8,24 +8,25 @@
   cases as published by the Free Software Foundation.
 */
 
-
 #ifndef _GLFS_INTERNAL_H
 #define _GLFS_INTERNAL_H
 
-#include "xlator.h"
-#include "glusterfs.h"
-#include "upcall-utils.h"
+#include <glusterfs/xlator.h>
+#include <glusterfs/glusterfs.h>
+#include <glusterfs/upcall-utils.h>
 #include "glfs-handles.h"
+#include <glusterfs/refcount.h>
+#include <glusterfs/syncop.h>
 
 #define GLFS_SYMLINK_MAX_FOLLOW 2048
 
 #define DEFAULT_REVAL_COUNT 1
 
 /*
- * According to  pthread mutex and conditional variable ( cond, child_down_count,
- * upcall mutex and mutex) initialization of struct glfs members,
- * below GLFS_INIT_* flags are set in 'pthread_flags' member of struct glfs.
- * The flags are set from glfs_init() and  glfs_new_from_ctx() functions
+ * According to  pthread mutex and conditional variable ( cond,
+ * child_down_count, upcall mutex and mutex) initialization of struct glfs
+ * members, below GLFS_INIT_* flags are set in 'pthread_flags' member of struct
+ * glfs. The flags are set from glfs_init() and  glfs_new_from_ctx() functions
  * as part of fs inititialization.
  *
  * These flag bits are validated in glfs_fini() to destroy all or partially
@@ -36,39 +37,42 @@
  *
  */
 
-#define PTHREAD_MUTEX_INIT(mutex, attr, flags, mask, label) do { \
-        int __ret = -1;                                          \
-        __ret = pthread_mutex_init (mutex, attr);                \
-        if (__ret == 0)                                          \
-                flags |= mask;                                   \
-        else                                                     \
-                goto label;                                      \
-} while (0)
-
-#define PTHREAD_MUTEX_DESTROY(mutex, flags, mask) do {           \
-        if (flags & mask)                                        \
-                (void) pthread_mutex_destroy (mutex);            \
-} while (0)
-
-#define PTHREAD_COND_INIT(cond, attr, flags, mask, label) do {   \
-        int __ret = -1;                                          \
-        __ret = pthread_cond_init (cond, attr);                  \
-        if (__ret == 0)                                          \
-                flags |= mask;                                   \
-        else                                                     \
-                goto label;                                      \
-} while (0)
-
-#define PTHREAD_COND_DESTROY(cond, flags, mask) do {             \
-        if (flags & mask)                                        \
-                (void) pthread_cond_destroy (cond);              \
-} while (0)
-
-#define GLFS_INIT_MUTEX              0x00000001   /* pthread_mutex_flag */
-#define GLFS_INIT_COND               0x00000002   /* pthread_cond_flag */
-#define GLFS_INIT_COND_CHILD         0x00000004   /* pthread_cond_child_down_flag */
-#define GLFS_INIT_MUTEX_UPCALL       0x00000008   /* pthread_mutex_upcall_flag */
-
+#define PTHREAD_MUTEX_INIT(mutex, attr, flags, mask, label)                    \
+    do {                                                                       \
+        int __ret = -1;                                                        \
+        __ret = pthread_mutex_init(mutex, attr);                               \
+        if (__ret == 0)                                                        \
+            flags |= mask;                                                     \
+        else                                                                   \
+            goto label;                                                        \
+    } while (0)
+
+#define PTHREAD_MUTEX_DESTROY(mutex, flags, mask)                              \
+    do {                                                                       \
+        if (flags & mask)                                                      \
+            (void)pthread_mutex_destroy(mutex);                                \
+    } while (0)
+
+#define PTHREAD_COND_INIT(cond, attr, flags, mask, label)                      \
+    do {                                                                       \
+        int __ret = -1;                                                        \
+        __ret = pthread_cond_init(cond, attr);                                 \
+        if (__ret == 0)                                                        \
+            flags |= mask;                                                     \
+        else                                                                   \
+            goto label;                                                        \
+    } while (0)
+
+#define PTHREAD_COND_DESTROY(cond, flags, mask)                                \
+    do {                                                                       \
+        if (flags & mask)                                                      \
+            (void)pthread_cond_destroy(cond);                                  \
+    } while (0)
+
+#define GLFS_INIT_MUTEX 0x00000001        /* pthread_mutex_flag */
+#define GLFS_INIT_COND 0x00000002         /* pthread_cond_flag */
+#define GLFS_INIT_COND_CHILD 0x00000004   /* pthread_cond_child_down_flag */
+#define GLFS_INIT_MUTEX_UPCALL 0x00000008 /* pthread_mutex_upcall_flag */
 
 #ifndef GF_DARWIN_HOST_OS
 #ifndef GFAPI_PUBLIC
@@ -77,206 +81,411 @@
 #ifndef GFAPI_PRIVATE
 #define GFAPI_PRIVATE(sym, ver) /**/
 #endif
-#define GFAPI_SYMVER_PUBLIC_DEFAULT(fn, ver) \
-        asm(".symver pub_"STR(fn)", "STR(fn)"@@GFAPI_"STR(ver))
+#if __GNUC__ >= 10
+#define GFAPI_SYMVER_PUBLIC_DEFAULT(fn, ver)                                   \
+    __attribute__((__symver__(STR(fn) "@@GFAPI_" STR(ver))))
+
+#define GFAPI_SYMVER_PRIVATE_DEFAULT(fn, ver)                                  \
+    __attribute__((__symver__(STR(fn) "@@GFAPI_PRIVATE_" STR(ver))))
+
+#define GFAPI_SYMVER_PUBLIC(fn1, fn2, ver)                                     \
+    __attribute__((__symver__(STR(fn2) "@GFAPI_" STR(ver))))
 
-#define GFAPI_SYMVER_PRIVATE_DEFAULT(fn, ver) \
-        asm(".symver priv_"STR(fn)", "STR(fn)"@@GFAPI_PRIVATE_"STR(ver))
+#define GFAPI_SYMVER_PRIVATE(fn1, fn2, ver)                                    \
+    __attribute__((__symver__(STR(fn2) "@GFAPI_PRIVATE_" STR(ver))))
 
-#define GFAPI_SYMVER_PUBLIC(fn1, fn2, ver) \
-        asm(".symver pub_"STR(fn1)", "STR(fn2)"@GFAPI_"STR(ver))
+#else
+#define GFAPI_SYMVER_PUBLIC_DEFAULT(fn, ver)                                   \
+    asm(".symver pub_" STR(fn) ", " STR(fn) "@@GFAPI_" STR(ver));
+
+#define GFAPI_SYMVER_PRIVATE_DEFAULT(fn, ver)                                  \
+    asm(".symver priv_" STR(fn) ", " STR(fn) "@@GFAPI_PRIVATE_" STR(ver));
 
-#define GFAPI_SYMVER_PRIVATE(fn1, fn2, ver) \
-        asm(".symver priv_"STR(fn1)", "STR(fn2)"@GFAPI_PRIVATE_"STR(ver))
+#define GFAPI_SYMVER_PUBLIC(fn1, fn2, ver)                                     \
+    asm(".symver pub_" STR(fn1) ", " STR(fn2) "@GFAPI_" STR(ver));
+
+#define GFAPI_SYMVER_PRIVATE(fn1, fn2, ver)                                    \
+    asm(".symver priv_" STR(fn1) ", " STR(fn2) "@GFAPI_PRIVATE_" STR(ver));
+#endif
 #define STR(str) #str
 #else
 #ifndef GFAPI_PUBLIC
-#define GFAPI_PUBLIC(sym, ver) __asm("_" __STRING(sym) "$GFAPI_" __STRING(ver))
+#define GFAPI_PUBLIC(sym, ver) __asm("_" __STRING(sym) "$GFAPI_" __STRING(ver));
 #endif
 #ifndef GFAPI_PRIVATE
-#define GFAPI_PRIVATE(sym, ver) __asm("_" __STRING(sym) "$GFAPI_PRIVATE_" __STRING(ver))
+#define GFAPI_PRIVATE(sym, ver)                                                \
+    __asm("_" __STRING(sym) "$GFAPI_PRIVATE_" __STRING(ver));
 #endif
-#define GFAPI_SYMVER_PUBLIC_DEFAULT(fn, dotver) /**/
+#define GFAPI_SYMVER_PUBLIC_DEFAULT(fn, dotver)  /**/
 #define GFAPI_SYMVER_PRIVATE_DEFAULT(fn, dotver) /**/
-#define GFAPI_SYMVER_PUBLIC(fn1, fn2, dotver) /**/
-#define GFAPI_SYMVER_PRIVATE(fn1, fn2, dotver) /**/
+#define GFAPI_SYMVER_PUBLIC(fn1, fn2, dotver)    /**/
+#define GFAPI_SYMVER_PRIVATE(fn1, fn2, dotver)   /**/
 #endif
 
-/*
- * syncop_xxx() calls are executed in two ways, one is inside a synctask where
- * the executing function will do 'swapcontext' and the other is without
- * synctask where the executing thread is made to wait using pthread_cond_wait.
- * Executing thread may change when syncop_xxx() is executed inside a synctask.
- * This leads to errno_location change i.e. errno may give errno of
- * non-executing thread. So errno is not touched inside a synctask execution.
- * All gfapi calls are executed using the second way of executing syncop_xxx()
- * where the executing thread waits using pthread_cond_wait so it is ok to set
- * errno in these cases. The following macro makes syncop_xxx() behave just
- * like a system call, where -1 is returned and errno is set when a failure
- * occurs.
- */
-#define DECODE_SYNCOP_ERR(ret) do {  \
-        if (ret < 0) {          \
-                errno = -ret;   \
-                ret = -1;       \
-        } else {                \
-                errno = 0;      \
-        }                       \
-        } while (0)
-
-#define ESTALE_RETRY(ret,errno,reval,loc,label) do {	\
-	if (ret == -1 && errno == ESTALE) {	        \
-		if (reval < DEFAULT_REVAL_COUNT) {	\
-			reval++;			\
-			loc_wipe (loc);			\
-			goto label;			\
-		}					\
-	}						\
-	} while (0)
-
-#define GLFS_LOC_FILL_INODE(oinode, loc, label) do {   \
-	loc.inode = inode_ref (oinode);                \
-	gf_uuid_copy (loc.gfid, oinode->gfid);         \
-	ret = glfs_loc_touchup (&loc);                 \
-	if (ret != 0) {                                \
-		errno = EINVAL;                        \
-		goto label;                            \
-	}                                              \
-	} while (0)
-
-#define GLFS_LOC_FILL_PINODE(pinode, loc, ret, errno, label, path) do {   \
-	loc.inode = inode_new (pinode->table);                            \
-	if (!loc.inode) {                                                 \
-		ret = -1;                                                 \
-		errno = ENOMEM;                                           \
-		goto label;                                               \
-	}                                                                 \
-	loc.parent = inode_ref (pinode);                                  \
-	loc.name = path;                                                  \
-	ret = glfs_loc_touchup (&loc);                                    \
-	if (ret != 0) {                                                   \
-		errno = EINVAL;                                           \
-		goto label;                                               \
-	}                                                                 \
-	} while (0)
+#define ESTALE_RETRY(ret, errno, reval, loc, label)                            \
+    do {                                                                       \
+        if (ret == -1 && errno == ESTALE) {                                    \
+            if (reval < DEFAULT_REVAL_COUNT) {                                 \
+                reval++;                                                       \
+                loc_wipe(loc);                                                 \
+                goto label;                                                    \
+            }                                                                  \
+        }                                                                      \
+    } while (0)
+
+#define GLFS_LOC_FILL_INODE(oinode, loc, label)                                \
+    do {                                                                       \
+        loc.inode = inode_ref(oinode);                                         \
+        gf_uuid_copy(loc.gfid, oinode->gfid);                                  \
+        ret = glfs_loc_touchup(&loc);                                          \
+        if (ret != 0) {                                                        \
+            errno = EINVAL;                                                    \
+            goto label;                                                        \
+        }                                                                      \
+    } while (0)
+
+#define GLFS_LOC_FILL_PINODE(pinode, loc, ret, errno, label, path)             \
+    do {                                                                       \
+        loc.inode = inode_new(pinode->table);                                  \
+        if (!loc.inode) {                                                      \
+            ret = -1;                                                          \
+            errno = ENOMEM;                                                    \
+            goto label;                                                        \
+        }                                                                      \
+        loc.parent = inode_ref(pinode);                                        \
+        loc.name = path;                                                       \
+        ret = glfs_loc_touchup(&loc);                                          \
+        if (ret != 0) {                                                        \
+            errno = EINVAL;                                                    \
+            goto label;                                                        \
+        }                                                                      \
+    } while (0)
 
 struct glfs;
 
-struct _upcall_entry_t {
-        struct list_head  upcall_list;
-        struct gf_upcall  upcall_data;
+struct _upcall_entry {
+    struct list_head upcall_list;
+    struct gf_upcall upcall_data;
 };
-typedef struct _upcall_entry_t upcall_entry;
+typedef struct _upcall_entry upcall_entry;
 
-typedef int (*glfs_init_cbk) (struct glfs *fs, int ret);
+typedef int (*glfs_init_cbk)(struct glfs *fs, int ret);
 
 struct glfs {
-	char               *volname;
-        uuid_t              vol_uuid;
+    char *volname;
+    uuid_t vol_uuid;
+
+    glusterfs_ctx_t *ctx;
+
+    pthread_t poller;
+
+    glfs_init_cbk init_cbk;
+    pthread_mutex_t mutex;
+    pthread_cond_t cond;
+    pthread_cond_t child_down_cond; /* for broadcasting CHILD_DOWN */
+    int init;
+    int ret;
+    int err;
+
+    xlator_t *active_subvol; /* active graph */
+    xlator_t *mip_subvol;    /* graph for which migration is in
+                              * progress */
+    xlator_t *next_subvol;   /* Any new graph is put to
+                              * next_subvol, the graph in
+                              * next_subvol can either be moved
+                              * to mip_subvol (if any IO picks it
+                              * up for migration), or be
+                              * destroyed (if there is a new
+                              * graph, and this was never picked
+                              * for migration) */
+    xlator_t *old_subvol;
+
+    char *oldvolfile;
+    ssize_t oldvollen;
+
+    inode_t *cwd;
+
+    uint32_t dev_id; /* Used to fill st_dev in struct stat */
+
+    struct list_head openfds;
+
+    gf_boolean_t migration_in_progress;
+
+    gf_boolean_t cache_upcalls; /* add upcalls to the upcall_list? */
+    struct list_head upcall_list;
+    pthread_mutex_t upcall_list_mutex; /* mutex for upcall entry list */
+
+    uint32_t pin_refcnt;
+    uint32_t pthread_flags; /* GLFS_INIT_* # defines set this flag */
+
+    uint32_t upcall_events; /* Mask of upcall events application
+                             * is interested in */
+    glfs_upcall_cbk up_cbk; /* upcall cbk function to be registered */
+    void *up_data;          /* Opaque data provided by application
+                             * during upcall registration */
+    struct list_head waitq; /* waiting synctasks */
+};
 
-	glusterfs_ctx_t    *ctx;
+/* This enum is used to maintain the state of glfd. In case of async fops
+ * fd might be closed before the actual fop is complete. Therefore we need
+ * to track whether the fd is closed or not, instead actually closing it.*/
+enum glfs_fd_state { GLFD_INIT, GLFD_OPEN, GLFD_CLOSE };
 
-	pthread_t           poller;
+struct glfs_fd {
+    struct list_head openfds;
+    struct list_head list;
+    GF_REF_DECL;
+    struct glfs *fs;
+    enum glfs_fd_state state;
+    off_t offset;
+    fd_t *fd; /* Currently guared by @fs->mutex. TODO: per-glfd lock */
+    struct list_head entries;
+    gf_dirent_t *next;
+    struct dirent *readdirbuf;
+    gf_lkowner_t lk_owner;
+    glfs_leaseid_t lease_id; /* Stores lease_id of client in glfd */
+    gf_lock_t lock;          /* lock taken before updating fd state */
+    glfs_recall_cbk cbk;
+    void *cookie;
+};
 
-	glfs_init_cbk       init_cbk;
-	pthread_mutex_t     mutex;
-	pthread_cond_t      cond;
-        pthread_cond_t      child_down_cond; /* for broadcasting CHILD_DOWN */
-	int                 init;
-	int                 ret;
-	int                 err;
+/* glfs object handle introduced for the alternate gfapi implementation based
+   on glfs handles/gfid/inode
+*/
+struct glfs_object {
+    inode_t *inode;
+    uuid_t gfid;
+};
 
-	xlator_t           *active_subvol;
-	xlator_t           *next_subvol;
-	xlator_t           *old_subvol;
+struct glfs_upcall {
+    struct glfs *fs;                /* glfs object */
+    enum glfs_upcall_reason reason; /* Upcall event type */
+    void *event;                    /* changes based in the event type */
+    void (*free_event)(void *);     /* free event after the usage */
+};
 
-	char               *oldvolfile;
-	ssize_t             oldvollen;
+struct glfs_upcall_inode {
+    struct glfs_object *object;      /* Object which need to be acted upon */
+    int flags;                       /* Cache UPDATE/INVALIDATE flags */
+    struct stat buf;                 /* Latest stat of this entry */
+    unsigned int expire_time_attr;   /* the amount of time for which
+                                      * the application need to cache
+                                      * this entry */
+    struct glfs_object *p_object;    /* parent Object to be updated */
+    struct stat p_buf;               /* Latest stat of parent dir handle */
+    struct glfs_object *oldp_object; /* Old parent Object to be updated */
+    struct stat oldp_buf;            /* Latest stat of old parent dir handle */
+};
 
-	inode_t            *cwd;
+struct glfs_upcall_lease {
+    struct glfs_object *object; /* Object which need to be acted upon */
+    uint32_t lease_type;        /* Lease type to which client can downgrade to*/
+};
 
-	uint32_t            dev_id; /* Used to fill st_dev in struct stat */
+struct glfs_upcall_lease_fd {
+    uint32_t lease_type; /* Lease type to which client can downgrade to*/
+    void *fd_cookie;     /* Object which need to be acted upon */
+};
 
-	struct list_head    openfds;
+struct glfs_xreaddirp_stat {
+    struct stat
+        st; /* Stat for that dirent - corresponds to GFAPI_XREADDIRP_STAT */
+    struct glfs_object *object; /* handled for GFAPI_XREADDIRP_HANDLE */
+    uint32_t flags_handled;     /* final set of flags successfulyy handled */
+};
 
-	gf_boolean_t        migration_in_progress;
+#define DEFAULT_EVENT_POOL_SIZE 16384
+#define GF_MEMPOOL_COUNT_OF_DICT_T 4096
+#define GF_MEMPOOL_COUNT_OF_DATA_T (GF_MEMPOOL_COUNT_OF_DICT_T * 4)
+#define GF_MEMPOOL_COUNT_OF_DATA_PAIR_T (GF_MEMPOOL_COUNT_OF_DICT_T * 4)
 
-        struct list_head    upcall_list;
-        pthread_mutex_t     upcall_list_mutex; /* mutex for upcall entry list */
+#define GF_MEMPOOL_COUNT_OF_LRU_BUF_T 256
 
-        uint32_t            pin_refcnt;
-        uint32_t            pthread_flags; /* GLFS_INIT_* # defines set this flag */
-};
+typedef void(glfs_mem_release_t)(void *ptr);
 
-struct glfs_fd {
-	struct list_head   openfds;
-	struct glfs       *fs;
-	off_t              offset;
-	fd_t              *fd; /* Currently guared by @fs->mutex. TODO: per-glfd lock */
-	struct list_head   entries;
-	gf_dirent_t       *next;
-	struct dirent     *readdirbuf;
+struct glfs_mem_header {
+    uint32_t magic;
+    size_t nmemb;
+    size_t size;
+    glfs_mem_release_t *release;
 };
 
-/* glfs object handle introduced for the alternate gfapi implementation based
-   on glfs handles/gfid/inode
-*/
-struct glfs_object {
-        inode_t         *inode;
-        uuid_t          gfid;
-};
+#define GLFS_MEM_HEADER_SIZE (sizeof(struct glfs_mem_header))
+#define GLFS_MEM_HEADER_MAGIC 0x20170830
+
+static inline void *
+__glfs_calloc(size_t nmemb, size_t size, glfs_mem_release_t release,
+              uint32_t type, const char *typestr)
+{
+    struct glfs_mem_header *header = NULL;
+
+    header = __gf_calloc(nmemb, (size + GLFS_MEM_HEADER_SIZE), type, typestr);
+    if (!header)
+        return NULL;
+
+    header->magic = GLFS_MEM_HEADER_MAGIC;
+    header->nmemb = nmemb;
+    header->size = size;
+    header->release = release;
+
+    return header + 1;
+}
+
+static inline void *
+__glfs_malloc(size_t size, glfs_mem_release_t release, uint32_t type,
+              const char *typestr)
+{
+    struct glfs_mem_header *header = NULL;
+
+    header = __gf_malloc((size + GLFS_MEM_HEADER_SIZE), type, typestr);
+    if (!header)
+        return NULL;
+
+    header->magic = GLFS_MEM_HEADER_MAGIC;
+    header->nmemb = 1;
+    header->size = size;
+    header->release = release;
+
+    return header + 1;
+}
+
+static inline void *
+__glfs_realloc(void *ptr, size_t size)
+{
+    struct glfs_mem_header *old_header = NULL;
+    struct glfs_mem_header *new_header = NULL;
+    struct glfs_mem_header tmp_header;
+    void *new_ptr = NULL;
+
+    GF_ASSERT(NULL != ptr);
+
+    old_header = (struct glfs_mem_header *)(ptr - GLFS_MEM_HEADER_SIZE);
+    GF_ASSERT(old_header->magic == GLFS_MEM_HEADER_MAGIC);
+    tmp_header = *old_header;
+
+    new_ptr = __gf_realloc(old_header, (size + GLFS_MEM_HEADER_SIZE));
+    if (!new_ptr)
+        return NULL;
+
+    new_header = (struct glfs_mem_header *)new_ptr;
+    *new_header = tmp_header;
+    new_header->size = size;
+
+    return new_header + 1;
+}
+
+static inline void
+__glfs_free(void *free_ptr)
+{
+    struct glfs_mem_header *header = NULL;
+    void *release_ptr = NULL;
+    int i = 0;
+
+    if (!free_ptr)
+        return;
+
+    header = (struct glfs_mem_header *)(free_ptr - GLFS_MEM_HEADER_SIZE);
+    GF_ASSERT(header->magic == GLFS_MEM_HEADER_MAGIC);
+
+    if (header->release) {
+        release_ptr = free_ptr;
+        for (i = 0; i < header->nmemb; i++) {
+            header->release(release_ptr);
+            release_ptr += header->size;
+        }
+    }
+
+    __gf_free(header);
+}
+
+#define GLFS_CALLOC(nmemb, size, release, type)                                \
+    __glfs_calloc(nmemb, size, release, type, #type)
 
-#define DEFAULT_EVENT_POOL_SIZE           16384
-#define GF_MEMPOOL_COUNT_OF_DICT_T        4096
-#define GF_MEMPOOL_COUNT_OF_DATA_T        (GF_MEMPOOL_COUNT_OF_DICT_T * 4)
-#define GF_MEMPOOL_COUNT_OF_DATA_PAIR_T   (GF_MEMPOOL_COUNT_OF_DICT_T * 4)
-
-#define GF_MEMPOOL_COUNT_OF_LRU_BUF_T     256
-
-int glfs_mgmt_init (struct glfs *fs);
-void glfs_init_done (struct glfs *fs, int ret)
-        GFAPI_PRIVATE(glfs_init_done, 3.4.0);
-int glfs_process_volfp (struct glfs *fs, FILE *fp);
-int glfs_resolve (struct glfs *fs, xlator_t *subvol, const char *path,
-                       loc_t *loc, struct iatt *iatt, int reval)
-        GFAPI_PRIVATE(glfs_resolve, 3.7.0);
-int glfs_lresolve (struct glfs *fs, xlator_t *subvol, const char *path, loc_t *loc,
-		   struct iatt *iatt, int reval);
-fd_t *glfs_resolve_fd (struct glfs *fs, xlator_t *subvol, struct glfs_fd *glfd);
-
-fd_t *__glfs_migrate_fd (struct glfs *fs, xlator_t *subvol, struct glfs_fd *glfd);
-
-int glfs_first_lookup (xlator_t *subvol);
-
-void glfs_process_upcall_event (struct glfs *fs, void *data)
-        GFAPI_PRIVATE(glfs_process_upcall_event, 3.7.0);
-
-
-#define __GLFS_ENTRY_VALIDATE_FS(fs, label)                         \
-do {                                                                \
-        if (!fs) {                                                  \
-                errno = EINVAL;                                     \
-                goto label;                                         \
-        }                                                           \
-        old_THIS = THIS;                                            \
-        THIS = fs->ctx->master;                                     \
-} while (0)
-
-#define __GLFS_EXIT_FS                                              \
-do {                                                                \
-        THIS = old_THIS;                                            \
-} while (0)
-
-#define __GLFS_ENTRY_VALIDATE_FD(glfd, label)                       \
-do {                                                                \
-        if (!glfd || !glfd->fd || !glfd->fd->inode) {               \
-                errno = EBADF;                                      \
-                goto label;                                         \
-        }                                                           \
-        old_THIS = THIS;                                            \
-        THIS = glfd->fd->inode->table->xl->ctx->master;             \
-} while (0)
+#define GLFS_MALLOC(size, release, type)                                       \
+    __glfs_malloc(size, release, type, #type)
 
+#define GLFS_REALLOC(ptr, size) __glfs_realloc(ptr, size)
+
+#define GLFS_FREE(free_ptr) __glfs_free(free_ptr)
+
+int
+glfs_mgmt_init(struct glfs *fs);
+void
+glfs_init_done(struct glfs *fs, int ret) GFAPI_PRIVATE(glfs_init_done, 3.4.0);
+int
+glfs_process_volfp(struct glfs *fs, FILE *fp);
+int
+glfs_resolve(struct glfs *fs, xlator_t *subvol, const char *path, loc_t *loc,
+             struct iatt *iatt, int reval) GFAPI_PRIVATE(glfs_resolve, 3.7.0);
+int
+glfs_lresolve(struct glfs *fs, xlator_t *subvol, const char *path, loc_t *loc,
+              struct iatt *iatt, int reval);
+fd_t *
+glfs_resolve_fd(struct glfs *fs, xlator_t *subvol, struct glfs_fd *glfd);
+
+fd_t *
+__glfs_migrate_fd(struct glfs *fs, xlator_t *subvol, struct glfs_fd *glfd);
+
+int
+glfs_first_lookup(xlator_t *subvol);
+
+void
+glfs_process_upcall_event(struct glfs *fs, void *data)
+    GFAPI_PRIVATE(glfs_process_upcall_event, 3.7.0);
+
+#define __GLFS_ENTRY_VALIDATE_FS(fs, label)                                    \
+    do {                                                                       \
+        if (!fs) {                                                             \
+            errno = EINVAL;                                                    \
+            goto label;                                                        \
+        }                                                                      \
+        old_THIS = THIS;                                                       \
+        THIS = fs->ctx->master;                                                \
+    } while (0)
+
+#define __GLFS_EXIT_FS                                                         \
+    do {                                                                       \
+        THIS = old_THIS;                                                       \
+    } while (0)
+
+#define __GLFS_ENTRY_VALIDATE_FD(glfd, label)                                  \
+    do {                                                                       \
+        if (!glfd || !glfd->fd || !glfd->fd->inode ||                          \
+            glfd->state != GLFD_OPEN) {                                        \
+            errno = EBADF;                                                     \
+            goto label;                                                        \
+        }                                                                      \
+        old_THIS = THIS;                                                       \
+        THIS = glfd->fd->inode->table->xl->ctx->master;                        \
+    } while (0)
+
+#define __GLFS_LOCK_WAIT(fs)                                                   \
+    do {                                                                       \
+        struct synctask *task = NULL;                                          \
+                                                                               \
+        task = synctask_get();                                                 \
+                                                                               \
+        if (task) {                                                            \
+            list_add_tail(&task->waitq, &fs->waitq);                           \
+            pthread_mutex_unlock(&fs->mutex);                                  \
+            synctask_yield(task, NULL);                                        \
+            pthread_mutex_lock(&fs->mutex);                                    \
+        } else {                                                               \
+            /* non-synctask */                                                 \
+            pthread_cond_wait(&fs->cond, &fs->mutex);                          \
+        }                                                                      \
+    } while (0)
+
+#define __GLFS_SYNCTASK_WAKE(fs)                                               \
+    do {                                                                       \
+        struct synctask *waittask = NULL;                                      \
+                                                                               \
+        while (!list_empty(&fs->waitq)) {                                      \
+            waittask = list_entry(fs->waitq.next, struct synctask, waitq);     \
+            list_del_init(&waittask->waitq);                                   \
+            synctask_wake(waittask);                                           \
+        }                                                                      \
+    } while (0)
 
 /*
   By default all lock attempts from user context must
@@ -285,67 +494,93 @@ do {                                                                \
   we can give up the mutex during syncop calls so
   that bottom up calls (particularly CHILD_UP notify)
   can do a mutex_lock() on @glfs without deadlocking
-  the filesystem
+  the filesystem.
+
+  All the fops should wait for graph migration to finish
+  before starting the fops. Therefore these functions should
+  call glfs_lock with wait_for_migration as true. But waiting
+  for migration to finish in call-back path can result thread
+  dead-locks. The reason for this is we only have finite
+  number of epoll threads. so if we wait on epoll threads
+  there will not be any thread left to handle outstanding
+  rpc replies.
 */
 static inline int
-glfs_lock (struct glfs *fs)
+glfs_lock(struct glfs *fs, gf_boolean_t wait_for_migration)
 {
-	pthread_mutex_lock (&fs->mutex);
+    pthread_mutex_lock(&fs->mutex);
 
-	while (!fs->init)
-		pthread_cond_wait (&fs->cond, &fs->mutex);
+    while (!fs->init)
+        __GLFS_LOCK_WAIT(fs);
 
-	while (fs->migration_in_progress)
-		pthread_cond_wait (&fs->cond, &fs->mutex);
+    while (wait_for_migration && fs->migration_in_progress)
+        __GLFS_LOCK_WAIT(fs);
 
-	return 0;
+    return 0;
 }
 
-
 static inline void
-glfs_unlock (struct glfs *fs)
+glfs_unlock(struct glfs *fs)
 {
-	pthread_mutex_unlock (&fs->mutex);
+    pthread_mutex_unlock(&fs->mutex);
 }
 
+struct glfs_fd *
+glfs_fd_new(struct glfs *fs);
+void
+glfs_fd_bind(struct glfs_fd *glfd);
+void
+glfd_set_state_bind(struct glfs_fd *glfd);
+
+xlator_t *
+glfs_active_subvol(struct glfs *fs) GFAPI_PRIVATE(glfs_active_subvol, 3.4.0);
+xlator_t *
+__glfs_active_subvol(struct glfs *fs);
+void
+glfs_subvol_done(struct glfs *fs, xlator_t *subvol)
+    GFAPI_PRIVATE(glfs_subvol_done, 3.4.0);
+
+inode_t *
+glfs_refresh_inode(xlator_t *subvol, inode_t *inode);
+
+inode_t *
+glfs_cwd_get(struct glfs *fs);
+int
+glfs_cwd_set(struct glfs *fs, inode_t *inode);
+inode_t *
+glfs_resolve_inode(struct glfs *fs, xlator_t *subvol,
+                   struct glfs_object *object);
+int
+glfs_create_object(loc_t *loc, struct glfs_object **retobject);
+int
+__glfs_cwd_set(struct glfs *fs, inode_t *inode);
+
+int
+glfs_resolve_base(struct glfs *fs, xlator_t *subvol, inode_t *inode,
+                  struct iatt *iatt);
 
-void glfs_fd_destroy (struct glfs_fd *glfd);
-
-struct glfs_fd *glfs_fd_new (struct glfs *fs);
-void glfs_fd_bind (struct glfs_fd *glfd);
-
-xlator_t *glfs_active_subvol (struct glfs *fs)
-        GFAPI_PRIVATE(glfs_active_subvol, 3.4.0);
-xlator_t *__glfs_active_subvol (struct glfs *fs);
-void glfs_subvol_done (struct glfs *fs, xlator_t *subvol)
-        GFAPI_PRIVATE(glfs_subvol_done, 3.4.0);
-
-inode_t *glfs_refresh_inode (xlator_t *subvol, inode_t *inode);
-
-inode_t *glfs_cwd_get (struct glfs *fs);
-int glfs_cwd_set (struct glfs *fs, inode_t *inode);
-inode_t *glfs_resolve_inode (struct glfs *fs, xlator_t *subvol,
-			     struct glfs_object *object);
-int glfs_create_object (loc_t *loc, struct glfs_object **retobject);
-int __glfs_cwd_set (struct glfs *fs, inode_t *inode);
-
-int glfs_resolve_base (struct glfs *fs, xlator_t *subvol, inode_t *inode,
-		       struct iatt *iatt);
-int glfs_resolve_at (struct glfs *fs, xlator_t *subvol, inode_t *at,
-                          const char *origpath, loc_t *loc, struct iatt *iatt,
-                          int follow, int reval)
-        GFAPI_PRIVATE(glfs_resolve_at, 3.4.0);
-int glfs_loc_touchup (loc_t *loc)
-	GFAPI_PRIVATE(glfs_loc_touchup, 3.4.0);
-void glfs_iatt_to_stat (struct glfs *fs, struct iatt *iatt, struct stat *stat);
-int glfs_loc_link (loc_t *loc, struct iatt *iatt);
-int glfs_loc_unlink (loc_t *loc);
-dict_t *dict_for_key_value (const char *name, const char *value, size_t size);
-int glfs_getxattr_process (void *value, size_t size, dict_t *xattr,
-			   const char *name);
+int
+glfs_resolve_at(struct glfs *fs, xlator_t *subvol, inode_t *at,
+                const char *origpath, loc_t *loc, struct iatt *iatt, int follow,
+                int reval) GFAPI_PRIVATE(glfs_resolve_at, 3.4.0);
+int
+glfs_loc_touchup(loc_t *loc) GFAPI_PRIVATE(glfs_loc_touchup, 3.4.0);
+void
+glfs_iatt_to_stat(struct glfs *fs, struct iatt *iatt, struct stat *stat);
+void
+glfs_iatt_from_stat(struct stat *stat, int valid, struct iatt *iatt,
+                    int *gvalid);
+int
+glfs_loc_link(loc_t *loc, struct iatt *iatt);
+int
+glfs_loc_unlink(loc_t *loc);
+int
+glfs_getxattr_process(void *value, size_t size, dict_t *xattr,
+                      const char *name);
 
 /* Sends RPC call to glusterd to fetch required volume info */
-int glfs_get_volume_info (struct glfs *fs);
+int
+glfs_get_volume_info(struct glfs *fs);
 
 /*
   SYNOPSIS
@@ -371,8 +606,8 @@ int glfs_get_volume_info (struct glfs *fs);
        NULL   : Otherwise.
 */
 
-struct glfs *glfs_new_from_ctx (glusterfs_ctx_t *ctx)
-        GFAPI_PRIVATE(glfs_new_from_ctx, 3.7.0);
+struct glfs *
+glfs_new_from_ctx(glusterfs_ctx_t *ctx) GFAPI_PRIVATE(glfs_new_from_ctx, 3.7.0);
 
 /*
   SYNOPSIS
@@ -397,26 +632,125 @@ struct glfs *glfs_new_from_ctx (glusterfs_ctx_t *ctx)
        void
 */
 
-void glfs_free_from_ctx (struct glfs *fs)
-         GFAPI_PRIVATE(glfs_free_from_ctx, 3.7.0);
+void
+glfs_free_from_ctx(struct glfs *fs) GFAPI_PRIVATE(glfs_free_from_ctx, 3.7.0);
 
-int glfs_get_upcall_cache_invalidation (struct gf_upcall *to_up_data,
-                                        struct gf_upcall *from_up_data);
 int
-glfs_h_poll_cache_invalidation (struct glfs *fs,
-                                struct callback_arg *up_arg,
-                                struct gf_upcall *upcall_data);
+glfs_recall_lease_fd(struct glfs *fs, struct gf_upcall *up_data);
+
+int
+glfs_get_upcall_cache_invalidation(struct gf_upcall *to_up_data,
+                                   struct gf_upcall *from_up_data);
+int
+glfs_h_poll_cache_invalidation(struct glfs *fs, struct glfs_upcall *up_arg,
+                               struct gf_upcall *upcall_data);
 
 ssize_t
-glfs_anonymous_preadv (struct glfs *fs, struct glfs_object *object,
-                       const struct iovec *iovec, int iovcnt,
-                       off_t offset, int flags);
+glfs_anonymous_preadv(struct glfs *fs, struct glfs_object *object,
+                      const struct iovec *iovec, int iovcnt, off_t offset,
+                      int flags);
 ssize_t
-glfs_anonymous_pwritev (struct glfs *fs, struct glfs_object *object,
-                        const struct iovec *iovec, int iovcnt,
-                        off_t offset, int flags);
+glfs_anonymous_pwritev(struct glfs *fs, struct glfs_object *object,
+                       const struct iovec *iovec, int iovcnt, off_t offset,
+                       int flags);
 
 struct glfs_object *
-glfs_h_resolve_symlink (struct glfs *fs, struct glfs_object *object);
+glfs_h_resolve_symlink(struct glfs *fs, struct glfs_object *object);
 
+/* Deprecated structures that were passed to client applications, replaced by
+ * accessor functions. Do not use these in new applications, and update older
+ * usage.
+ *
+ * See http://review.gluster.org/14701 for more details.
+ *
+ * WARNING: These structures will be removed in the future.
+ */
+struct glfs_callback_arg {
+    struct glfs *fs;
+    enum glfs_upcall_reason reason;
+    void *event_arg;
+};
+
+struct glfs_callback_inode_arg {
+    struct glfs_object *object;      /* Object which need to be acted upon */
+    int flags;                       /* Cache UPDATE/INVALIDATE flags */
+    struct stat buf;                 /* Latest stat of this entry */
+    unsigned int expire_time_attr;   /* the amount of time for which
+                                      * the application need to cache
+                                      * this entry
+                                      */
+    struct glfs_object *p_object;    /* parent Object to be updated */
+    struct stat p_buf;               /* Latest stat of parent dir handle */
+    struct glfs_object *oldp_object; /* Old parent Object
+                                      * to be updated */
+    struct stat oldp_buf;            /* Latest stat of old parent
+                                      * dir handle */
+};
+struct dirent *
+glfs_readdirbuf_get(struct glfs_fd *glfd);
+
+gf_dirent_t *
+glfd_entry_next(struct glfs_fd *glfd, int plus);
+
+void
+gf_dirent_to_dirent(gf_dirent_t *gf_dirent, struct dirent *dirent);
+
+void
+gf_lease_to_glfs_lease(struct gf_lease *gf_lease, struct glfs_lease *lease);
+
+void
+glfs_lease_to_gf_lease(struct glfs_lease *lease, struct gf_lease *gf_lease);
+
+void
+glfs_release_upcall(void *ptr);
+
+int
+get_fop_attr_glfd(dict_t **fop_attr, struct glfs_fd *glfd);
+
+int
+set_fop_attr_glfd(struct glfs_fd *glfd);
+
+int
+get_fop_attr_thrd_key(dict_t **fop_attr);
+
+void
+unset_fop_attr(dict_t **fop_attr);
+
+/*
+  SYNOPSIS
+  glfs_statx: Fetch extended file attributes for the given path.
+
+  DESCRIPTION
+  This function fetches extended file attributes for the given path.
+
+  PARAMETERS
+  @fs: The 'virtual mount' object referencing a volume, under which file exists.
+  @path: Path of the file within the virtual mount.
+  @mask: Requested extended file attributes mask, (See mask defines above)
+
+  RETURN VALUES
+  -1 : Failure. @errno will be set with the type of failure.
+  0  : Filled in statxbuf with appropriate masks for valid items in the
+       structure.
+
+  ERRNO VALUES
+    EINVAL: fs is invalid
+    EINVAL: mask has unsupported bits set
+    Other errors as returned by stat(2)
+ */
+
+int
+glfs_statx(struct glfs *fs, const char *path, unsigned int mask,
+           struct glfs_stat *statxbuf) GFAPI_PRIVATE(glfs_statx, 6.0);
+
+void
+glfs_iatt_from_statx(struct iatt *, const struct glfs_stat *)
+    GFAPI_PRIVATE(glfs_iatt_from_statx, 6.0);
+
+/*
+ * This API is a per thread setting, similar to glfs_setfs{u/g}id, because of
+ * the call to syncopctx_setfspid.
+ */
+int
+glfs_setfspid(struct glfs *, pid_t) GFAPI_PRIVATE(glfs_setfspid, 6.1);
 #endif /* !_GLFS_INTERNAL_H */
diff --git a/api/src/glfs-master.c b/api/src/glfs-master.c
index bb657695489..100dcc16cc0 100644
--- a/api/src/glfs-master.c
+++ b/api/src/glfs-master.c
@@ -1,5 +1,5 @@
 /*
-  Copyright (c) 2012 Red Hat, Inc. <http://www.redhat.com>
+  Copyright (c) 2012-2016 Red Hat, Inc. <http://www.redhat.com>
   This file is part of GlusterFS.
 
   This file is licensed to you under your choice of the GNU Lesser
@@ -8,158 +8,176 @@
   cases as published by the Free Software Foundation.
 */
 
-#include <unistd.h>
-#include <string.h>
-#include <stdlib.h>
 #include <stdio.h>
-#include <inttypes.h>
-#include <limits.h>
 
-#include "xlator.h"
-#include "glusterfs.h"
+#include <glusterfs/glusterfs.h>
 
 #include "glfs-internal.h"
 #include "glfs-mem-types.h"
 #include "gfapi-messages.h"
 
-
 int
-graph_setup (struct glfs *fs, glusterfs_graph_t *graph)
+graph_setup(struct glfs *fs, glusterfs_graph_t *graph)
 {
-	xlator_t      *new_subvol = NULL;
-	xlator_t      *old_subvol = NULL;
-	inode_table_t *itable = NULL;
-	int            ret = -1;
-
-	new_subvol = graph->top;
-
-	/* This is called in a bottom-up context, it should specifically
-	   NOT be glfs_lock()
-	*/
-	pthread_mutex_lock (&fs->mutex);
-	{
-		if (new_subvol->switched ||
-		    new_subvol == fs->active_subvol ||
-		    new_subvol == fs->next_subvol) {
-			/* Spurious CHILD_UP event on old graph */
-			ret = 0;
-			goto unlock;
-		}
-
-		if (!new_subvol->itable) {
-			itable = inode_table_new (131072, new_subvol);
-			if (!itable) {
-				errno = ENOMEM;
-				ret = -1;
-				goto unlock;
-			}
-
-			new_subvol->itable = itable;
-		}
-
-		old_subvol = fs->next_subvol;
-		fs->next_subvol = new_subvol;
-		fs->next_subvol->winds++; /* first ref */
-		ret = 0;
-	}
+    xlator_t *new_subvol = NULL;
+    xlator_t *old_subvol = NULL;
+    inode_table_t *itable = NULL;
+    int ret = -1;
+
+    new_subvol = graph->top;
+
+    /* This is called in a bottom-up context, it should specifically
+       NOT be glfs_lock()
+    */
+    pthread_mutex_lock(&fs->mutex);
+    {
+        if (new_subvol->switched || new_subvol == fs->active_subvol ||
+            new_subvol == fs->next_subvol || new_subvol == fs->mip_subvol) {
+            /* Spurious CHILD_UP event on old graph */
+            ret = 0;
+            goto unlock;
+        }
+
+        if (!new_subvol->itable) {
+            itable = inode_table_new(131072, new_subvol);
+            if (!itable) {
+                errno = ENOMEM;
+                ret = -1;
+                goto unlock;
+            }
+
+            new_subvol->itable = itable;
+        }
+
+        old_subvol = fs->next_subvol;
+        fs->next_subvol = new_subvol;
+        fs->next_subvol->winds++; /* first ref */
+        ret = 0;
+    }
 unlock:
-	pthread_mutex_unlock (&fs->mutex);
+    pthread_mutex_unlock(&fs->mutex);
 
-	if (old_subvol)
-		/* wasn't picked up so far, skip */
-		glfs_subvol_done (fs, old_subvol);
+    if (old_subvol)
+        /* wasn't picked up so far, skip */
+        glfs_subvol_done(fs, old_subvol);
 
-	return ret;
+    return ret;
 }
 
-
 int
-notify (xlator_t *this, int event, void *data, ...)
+notify(xlator_t *this, int event, void *data, ...)
 {
-	glusterfs_graph_t   *graph = NULL;
-	struct glfs	    *fs = NULL;
-
-	graph = data;
-	fs = this->private;
-
-	switch (event) {
-	case GF_EVENT_GRAPH_NEW:
-		gf_msg (this->name, GF_LOG_INFO, 0, API_MSG_NEW_GRAPH,
-                        "New graph %s (%d) coming up",
-			uuid_utoa ((unsigned char *)graph->graph_uuid),
-			graph->id);
-		break;
-	case GF_EVENT_CHILD_UP:
-                pthread_mutex_lock (&fs->mutex);
-                {
-                        graph->used = 1;
-                }
-                pthread_mutex_unlock (&fs->mutex);
-		graph_setup (fs, graph);
-		glfs_init_done (fs, 0);
-		break;
-	case GF_EVENT_CHILD_DOWN:
-                pthread_mutex_lock (&fs->mutex);
-                {
-                        graph->used = 0;
-                        pthread_cond_broadcast (&fs->child_down_cond);
-                }
-                pthread_mutex_unlock (&fs->mutex);
-		graph_setup (fs, graph);
-		glfs_init_done (fs, 1);
-		break;
-	case GF_EVENT_CHILD_CONNECTING:
-		break;
+    glusterfs_graph_t *graph = NULL;
+    struct glfs *fs = NULL;
+
+    graph = data;
+    fs = this->private;
+
+    switch (event) {
+        case GF_EVENT_GRAPH_NEW:
+            gf_smsg(this->name, GF_LOG_INFO, 0, API_MSG_NEW_GRAPH,
+                    "graph-uuid=%s",
+                    uuid_utoa((unsigned char *)graph->graph_uuid), "id=%d",
+                    graph->id, NULL);
+            break;
+        case GF_EVENT_CHILD_UP:
+            pthread_mutex_lock(&fs->mutex);
+            {
+                graph->used = 1;
+            }
+            pthread_mutex_unlock(&fs->mutex);
+            graph_setup(fs, graph);
+            glfs_init_done(fs, 0);
+            break;
+        case GF_EVENT_CHILD_DOWN:
+            pthread_mutex_lock(&fs->mutex);
+            {
+                graph->used = 0;
+                pthread_cond_broadcast(&fs->child_down_cond);
+            }
+            pthread_mutex_unlock(&fs->mutex);
+            glfs_init_done(fs, 1);
+            break;
+        case GF_EVENT_CHILD_CONNECTING:
+            break;
         case GF_EVENT_UPCALL:
-                glfs_process_upcall_event (fs, data);
-                break;
-	default:
-		gf_msg_debug (this->name, 0, "got notify event %d", event);
-		break;
-	}
-
-	return 0;
+            glfs_process_upcall_event(fs, data);
+            break;
+        default:
+            gf_msg_debug(this->name, 0, "got notify event %d", event);
+            break;
+    }
+
+    return 0;
 }
 
-
 int
-mem_acct_init (xlator_t *this)
+mem_acct_init(xlator_t *this)
 {
-	int ret = -1;
+    int ret = -1;
 
-	if (!this)
-		return ret;
+    if (!this)
+        return ret;
 
-	ret = xlator_mem_acct_init (this, glfs_mt_end + 1);
-	if (ret) {
-		gf_msg (this->name, GF_LOG_ERROR, ENOMEM,
-                        API_MSG_MEM_ACCT_INIT_FAILED, "Failed to initialise "
-                        "memory accounting");
-		return ret;
-	}
+    ret = xlator_mem_acct_init(this, glfs_mt_end + 1);
+    if (ret) {
+        gf_smsg(this->name, GF_LOG_ERROR, ENOMEM, API_MSG_MEM_ACCT_INIT_FAILED,
+                NULL);
+        return ret;
+    }
 
-	return 0;
+    return 0;
 }
 
-
 int
-init (xlator_t *this)
+init(xlator_t *this)
 {
-	return 0;
+    return 0;
 }
 
-
 void
-fini (xlator_t *this)
+fini(xlator_t *this)
 {
+}
 
+/* place-holder fops */
+int
+glfs_forget(xlator_t *this, inode_t *inode)
+{
+    return 0;
 }
 
+int
+glfs_release(xlator_t *this, fd_t *fd)
+{
+    return 0;
+}
 
-struct xlator_dumpops dumpops;
+int
+glfs_releasedir(xlator_t *this, fd_t *fd)
+{
+    return 0;
+}
 
+struct xlator_dumpops dumpops;
 
 struct xlator_fops fops;
 
-
-struct xlator_cbks cbks;
+struct xlator_cbks cbks = {
+    .forget = glfs_forget,
+    .release = glfs_release,
+    .releasedir = glfs_releasedir,
+};
+
+xlator_api_t xlator_api = {
+    .init = init,
+    .fini = fini,
+    .notify = notify,
+    .mem_acct_init = mem_acct_init,
+    .op_version = {1},
+    .dumpops = &dumpops,
+    .fops = &fops,
+    .cbks = &cbks,
+    .identifier = "glfs-api",
+    .category = GF_MAINTAINED,
+};
diff --git a/api/src/glfs-mem-types.h b/api/src/glfs-mem-types.h
index cad1ca95d4f..bfa325a3ad9 100644
--- a/api/src/glfs-mem-types.h
+++ b/api/src/glfs-mem-types.h
@@ -1,5 +1,5 @@
 /*
-  Copyright (c) 2012 Red Hat, Inc. <http://www.redhat.com>
+  Copyright (c) 2012-2017 Red Hat, Inc. <http://www.redhat.com>
   This file is part of GlusterFS.
 
   This file is licensed to you under your choice of the GNU Lesser
@@ -11,22 +11,25 @@
 #ifndef _GLFS_MEM_TYPES_H
 #define _GLFS_MEM_TYPES_H
 
-#include "mem-types.h"
+#include <glusterfs/mem-types.h>
 
 #define GF_MEM_TYPE_START (gf_common_mt_end + 1)
 
 enum glfs_mem_types_ {
-        glfs_mt_call_pool_t = GF_MEM_TYPE_START,
-        glfs_mt_xlator_t,
-	glfs_mt_glfs_fd_t,
-	glfs_mt_glfs_io_t,
-	glfs_mt_volfile_t,
-	glfs_mt_xlator_cmdline_option_t,
-        glfs_mt_server_cmdline_t,
-	glfs_mt_glfs_object_t,
-	glfs_mt_readdirbuf_t,
-        glfs_mt_upcall_entry_t,
-	glfs_mt_acl_t,
-	glfs_mt_end
+    glfs_mt_call_pool_t = GF_MEM_TYPE_START,
+    glfs_mt_xlator_t,
+    glfs_mt_glfs_fd_t,
+    glfs_mt_glfs_io_t,
+    glfs_mt_volfile_t,
+    glfs_mt_xlator_cmdline_option_t,
+    glfs_mt_server_cmdline_t,
+    glfs_mt_glfs_object_t,
+    glfs_mt_readdirbuf_t,
+    glfs_mt_upcall_entry_t,
+    glfs_mt_acl_t,
+    glfs_mt_upcall_inode_t,
+    glfs_mt_realpath_t,
+    glfs_mt_xreaddirp_stat_t,
+    glfs_mt_end
 };
 #endif
diff --git a/api/src/glfs-mgmt.c b/api/src/glfs-mgmt.c
index 593611612b8..7c82b8cd162 100644
--- a/api/src/glfs-mgmt.c
+++ b/api/src/glfs-mgmt.c
@@ -1,5 +1,5 @@
 /*
-  Copyright (c) 2012 Red Hat, Inc. <http://www.redhat.com>
+  Copyright (c) 2012-2018 Red Hat, Inc. <http://www.redhat.com>
   This file is part of GlusterFS.
 
   This file is licensed to you under your choice of the GNU Lesser
@@ -8,7 +8,6 @@
   cases as published by the Free Software Foundation.
 */
 
-
 #include <stdio.h>
 #include <sys/types.h>
 #include <sys/wait.h>
@@ -16,895 +15,1035 @@
 #include <signal.h>
 #include <pthread.h>
 
-#include "glusterfs.h"
+#include <glusterfs/glusterfs.h>
 #include "glfs.h"
-#include "stack.h"
-#include "dict.h"
-#include "event.h"
-#include "defaults.h"
+#include <glusterfs/dict.h>
 
 #include "rpc-clnt.h"
 #include "protocol-common.h"
-#include "glusterfs3.h"
-#include "portmap-xdr.h"
-#include "xdr-common.h"
 #include "xdr-generic.h"
+#include "rpc-common-xdr.h"
 
-#include "syncop.h"
-#include "xlator.h"
+#include <glusterfs/syncop.h>
 
 #include "glfs-internal.h"
-#include "glfs-mem-types.h"
 #include "gfapi-messages.h"
+#include <glusterfs/syscall.h>
 
-int glfs_volfile_fetch (struct glfs *fs);
-int32_t glfs_get_volume_info_rpc (call_frame_t *frame, xlator_t *this,
-                                  struct glfs *fs);
+int
+glfs_volfile_fetch(struct glfs *fs);
+int32_t
+glfs_get_volume_info_rpc(call_frame_t *frame, xlator_t *this, struct glfs *fs);
 
 int
-glfs_process_volfp (struct glfs *fs, FILE *fp)
+glfs_process_volfp(struct glfs *fs, FILE *fp)
 {
-	glusterfs_graph_t  *graph = NULL;
-	int		    ret = -1;
-	xlator_t	   *trav = NULL;
-	glusterfs_ctx_t	   *ctx = NULL;
-
-	ctx = fs->ctx;
-	graph = glusterfs_graph_construct (fp);
-	if (!graph) {
-		gf_msg ("glfs", GF_LOG_ERROR, errno,
-                        API_MSG_GRAPH_CONSTRUCT_FAILED,
-                        "failed to construct the graph");
-		goto out;
-	}
-
-	for (trav = graph->first; trav; trav = trav->next) {
-		if (strcmp (trav->type, "mount/fuse") == 0) {
-			gf_msg ("glfs", GF_LOG_ERROR, EINVAL,
-                                API_MSG_FUSE_XLATOR_ERROR,
-				"fuse xlator cannot be specified "
-				"in volume file");
-			goto out;
-		}
-	}
-
-	ret = glusterfs_graph_prepare (graph, ctx);
-	if (ret) {
-		glusterfs_graph_destroy (graph);
-		goto out;
-	}
-
-	ret = glusterfs_graph_activate (graph, ctx);
-
-	if (ret) {
-		glusterfs_graph_destroy (graph);
-		goto out;
-	}
-
-	ret = 0;
+    glusterfs_graph_t *graph = NULL;
+    int ret = -1;
+    xlator_t *trav = NULL;
+    glusterfs_ctx_t *ctx = NULL;
+
+    ctx = fs->ctx;
+    graph = glusterfs_graph_construct(fp);
+    if (!graph) {
+        gf_smsg("glfs", GF_LOG_ERROR, errno, API_MSG_GRAPH_CONSTRUCT_FAILED,
+                NULL);
+        goto out;
+    }
+
+    for (trav = graph->first; trav; trav = trav->next) {
+        if (strcmp(trav->type, "mount/api") == 0) {
+            gf_smsg("glfs", GF_LOG_ERROR, EINVAL, API_MSG_API_XLATOR_ERROR,
+                    NULL);
+            goto out;
+        }
+    }
+
+    ret = glusterfs_graph_prepare(graph, ctx, fs->volname);
+    if (ret) {
+        glusterfs_graph_destroy(graph);
+        goto out;
+    }
+
+    ret = glusterfs_graph_activate(graph, ctx);
+
+    if (ret) {
+        glusterfs_graph_destroy(graph);
+        goto out;
+    }
+
+    gf_log_dump_graph(fp, graph);
+
+    ret = 0;
 out:
-	if (fp)
-		fclose (fp);
+    if (fp)
+        fclose(fp);
 
-	if (!ctx->active) {
-		ret = -1;
-	}
+    if (!ctx->active) {
+        ret = -1;
+    }
 
-	return ret;
+    return ret;
 }
 
-
 int
-mgmt_cbk_spec (struct rpc_clnt *rpc, void *mydata, void *data)
+mgmt_cbk_spec(struct rpc_clnt *rpc, void *mydata, void *data)
 {
-	struct glfs *fs = NULL;
-	xlator_t    *this = NULL;
+    struct glfs *fs = NULL;
+    xlator_t *this = NULL;
 
-	this = mydata;
-	fs = this->private;
+    this = mydata;
+    fs = this->private;
 
-	glfs_volfile_fetch (fs);
+    glfs_volfile_fetch(fs);
 
-	return 0;
+    return 0;
 }
 
-
 int
-mgmt_cbk_event (struct rpc_clnt *rpc, void *mydata, void *data)
+mgmt_cbk_event(struct rpc_clnt *rpc, void *mydata, void *data)
 {
-	return 0;
+    return 0;
 }
 
+static int
+mgmt_cbk_statedump(struct rpc_clnt *rpc, void *mydata, void *data)
+{
+    struct glfs *fs = NULL;
+    xlator_t *this = NULL;
+    gf_statedump target_pid = {
+        0,
+    };
+    struct iovec *iov = NULL;
+    int ret = -1;
+
+    this = mydata;
+    if (!this) {
+        gf_smsg("glfs", GF_LOG_ERROR, EINVAL, API_MSG_NULL, "mydata", NULL);
+        errno = EINVAL;
+        goto out;
+    }
+
+    fs = this->private;
+    if (!fs) {
+        gf_smsg("glfs", GF_LOG_ERROR, EINVAL, API_MSG_NULL, "glfs", NULL);
+        errno = EINVAL;
+        goto out;
+    }
+
+    iov = (struct iovec *)data;
+    if (!iov) {
+        gf_smsg("glfs", GF_LOG_ERROR, EINVAL, API_MSG_NULL, "iovec data", NULL);
+        errno = EINVAL;
+        goto out;
+    }
+
+    ret = xdr_to_generic(*iov, &target_pid, (xdrproc_t)xdr_gf_statedump);
+    if (ret < 0) {
+        gf_smsg("glfs", GF_LOG_ERROR, EINVAL, API_MSG_DECODE_XDR_FAILED, NULL);
+        goto out;
+    }
+
+    gf_msg_trace("glfs", 0, "statedump requested for pid: %d", target_pid.pid);
+
+    if ((uint64_t)getpid() == target_pid.pid) {
+        gf_msg_debug("glfs", 0, "Taking statedump for pid: %d", target_pid.pid);
+
+        ret = glfs_sysrq(fs, GLFS_SYSRQ_STATEDUMP);
+        if (ret < 0) {
+            gf_smsg("glfs", GF_LOG_INFO, 0, API_MSG_STATEDUMP_FAILED, NULL);
+        }
+    }
+out:
+    return ret;
+}
 
-rpcclnt_cb_actor_t mgmt_cbk_actors[GF_CBK_MAXVALUE] = {
-	[GF_CBK_FETCHSPEC] = {"FETCHSPEC", GF_CBK_FETCHSPEC, mgmt_cbk_spec },
-	[GF_CBK_EVENT_NOTIFY] = {"EVENTNOTIFY", GF_CBK_EVENT_NOTIFY,
-				 mgmt_cbk_event},
+static rpcclnt_cb_actor_t mgmt_cbk_actors[GF_CBK_MAXVALUE] = {
+    [GF_CBK_FETCHSPEC] = {"FETCHSPEC", mgmt_cbk_spec, GF_CBK_FETCHSPEC},
+    [GF_CBK_EVENT_NOTIFY] = {"EVENTNOTIFY", mgmt_cbk_event,
+                             GF_CBK_EVENT_NOTIFY},
+    [GF_CBK_STATEDUMP] = {"STATEDUMP", mgmt_cbk_statedump, GF_CBK_STATEDUMP},
 };
 
-
-struct rpcclnt_cb_program mgmt_cbk_prog = {
-	.progname  = "GlusterFS Callback",
-	.prognum   = GLUSTER_CBK_PROGRAM,
-	.progver   = GLUSTER_CBK_VERSION,
-	.actors	   = mgmt_cbk_actors,
-	.numactors = GF_CBK_MAXVALUE,
+static struct rpcclnt_cb_program mgmt_cbk_prog = {
+    .progname = "GlusterFS Callback",
+    .prognum = GLUSTER_CBK_PROGRAM,
+    .progver = GLUSTER_CBK_VERSION,
+    .actors = mgmt_cbk_actors,
+    .numactors = GF_CBK_MAXVALUE,
 };
 
-char *clnt_handshake_procs[GF_HNDSK_MAXVALUE] = {
-	[GF_HNDSK_NULL]		= "NULL",
-	[GF_HNDSK_SETVOLUME]	= "SETVOLUME",
-	[GF_HNDSK_GETSPEC]	= "GETSPEC",
-	[GF_HNDSK_PING]		= "PING",
-	[GF_HNDSK_EVENT_NOTIFY] = "EVENTNOTIFY",
-        [GF_HNDSK_GET_VOLUME_INFO] = "GETVOLUMEINFO",
+static char *clnt_handshake_procs[GF_HNDSK_MAXVALUE] = {
+    [GF_HNDSK_NULL] = "NULL",
+    [GF_HNDSK_SETVOLUME] = "SETVOLUME",
+    [GF_HNDSK_GETSPEC] = "GETSPEC",
+    [GF_HNDSK_PING] = "PING",
+    [GF_HNDSK_EVENT_NOTIFY] = "EVENTNOTIFY",
+    [GF_HNDSK_GET_VOLUME_INFO] = "GETVOLUMEINFO",
 };
 
-rpc_clnt_prog_t clnt_handshake_prog = {
-	.progname  = "GlusterFS Handshake",
-	.prognum   = GLUSTER_HNDSK_PROGRAM,
-	.progver   = GLUSTER_HNDSK_VERSION,
-	.procnames = clnt_handshake_procs,
+static rpc_clnt_prog_t clnt_handshake_prog = {
+    .progname = "GlusterFS Handshake",
+    .prognum = GLUSTER_HNDSK_PROGRAM,
+    .progver = GLUSTER_HNDSK_VERSION,
+    .procnames = clnt_handshake_procs,
 };
 
-
 int
-mgmt_submit_request (void *req, call_frame_t *frame,
-		     glusterfs_ctx_t *ctx,
-		     rpc_clnt_prog_t *prog, int procnum,
-		     fop_cbk_fn_t cbkfn, xdrproc_t xdrproc)
+mgmt_submit_request(void *req, call_frame_t *frame, glusterfs_ctx_t *ctx,
+                    rpc_clnt_prog_t *prog, int procnum, fop_cbk_fn_t cbkfn,
+                    xdrproc_t xdrproc)
 {
-	int			ret	    = -1;
-	int			count	   = 0;
-	struct iovec		iov	    = {0, };
-	struct iobuf		*iobuf = NULL;
-	struct iobref		*iobref = NULL;
-	ssize_t			xdr_size = 0;
-
-	iobref = iobref_new ();
-	if (!iobref) {
-		goto out;
-	}
-
-	if (req) {
-		xdr_size = xdr_sizeof (xdrproc, req);
-
-		iobuf = iobuf_get2 (ctx->iobuf_pool, xdr_size);
-		if (!iobuf) {
-			goto out;
-		};
-
-		iobref_add (iobref, iobuf);
-
-		iov.iov_base = iobuf->ptr;
-		iov.iov_len  = iobuf_pagesize (iobuf);
-
-		/* Create the xdr payload */
-		ret = xdr_serialize_generic (iov, req, xdrproc);
-		if (ret == -1) {
-			gf_msg (THIS->name, GF_LOG_WARNING, 0,
-                                API_MSG_XDR_PAYLOAD_FAILED,
-				"failed to create XDR payload");
-			goto out;
-		}
-		iov.iov_len = ret;
-		count = 1;
-	}
-
-	/* Send the msg */
-	ret = rpc_clnt_submit (ctx->mgmt, prog, procnum, cbkfn,
-			       &iov, count,
-			       NULL, 0, iobref, frame, NULL, 0, NULL, 0, NULL);
+    int ret = -1;
+    int count = 0;
+    struct iovec iov = {
+        0,
+    };
+    struct iobuf *iobuf = NULL;
+    struct iobref *iobref = NULL;
+    ssize_t xdr_size = 0;
+
+    iobref = iobref_new();
+    if (!iobref) {
+        goto out;
+    }
+
+    if (req) {
+        xdr_size = xdr_sizeof(xdrproc, req);
+
+        iobuf = iobuf_get2(ctx->iobuf_pool, xdr_size);
+        if (!iobuf) {
+            goto out;
+        };
+
+        iobref_add(iobref, iobuf);
+
+        iov.iov_base = iobuf->ptr;
+        iov.iov_len = iobuf_pagesize(iobuf);
+
+        /* Create the xdr payload */
+        ret = xdr_serialize_generic(iov, req, xdrproc);
+        if (ret == -1) {
+            gf_smsg(THIS->name, GF_LOG_WARNING, 0, API_MSG_XDR_PAYLOAD_FAILED,
+                    NULL);
+            goto out;
+        }
+        iov.iov_len = ret;
+        count = 1;
+    }
+
+    /* Send the msg */
+    ret = rpc_clnt_submit(ctx->mgmt, prog, procnum, cbkfn, &iov, count, NULL, 0,
+                          iobref, frame, NULL, 0, NULL, 0, NULL);
 
 out:
-	if (iobref)
-		iobref_unref (iobref);
+    if (iobref)
+        iobref_unref(iobref);
 
-	if (iobuf)
-		iobuf_unref (iobuf);
-	return ret;
+    if (iobuf)
+        iobuf_unref(iobuf);
+    return ret;
 }
 
 /*
  * Callback routine for 'GF_HNDSK_GET_VOLUME_INFO' rpc request
  */
 int
-mgmt_get_volinfo_cbk (struct rpc_req *req, struct iovec *iov,
-                      int count, void *myframe)
+mgmt_get_volinfo_cbk(struct rpc_req *req, struct iovec *iov, int count,
+                     void *myframe)
 {
-        int                        ret                  = 0;
-        char                       *volume_id_str       = NULL;
-        dict_t                     *dict                = NULL;
-        char                       key[1024]            = {0};
-        gf_get_volume_info_rsp     rsp                  = {0,};
-        call_frame_t               *frame               = NULL;
-        glusterfs_ctx_t            *ctx                 = NULL;
-        struct glfs                *fs                  = NULL;
-        struct syncargs            *args;
-
-        frame = myframe;
-        ctx = frame->this->ctx;
-        args = frame->local;
-
-        if (!ctx) {
-                gf_msg (frame->this->name, GF_LOG_ERROR, EINVAL,
-                        API_MSG_INVALID_ENTRY, "NULL context");
-                errno = EINVAL;
-                ret = -1;
-                goto out;
-        }
-
-        fs = ((xlator_t *)ctx->master)->private;
-
-        if (-1 == req->rpc_status) {
-                gf_msg (frame->this->name, GF_LOG_ERROR, EINVAL,
-                        API_MSG_INVALID_ENTRY,
-                        "GET_VOLUME_INFO RPC call is not successfull");
-                errno = EINVAL;
-                ret = -1;
-                goto out;
-        }
-
-        ret = xdr_to_generic (*iov, &rsp, (xdrproc_t)xdr_gf_get_volume_info_rsp);
-
-        if (ret < 0) {
-                gf_msg (frame->this->name, GF_LOG_ERROR, 0,
-                        API_MSG_XDR_RESPONSE_DECODE_FAILED,
-                        "Failed to decode xdr response for GET_VOLUME_INFO");
-                goto out;
-        }
-
-        gf_msg_debug (frame->this->name, 0, "Received resp to GET_VOLUME_INFO "
-                      "RPC: %d", rsp.op_ret);
-
-        if (rsp.op_ret == -1) {
-                errno = rsp.op_errno;
-                ret = -1;
-                goto out;
-        }
-
-        if (!rsp.dict.dict_len) {
-                gf_msg (frame->this->name, GF_LOG_ERROR, EINVAL,
-                        API_MSG_INVALID_ENTRY, "Response received for "
-                        "GET_VOLUME_INFO RPC call is not valid");
-                ret = -1;
-                errno = EINVAL;
-                goto out;
-        }
-
-        dict = dict_new ();
-
-        if (!dict) {
-                ret = -1;
-                errno = ENOMEM;
-                goto out;
-        }
-
-        ret = dict_unserialize (rsp.dict.dict_val,
-                                rsp.dict.dict_len,
-                                &dict);
-
-        if (ret) {
-                errno = ENOMEM;
-                goto out;
-        }
-
-        snprintf (key, sizeof (key), "volume_id");
-        ret = dict_get_str (dict, key, &volume_id_str);
-        if (ret) {
-                errno = EINVAL;
-                goto out;
-        }
-
-        ret = 0;
+    int ret = 0;
+    char *volume_id_str = NULL;
+    dict_t *dict = NULL;
+    gf_get_volume_info_rsp rsp = {
+        0,
+    };
+    call_frame_t *frame = NULL;
+    glusterfs_ctx_t *ctx = NULL;
+    struct glfs *fs = NULL;
+    struct syncargs *args;
+
+    frame = myframe;
+    ctx = frame->this->ctx;
+    args = frame->local;
+
+    if (!ctx) {
+        gf_smsg(frame->this->name, GF_LOG_ERROR, EINVAL, API_MSG_NULL,
+                "context", NULL);
+        errno = EINVAL;
+        ret = -1;
+        goto out;
+    }
+
+    fs = ((xlator_t *)ctx->master)->private;
+
+    if (-1 == req->rpc_status) {
+        gf_smsg(frame->this->name, GF_LOG_ERROR, EINVAL,
+                API_MSG_CALL_NOT_SUCCESSFUL, NULL);
+        errno = EINVAL;
+        ret = -1;
+        goto out;
+    }
+
+    ret = xdr_to_generic(*iov, &rsp, (xdrproc_t)xdr_gf_get_volume_info_rsp);
+
+    if (ret < 0) {
+        gf_smsg(frame->this->name, GF_LOG_ERROR, 0,
+                API_MSG_XDR_RESPONSE_DECODE_FAILED, NULL);
+        goto out;
+    }
+
+    gf_msg_debug(frame->this->name, 0,
+                 "Received resp to GET_VOLUME_INFO "
+                 "RPC: %d",
+                 rsp.op_ret);
+
+    if (rsp.op_ret == -1) {
+        errno = rsp.op_errno;
+        ret = -1;
+        goto out;
+    }
+
+    if (!rsp.dict.dict_len) {
+        gf_smsg(frame->this->name, GF_LOG_ERROR, EINVAL, API_MSG_CALL_NOT_VALID,
+                NULL);
+        ret = -1;
+        errno = EINVAL;
+        goto out;
+    }
+
+    dict = dict_new();
+
+    if (!dict) {
+        ret = -1;
+        errno = ENOMEM;
+        goto out;
+    }
+
+    ret = dict_unserialize(rsp.dict.dict_val, rsp.dict.dict_len, &dict);
+
+    if (ret) {
+        errno = ENOMEM;
+        goto out;
+    }
+
+    ret = dict_get_str_sizen(dict, "volume_id", &volume_id_str);
+    if (ret) {
+        errno = EINVAL;
+        goto out;
+    }
+
+    ret = 0;
 out:
-        if (volume_id_str) {
-                gf_msg_debug (frame->this->name, 0,
-                              "Volume Id: %s", volume_id_str);
-                pthread_mutex_lock (&fs->mutex);
-                gf_uuid_parse (volume_id_str, fs->vol_uuid);
-                pthread_mutex_unlock (&fs->mutex);
-        }
+    if (volume_id_str) {
+        gf_msg_debug(frame->this->name, 0, "Volume Id: %s", volume_id_str);
+        pthread_mutex_lock(&fs->mutex);
+        gf_uuid_parse(volume_id_str, fs->vol_uuid);
+        pthread_mutex_unlock(&fs->mutex);
+    }
 
-        if (ret) {
-                gf_msg (frame->this->name, GF_LOG_ERROR, errno,
-                        API_MSG_GET_VOLINFO_CBK_FAILED, "In GET_VOLUME_INFO "
-                        "cbk, received error: %s", strerror(errno));
-        }
+    if (ret) {
+        gf_smsg(frame->this->name, GF_LOG_ERROR, errno,
+                API_MSG_GET_VOLINFO_CBK_FAILED, "error=%s", strerror(errno),
+                NULL);
+    }
 
-        if (dict)
-                dict_destroy (dict);
+    if (dict)
+        dict_unref(dict);
 
-        if (rsp.dict.dict_val)
-                free (rsp.dict.dict_val);
+    if (rsp.dict.dict_val)
+        free(rsp.dict.dict_val);
 
-        if (rsp.op_errstr && *rsp.op_errstr)
-                free (rsp.op_errstr);
+    if (rsp.op_errstr)
+        free(rsp.op_errstr);
 
-        gf_msg_debug (frame->this->name, 0, "Returning: %d", ret);
+    gf_msg_debug(frame->this->name, 0, "Returning: %d", ret);
 
-        __wake (args);
+    __wake(args);
 
-        return ret;
+    return ret;
 }
 
+GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_get_volumeid, 3.5.0)
 int
-pub_glfs_get_volumeid (struct glfs *fs, char *volid, size_t size)
+pub_glfs_get_volumeid(struct glfs *fs, char *volid, size_t size)
 {
-        /* TODO: Define a global macro to store UUID size */
-        size_t uuid_size = 16;
-
-        DECLARE_OLD_THIS;
-        __GLFS_ENTRY_VALIDATE_FS (fs, invalid_fs);
-
-        pthread_mutex_lock (&fs->mutex);
-        {
-                /* check if the volume uuid is initialized */
-                if (!gf_uuid_is_null (fs->vol_uuid)) {
-                        pthread_mutex_unlock (&fs->mutex);
-                        goto done;
-                }
+    /* TODO: Define a global macro to store UUID size */
+    size_t uuid_size = 16;
+
+    DECLARE_OLD_THIS;
+    __GLFS_ENTRY_VALIDATE_FS(fs, invalid_fs);
+
+    pthread_mutex_lock(&fs->mutex);
+    {
+        /* check if the volume uuid is initialized */
+        if (!gf_uuid_is_null(fs->vol_uuid)) {
+            pthread_mutex_unlock(&fs->mutex);
+            goto done;
         }
-        pthread_mutex_unlock (&fs->mutex);
+    }
+    pthread_mutex_unlock(&fs->mutex);
 
-        /* Need to fetch volume_uuid */
-        glfs_get_volume_info (fs);
+    /* Need to fetch volume_uuid */
+    glfs_get_volume_info(fs);
 
-        if (gf_uuid_is_null (fs->vol_uuid)) {
-                gf_msg (THIS->name, GF_LOG_ERROR, EINVAL,
-                        API_MSG_FETCH_VOLUUID_FAILED, "Unable to fetch "
-                        "volume UUID");
-                goto out;
-        }
+    if (gf_uuid_is_null(fs->vol_uuid)) {
+        gf_smsg(THIS->name, GF_LOG_ERROR, EINVAL, API_MSG_FETCH_VOLUUID_FAILED,
+                NULL);
+        goto out;
+    }
 
 done:
-        if (!volid || !size) {
-                gf_msg_debug (THIS->name, 0, "volumeid/size is null");
-                __GLFS_EXIT_FS;
-                return uuid_size;
-        }
+    if (!volid || !size) {
+        gf_msg_debug(THIS->name, 0, "volumeid/size is null");
+        __GLFS_EXIT_FS;
+        return uuid_size;
+    }
 
-        if (size < uuid_size) {
-                gf_msg (THIS->name, GF_LOG_ERROR, ERANGE, API_MSG_INSUFF_SIZE,
-                        "Insufficient size passed");
-                errno = ERANGE;
-                goto out;
-        }
+    if (size < uuid_size) {
+        gf_smsg(THIS->name, GF_LOG_ERROR, ERANGE, API_MSG_INSUFF_SIZE, NULL);
+        errno = ERANGE;
+        goto out;
+    }
 
-        memcpy (volid, fs->vol_uuid, uuid_size);
+    memcpy(volid, fs->vol_uuid, uuid_size);
 
-        __GLFS_EXIT_FS;
+    __GLFS_EXIT_FS;
 
-        return uuid_size;
+    return uuid_size;
 
 out:
-        __GLFS_EXIT_FS;
+    __GLFS_EXIT_FS;
 
 invalid_fs:
-        return -1;
+    return -1;
 }
 
-GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_get_volumeid, 3.5.0);
-
 int
-glfs_get_volume_info (struct glfs *fs)
+glfs_get_volume_info(struct glfs *fs)
 {
-        call_frame_t     *frame = NULL;
-        glusterfs_ctx_t  *ctx   = NULL;
-        struct syncargs  args   = {0, };
-        int              ret    = 0;
-
-        ctx = fs->ctx;
-        frame = create_frame (THIS, ctx->pool);
-        if (!frame) {
-                gf_msg ("glfs", GF_LOG_ERROR, ENOMEM,
-                        API_MSG_FRAME_CREAT_FAILED,
-                        "failed to create the frame");
-                ret = -1;
-                goto out;
-        }
+    call_frame_t *frame = NULL;
+    glusterfs_ctx_t *ctx = NULL;
+    struct syncargs args = {
+        0,
+    };
+    int ret = 0;
 
-        frame->local = &args;
+    ctx = fs->ctx;
+    frame = create_frame(THIS, ctx->pool);
+    if (!frame) {
+        gf_smsg("glfs", GF_LOG_ERROR, ENOMEM, API_MSG_FRAME_CREAT_FAILED, NULL);
+        ret = -1;
+        goto out;
+    }
 
-        __yawn ((&args));
+    frame->local = &args;
 
-        ret = glfs_get_volume_info_rpc (frame, THIS, fs);
-        if (ret)
-                goto out;
+    __yawn((&args));
+
+    ret = glfs_get_volume_info_rpc(frame, THIS, fs);
+    if (ret)
+        goto out;
 
-        __yield ((&args));
+    __yield((&args));
 
-        frame->local = NULL;
-        STACK_DESTROY (frame->root);
+    frame->local = NULL;
+    STACK_DESTROY(frame->root);
 
 out:
-        return ret;
+    return ret;
 }
 
 int32_t
-glfs_get_volume_info_rpc (call_frame_t *frame, xlator_t *this,
-                          struct glfs *fs)
+glfs_get_volume_info_rpc(call_frame_t *frame, xlator_t *this, struct glfs *fs)
 {
-        gf_get_volume_info_req  req       = {{0,}};
-        int                     ret       = 0;
-        glusterfs_ctx_t         *ctx      = NULL;
-        dict_t                  *dict     = NULL;
-        int32_t                 flags     = 0;
-
-        if (!frame || !this || !fs) {
-                ret = -1;
-                goto out;
-        }
-
-        ctx = fs->ctx;
-
-        dict = dict_new ();
-        if (!dict) {
-                ret = -1;
-                goto out;
-        }
-
-        if (fs->volname) {
-                ret = dict_set_str (dict, "volname", fs->volname);
-                if (ret)
-                        goto out;
-        }
-
-        // Set the flags for the fields which we are interested in
-        flags = (int32_t)GF_GET_VOLUME_UUID; //ctx->flags;
-        ret = dict_set_int32 (dict, "flags", flags);
-        if (ret) {
-                gf_msg (frame->this->name, GF_LOG_ERROR, EINVAL,
-                        API_MSG_DICT_SET_FAILED, "failed to set flags");
-                goto out;
-        }
-
-        ret = dict_allocate_and_serialize (dict, &req.dict.dict_val,
-                                           &req.dict.dict_len);
-
-
-        ret = mgmt_submit_request (&req, frame, ctx, &clnt_handshake_prog,
-                                   GF_HNDSK_GET_VOLUME_INFO,
-                                   mgmt_get_volinfo_cbk,
-                                   (xdrproc_t)xdr_gf_get_volume_info_req);
+    gf_get_volume_info_req req = {{
+        0,
+    }};
+    int ret = 0;
+    glusterfs_ctx_t *ctx = NULL;
+    dict_t *dict = NULL;
+    int32_t flags = 0;
+
+    if (!frame || !this || !fs) {
+        ret = -1;
+        goto out;
+    }
+
+    ctx = fs->ctx;
+
+    dict = dict_new();
+    if (!dict) {
+        ret = -1;
+        goto out;
+    }
+
+    if (fs->volname) {
+        ret = dict_set_str(dict, "volname", fs->volname);
+        if (ret)
+            goto out;
+    }
+
+    // Set the flags for the fields which we are interested in
+    flags = (int32_t)GF_GET_VOLUME_UUID;  // ctx->flags;
+    ret = dict_set_int32(dict, "flags", flags);
+    if (ret) {
+        gf_smsg(frame->this->name, GF_LOG_ERROR, EINVAL,
+                API_MSG_DICT_SET_FAILED, "flags", NULL);
+        goto out;
+    }
+
+    ret = dict_allocate_and_serialize(dict, &req.dict.dict_val,
+                                      &req.dict.dict_len);
+
+    ret = mgmt_submit_request(&req, frame, ctx, &clnt_handshake_prog,
+                              GF_HNDSK_GET_VOLUME_INFO, mgmt_get_volinfo_cbk,
+                              (xdrproc_t)xdr_gf_get_volume_info_req);
 out:
-        if (dict) {
-                dict_unref (dict);
-        }
+    if (dict) {
+        dict_unref(dict);
+    }
 
-        GF_FREE (req.dict.dict_val);
+    GF_FREE(req.dict.dict_val);
 
-        return ret;
+    return ret;
 }
 
 static int
-glusterfs_oldvolfile_update (struct glfs *fs, char *volfile, ssize_t size)
+glusterfs_oldvolfile_update(struct glfs *fs, char *volfile, ssize_t size)
 {
-	int ret = -1;
-
-        pthread_mutex_lock (&fs->mutex);
-
-	fs->oldvollen = size;
-	if (!fs->oldvolfile) {
-		fs->oldvolfile = GF_CALLOC (1, size+1, glfs_mt_volfile_t);
-	} else {
-		fs->oldvolfile = GF_REALLOC (fs->oldvolfile, size+1);
-	}
-
-	if (!fs->oldvolfile) {
-		fs->oldvollen = 0;
-	} else {
-		memcpy (fs->oldvolfile, volfile, size);
-		fs->oldvollen = size;
-		ret = 0;
-	}
+    int ret = -1;
+
+    pthread_mutex_lock(&fs->mutex);
+
+    fs->oldvollen = size;
+    if (!fs->oldvolfile) {
+        fs->oldvolfile = CALLOC(1, size + 1);
+    } else {
+        fs->oldvolfile = REALLOC(fs->oldvolfile, size + 1);
+    }
+
+    if (!fs->oldvolfile) {
+        fs->oldvollen = 0;
+    } else {
+        memcpy(fs->oldvolfile, volfile, size);
+        fs->oldvollen = size;
+        ret = 0;
+    }
 
-        pthread_mutex_unlock (&fs->mutex);
+    pthread_mutex_unlock(&fs->mutex);
 
-	return ret;
+    return ret;
 }
 
-
 int
-glfs_mgmt_getspec_cbk (struct rpc_req *req, struct iovec *iov, int count,
-                       void *myframe)
+glfs_mgmt_getspec_cbk(struct rpc_req *req, struct iovec *iov, int count,
+                      void *myframe)
 {
-	gf_getspec_rsp		 rsp   = {0,};
-	call_frame_t		*frame = NULL;
-	glusterfs_ctx_t		*ctx = NULL;
-	int			 ret   = 0;
-	ssize_t			 size = 0;
-	FILE			*tmpfp = NULL;
-	int			 need_retry = 0;
-	struct glfs		*fs = NULL;
-
-	frame = myframe;
-	ctx = frame->this->ctx;
-
-	if (!ctx) {
-		gf_msg (frame->this->name, GF_LOG_ERROR, EINVAL,
-                        API_MSG_INVALID_ENTRY, "NULL context");
-		errno = EINVAL;
-		ret = -1;
-		goto out;
-	}
-
-	fs = ((xlator_t *)ctx->master)->private;
-
-	if (-1 == req->rpc_status) {
-		ret = -1;
-		need_retry = 1;
-		goto out;
-	}
-
-	ret = xdr_to_generic (*iov, &rsp, (xdrproc_t)xdr_gf_getspec_rsp);
-	if (ret < 0) {
-		gf_msg (frame->this->name, GF_LOG_ERROR, 0,
-                        API_MSG_XDR_DECODE_FAILED, "XDR decoding error");
-		ret   = -1;
-		goto out;
-	}
-
-	if (-1 == rsp.op_ret) {
-		gf_msg (frame->this->name, GF_LOG_ERROR, rsp.op_errno,
-                        API_MSG_GET_VOLFILE_FAILED,
-			"failed to get the 'volume file' from server");
-		ret = -1;
-		errno = rsp.op_errno;
-		goto out;
-	}
-
-	ret = 0;
-	size = rsp.op_ret;
-
-	if ((size == fs->oldvollen) &&
-	    (memcmp (fs->oldvolfile, rsp.spec, size) == 0)) {
-		gf_msg (frame->this->name, GF_LOG_INFO, 0,
-                        API_MSG_VOLFILE_INFO,
-			"No change in volfile, continuing");
-		goto out;
-	}
-
-	tmpfp = tmpfile ();
-	if (!tmpfp) {
-		ret = -1;
-		goto out;
-	}
-
-	fwrite (rsp.spec, size, 1, tmpfp);
-	fflush (tmpfp);
-	if (ferror (tmpfp)) {
-		ret = -1;
-		goto out;
-	}
-
-	/*  Check if only options have changed. No need to reload the
-	*  volfile if topology hasn't changed.
-	*  glusterfs_volfile_reconfigure returns 3 possible return states
-	*  return 0	     =======> reconfiguration of options has succeeded
-	*  return 1	     =======> the graph has to be reconstructed and all the xlators should be inited
-	*  return -1(or -ve) =======> Some Internal Error occurred during the operation
-	*/
-
-	ret = glusterfs_volfile_reconfigure (fs->oldvollen, tmpfp, fs->ctx,
-					     fs->oldvolfile);
-	if (ret == 0) {
-		gf_msg_debug ("glusterfsd-mgmt", 0, "No need to re-load "
-                              "volfile, reconfigure done");
-		ret = glusterfs_oldvolfile_update (fs, rsp.spec, size);
-		goto out;
-	}
-
-	if (ret < 0) {
-		gf_msg_debug ("glusterfsd-mgmt", 0, "Reconfigure failed !!");
-		goto out;
-	}
-
-	ret = glfs_process_volfp (fs, tmpfp);
-	/* tmpfp closed */
-	tmpfp = NULL;
-	if (ret)
-		goto out;
-
-	ret = glusterfs_oldvolfile_update (fs, rsp.spec, size);
+    gf_getspec_rsp rsp = {
+        0,
+    };
+    call_frame_t *frame = NULL;
+    glusterfs_ctx_t *ctx = NULL;
+    int ret = 0;
+    ssize_t size = 0;
+    FILE *tmpfp = NULL;
+    int need_retry = 0;
+    struct glfs *fs = NULL;
+    dict_t *dict = NULL;
+    char *servers_list = NULL;
+    int tmp_fd = -1;
+    char template[] = "/tmp/gfapi.volfile.XXXXXX";
+
+    frame = myframe;
+    ctx = frame->this->ctx;
+
+    if (!ctx) {
+        gf_smsg(frame->this->name, GF_LOG_ERROR, EINVAL, API_MSG_NULL,
+                "context", NULL);
+        errno = EINVAL;
+        ret = -1;
+        goto out;
+    }
+
+    fs = ((xlator_t *)ctx->master)->private;
+
+    if (-1 == req->rpc_status) {
+        ret = -1;
+        need_retry = 1;
+        goto out;
+    }
+
+    ret = xdr_to_generic(*iov, &rsp, (xdrproc_t)xdr_gf_getspec_rsp);
+    if (ret < 0) {
+        gf_smsg(frame->this->name, GF_LOG_ERROR, 0, API_MSG_XDR_DECODE_FAILED,
+                NULL);
+        ret = -1;
+        goto out;
+    }
+
+    if (-1 == rsp.op_ret) {
+        gf_smsg(frame->this->name, GF_LOG_ERROR, rsp.op_errno,
+                API_MSG_GET_VOLFILE_FAILED, "from server", NULL);
+        ret = -1;
+        errno = rsp.op_errno;
+        goto out;
+    }
+
+    if (!rsp.xdata.xdata_len) {
+        goto volfile;
+    }
+
+    dict = dict_new();
+    if (!dict) {
+        ret = -1;
+        errno = ENOMEM;
+        goto out;
+    }
+
+    ret = dict_unserialize(rsp.xdata.xdata_val, rsp.xdata.xdata_len, &dict);
+    if (ret) {
+        gf_log(frame->this->name, GF_LOG_ERROR,
+               "failed to unserialize xdata to dictionary");
+        goto out;
+    }
+    dict->extra_stdfree = rsp.xdata.xdata_val;
+
+    /* glusterd2 only */
+    ret = dict_get_str(dict, "servers-list", &servers_list);
+    if (ret) {
+        goto volfile;
+    }
+
+    gf_log(frame->this->name, GF_LOG_INFO,
+           "Received list of available volfile servers: %s", servers_list);
+
+    ret = gf_process_getspec_servers_list(&ctx->cmd_args, servers_list);
+    if (ret) {
+        gf_log(frame->this->name, GF_LOG_ERROR,
+               "Failed (%s) to process servers list: %s", strerror(errno),
+               servers_list);
+    }
+
+volfile:
+    ret = 0;
+    size = rsp.op_ret;
+
+    pthread_mutex_lock(&fs->mutex);
+    if ((size == fs->oldvollen) &&
+        (memcmp(fs->oldvolfile, rsp.spec, size) == 0)) {
+        pthread_mutex_unlock(&fs->mutex);
+        gf_smsg(frame->this->name, GF_LOG_INFO, 0, API_MSG_VOLFILE_INFO, NULL);
+        goto out;
+    }
+    pthread_mutex_unlock(&fs->mutex);
+
+    /* coverity[secure_temp] mkstemp uses 0600 as the mode and is safe */
+    tmp_fd = mkstemp(template);
+    if (-1 == tmp_fd) {
+        ret = -1;
+        goto out;
+    }
+
+    /* Calling unlink so that when the file is closed or program
+     * terminates the temporary file is deleted.
+     */
+    ret = sys_unlink(template);
+    if (ret < 0) {
+        gf_smsg(frame->this->name, GF_LOG_INFO, 0, API_MSG_UNABLE_TO_DEL,
+                "template=%s", template, NULL);
+        ret = 0;
+    }
+
+    tmpfp = fdopen(tmp_fd, "w+b");
+    if (!tmpfp) {
+        ret = -1;
+        goto out;
+    }
+
+    fwrite(rsp.spec, size, 1, tmpfp);
+    fflush(tmpfp);
+    if (ferror(tmpfp)) {
+        ret = -1;
+        goto out;
+    }
+
+    /*  Check if only options have changed. No need to reload the
+     *  volfile if topology hasn't changed.
+     *  glusterfs_volfile_reconfigure returns 3 possible return states
+     *  return 0	     =======> reconfiguration of options has succeeded
+     *  return 1	     =======> the graph has to be reconstructed and all
+     * the xlators should be inited return -1(or -ve) =======> Some Internal
+     * Error occurred during the operation
+     */
+
+    pthread_mutex_lock(&fs->mutex);
+    ret = gf_volfile_reconfigure(fs->oldvollen, tmpfp, fs->ctx, fs->oldvolfile);
+    pthread_mutex_unlock(&fs->mutex);
+
+    if (ret == 0) {
+        gf_msg_debug("glusterfsd-mgmt", 0,
+                     "No need to re-load "
+                     "volfile, reconfigure done");
+        ret = glusterfs_oldvolfile_update(fs, rsp.spec, size);
+        goto out;
+    }
+
+    if (ret < 0) {
+        gf_msg_debug("glusterfsd-mgmt", 0, "Reconfigure failed !!");
+        goto out;
+    }
+
+    ret = glfs_process_volfp(fs, tmpfp);
+    /* tmpfp closed */
+    tmpfp = NULL;
+    tmp_fd = -1;
+    if (ret)
+        goto out;
+
+    ret = glusterfs_oldvolfile_update(fs, rsp.spec, size);
 out:
-	STACK_DESTROY (frame->root);
-
-	if (rsp.spec)
-		free (rsp.spec);
-
-	// Stop if server is running at an unsupported op-version
-	if (ENOTSUP == ret) {
-		gf_msg ("mgmt", GF_LOG_ERROR, ENOTSUP, API_MSG_WRONG_OPVERSION,
-                        "Server is operating at an op-version which is not "
-                        "supported");
-		errno = ENOTSUP;
-		glfs_init_done (fs, -1);
-	}
-
-	if (ret && ctx && !ctx->active) {
-		/* Do it only for the first time */
-		/* Failed to get the volume file, something wrong,
-		   restart the process */
-		gf_msg ("glfs-mgmt", GF_LOG_ERROR, EINVAL,
-                        API_MSG_INVALID_ENTRY,
-			"failed to fetch volume file (key:%s)",
-			ctx->cmd_args.volfile_id);
-		if (!need_retry) {
-			if (!errno)
-				errno = EINVAL;
-			glfs_init_done (fs, -1);
-		}
-	}
-
-	if (tmpfp)
-		fclose (tmpfp);
-
-	return 0;
-}
-
-
-int
-glfs_volfile_fetch (struct glfs *fs)
-{
-	cmd_args_t	 *cmd_args = NULL;
-	gf_getspec_req	  req = {0, };
-	int		  ret = 0;
-	call_frame_t	 *frame = NULL;
-	glusterfs_ctx_t	 *ctx = NULL;
-        dict_t           *dict = NULL;
-
-	ctx = fs->ctx;
-	cmd_args = &ctx->cmd_args;
-
-	frame = create_frame (THIS, ctx->pool);
-
-	req.key = cmd_args->volfile_id;
-	req.flags = 0;
-
-        dict = dict_new ();
-        if (!dict) {
-                ret = -1;
-                goto out;
-        }
-
-        // Set the supported min and max op-versions, so glusterd can make a
-        // decision
-        ret = dict_set_int32 (dict, "min-op-version", GD_OP_VERSION_MIN);
-        if (ret) {
-                gf_msg (THIS->name, GF_LOG_ERROR, EINVAL,
-                        API_MSG_DICT_SET_FAILED,
-                        "Failed to set min-op-version in request dict");
-                goto out;
+    STACK_DESTROY(frame->root);
+
+    if (rsp.spec)
+        free(rsp.spec);
+
+    if (dict)
+        dict_unref(dict);
+
+    // Stop if server is running at an unsupported op-version
+    if (ENOTSUP == ret) {
+        gf_smsg("mgmt", GF_LOG_ERROR, ENOTSUP, API_MSG_WRONG_OPVERSION, NULL);
+        errno = ENOTSUP;
+        glfs_init_done(fs, -1);
+    }
+
+    if (ret && ctx && !ctx->active) {
+        /* Do it only for the first time */
+        /* Failed to get the volume file, something wrong,
+           restart the process */
+        gf_smsg("glfs-mgmt", GF_LOG_ERROR, EINVAL, API_MSG_GET_VOLFILE_FAILED,
+                "key=%s", ctx->cmd_args.volfile_id, NULL);
+        if (!need_retry) {
+            if (!errno)
+                errno = EINVAL;
+            glfs_init_done(fs, -1);
         }
+    }
 
-        ret = dict_set_int32 (dict, "max-op-version", GD_OP_VERSION_MAX);
-        if (ret) {
-                gf_msg (THIS->name, GF_LOG_ERROR, EINVAL,
-                        API_MSG_DICT_SET_FAILED,
-                        "Failed to set max-op-version in request dict");
-                goto out;
-        }
+    if (tmpfp)
+        fclose(tmpfp);
+    else if (tmp_fd != -1)
+        sys_close(tmp_fd);
 
-        ret = dict_allocate_and_serialize (dict, &req.xdata.xdata_val,
-                                           &req.xdata.xdata_len);
-        if (ret < 0) {
-                gf_msg (THIS->name, GF_LOG_ERROR, 0,
-                        API_MSG_DICT_SERIALIZE_FAILED,
-                        "Failed to serialize dictionary");
-                goto out;
-        }
+    return 0;
+}
 
-	ret = mgmt_submit_request (&req, frame, ctx, &clnt_handshake_prog,
-				   GF_HNDSK_GETSPEC, glfs_mgmt_getspec_cbk,
-				   (xdrproc_t)xdr_gf_getspec_req);
+int
+glfs_volfile_fetch(struct glfs *fs)
+{
+    cmd_args_t *cmd_args = NULL;
+    gf_getspec_req req = {
+        0,
+    };
+    int ret = -1;
+    call_frame_t *frame = NULL;
+    glusterfs_ctx_t *ctx = NULL;
+    dict_t *dict = NULL;
+
+    ctx = fs->ctx;
+    cmd_args = &ctx->cmd_args;
+
+    req.key = cmd_args->volfile_id;
+    req.flags = 0;
+
+    dict = dict_new();
+    if (!dict) {
+        goto out;
+    }
+
+    // Set the supported min and max op-versions, so glusterd can make a
+    // decision
+    ret = dict_set_int32(dict, "min-op-version", GD_OP_VERSION_MIN);
+    if (ret) {
+        gf_smsg(THIS->name, GF_LOG_ERROR, EINVAL, API_MSG_DICT_SET_FAILED,
+                "min-op-version", NULL);
+        goto out;
+    }
+
+    ret = dict_set_int32(dict, "max-op-version", GD_OP_VERSION_MAX);
+    if (ret) {
+        gf_smsg(THIS->name, GF_LOG_ERROR, EINVAL, API_MSG_DICT_SET_FAILED,
+                "max-op-version", NULL);
+        goto out;
+    }
+
+    /* Ask for a list of volfile (glusterd2 only) servers */
+    if (GF_CLIENT_PROCESS == ctx->process_mode) {
+        req.flags = req.flags | GF_GETSPEC_FLAG_SERVERS_LIST;
+    }
+
+    ret = dict_allocate_and_serialize(dict, &req.xdata.xdata_val,
+                                      &req.xdata.xdata_len);
+    if (ret < 0) {
+        gf_smsg(THIS->name, GF_LOG_ERROR, 0, API_MSG_DICT_SERIALIZE_FAILED,
+                NULL);
+        goto out;
+    }
+
+    frame = create_frame(THIS, ctx->pool);
+    if (!frame) {
+        ret = -1;
+        goto out;
+    }
+
+    ret = mgmt_submit_request(&req, frame, ctx, &clnt_handshake_prog,
+                              GF_HNDSK_GETSPEC, glfs_mgmt_getspec_cbk,
+                              (xdrproc_t)xdr_gf_getspec_req);
 out:
-        return ret;
-}
+    if (req.xdata.xdata_val)
+        GF_FREE(req.xdata.xdata_val);
+    if (dict)
+        dict_unref(dict);
 
+    return ret;
+}
 
 static int
-mgmt_rpc_notify (struct rpc_clnt *rpc, void *mydata, rpc_clnt_event_t event,
-		 void *data)
+mgmt_rpc_notify(struct rpc_clnt *rpc, void *mydata, rpc_clnt_event_t event,
+                void *data)
 {
-	xlator_t	*this = NULL;
-	glusterfs_ctx_t *ctx = NULL;
-        server_cmdline_t *server = NULL;
-        rpc_transport_t  *rpc_trans = NULL;
-	struct glfs	 *fs = NULL;
-	int		 ret = 0;
-
-	this = mydata;
-        rpc_trans = rpc->conn.trans;
-
-	ctx = this->ctx;
-	if (!ctx)
-		goto out;
-
-	fs = ((xlator_t *)ctx->master)->private;
-
-	switch (event) {
-	case RPC_CLNT_DISCONNECT:
-		if (!ctx->active) {
-                        gf_msg ("glfs-mgmt", GF_LOG_ERROR, errno,
-                                API_MSG_REMOTE_HOST_CONN_FAILED,
-                                "failed to connect with remote-host: %s (%s)",
-                                ctx->cmd_args.volfile_server,
-                                strerror (errno));
-                        server = ctx->cmd_args.curr_server;
-                        if (server->list.next == &ctx->cmd_args.volfile_servers) {
-                                errno = ENOTCONN;
-                                gf_msg ("glfs-mgmt", GF_LOG_INFO, ENOTCONN,
-                                        API_MSG_VOLFILE_SERVER_EXHAUST,
-                                        "Exhausted all volfile servers");
-                                glfs_init_done (fs, -1);
-                                break;
-                        }
-                        server = list_entry (server->list.next, typeof(*server),
-                                             list);
-                        ctx->cmd_args.curr_server = server;
-                        ctx->cmd_args.volfile_server_port = server->port;
-                        ctx->cmd_args.volfile_server = server->volfile_server;
-                        ctx->cmd_args.volfile_server_transport = server->transport;
-
-                        ret = dict_set_int32 (rpc_trans->options,
-                                              "remote-port",
-                                              server->port);
-                        if (ret != 0) {
-                                gf_msg ("glfs-mgmt", GF_LOG_ERROR, ENOTCONN,
-                                        API_MSG_DICT_SET_FAILED,
-                                        "failed to set remote-port: %d",
-                                        server->port);
-                                errno = ENOTCONN;
-                                glfs_init_done (fs, -1);
-                                break;
-                        }
-
-                        ret = dict_set_str (rpc_trans->options,
-                                            "remote-host",
-                                            server->volfile_server);
-                        if (ret != 0) {
-                                gf_msg ("glfs-mgmt", GF_LOG_ERROR, ENOTCONN,
-                                        API_MSG_DICT_SET_FAILED,
-                                        "failed to set remote-host: %s",
-                                        server->volfile_server);
-                                errno = ENOTCONN;
-                                glfs_init_done (fs, -1);
-                                break;
-                        }
-
-                        ret = dict_set_str (rpc_trans->options,
-                                            "transport-type",
-                                            server->transport);
-                        if (ret != 0) {
-                                gf_msg ("glfs-mgmt", GF_LOG_ERROR, ENOTCONN,
-                                        API_MSG_DICT_SET_FAILED,
-                                        "failed to set transport-type: %s",
-                                        server->transport);
-                                errno = ENOTCONN;
-                                glfs_init_done (fs, -1);
-                                break;
-                        }
-                        gf_msg ("glfs-mgmt", GF_LOG_INFO, 0,
-                                API_MSG_VOLFILE_CONNECTING,
-                                "connecting to next volfile server %s"
-                                " at port %d with transport: %s",
-                                server->volfile_server, server->port,
-                                server->transport);
+    xlator_t *this = NULL;
+    glusterfs_ctx_t *ctx = NULL;
+    server_cmdline_t *server = NULL;
+    rpc_transport_t *rpc_trans = NULL;
+    struct glfs *fs = NULL;
+    int ret = 0;
+    struct dnscache6 *dnscache = NULL;
+
+    this = mydata;
+    rpc_trans = rpc->conn.trans;
+
+    ctx = this->ctx;
+    if (!ctx)
+        goto out;
+
+    fs = ((xlator_t *)ctx->master)->private;
+
+    switch (event) {
+        case RPC_CLNT_DISCONNECT:
+            if (!ctx->active) {
+                if (rpc_trans->connect_failed)
+                    gf_smsg("glfs-mgmt", GF_LOG_ERROR, 0,
+                            API_MSG_REMOTE_HOST_CONN_FAILED, "server=%s",
+                            ctx->cmd_args.volfile_server, NULL);
+                else
+                    gf_smsg("glfs-mgmt", GF_LOG_INFO, 0,
+                            API_MSG_REMOTE_HOST_CONN_FAILED, "server=%s",
+                            ctx->cmd_args.volfile_server, NULL);
+
+                if (!rpc->disabled) {
+                    /*
+                     * Check if dnscache is exhausted for current server
+                     * and continue until cache is exhausted
+                     */
+                    dnscache = rpc_trans->dnscache;
+                    if (dnscache && dnscache->next) {
+                        break;
+                    }
+                }
+                server = ctx->cmd_args.curr_server;
+                if (server->list.next == &ctx->cmd_args.volfile_servers) {
+                    errno = ENOTCONN;
+                    gf_smsg("glfs-mgmt", GF_LOG_INFO, ENOTCONN,
+                            API_MSG_VOLFILE_SERVER_EXHAUST, NULL);
+                    glfs_init_done(fs, -1);
+                    break;
                 }
-                break;
-	case RPC_CLNT_CONNECT:
-		rpc_clnt_set_connected (&((struct rpc_clnt*)ctx->mgmt)->conn);
-
-		ret = glfs_volfile_fetch (fs);
-		if (ret && (ctx->active == NULL)) {
-			/* Do it only for the first time */
-			/* Exit the process.. there are some wrong options */
-			gf_msg ("glfs-mgmt", GF_LOG_ERROR, EINVAL,
-                                API_MSG_INVALID_ENTRY,
-				"failed to fetch volume file (key:%s)",
-                                ctx->cmd_args.volfile_id);
-                        errno = EINVAL;
-                        glfs_init_done (fs, -1);
+                server = list_entry(server->list.next, typeof(*server), list);
+                ctx->cmd_args.curr_server = server;
+                ctx->cmd_args.volfile_server_port = server->port;
+                ctx->cmd_args.volfile_server = server->volfile_server;
+                ctx->cmd_args.volfile_server_transport = server->transport;
+
+                ret = dict_set_str(rpc_trans->options, "transport-type",
+                                   server->transport);
+                if (ret != 0) {
+                    gf_smsg("glfs-mgmt", GF_LOG_ERROR, ENOTCONN,
+                            API_MSG_DICT_SET_FAILED, "transport-type=%s",
+                            server->transport, NULL);
+                    errno = ENOTCONN;
+                    glfs_init_done(fs, -1);
+                    break;
                 }
 
-                break;
-	default:
-		break;
-	}
+                if (strcmp(server->transport, "unix") == 0) {
+                    ret = dict_set_str(rpc_trans->options,
+                                       "transport.socket.connect-path",
+                                       server->volfile_server);
+                    if (ret != 0) {
+                        gf_smsg("glfs-mgmt", GF_LOG_ERROR, ENOTCONN,
+                                API_MSG_DICT_SET_FAILED,
+                                "socket.connect-path=%s",
+                                server->volfile_server, NULL);
+                        errno = ENOTCONN;
+                        glfs_init_done(fs, -1);
+                        break;
+                    }
+                    /* delete the remote-host and remote-port keys
+                     * in case they were set while looping through
+                     * list of volfile servers previously
+                     */
+                    dict_del(rpc_trans->options, "remote-host");
+                    dict_del(rpc_trans->options, "remote-port");
+                } else {
+                    ret = dict_set_int32(rpc_trans->options, "remote-port",
+                                         server->port);
+                    if (ret != 0) {
+                        gf_smsg("glfs-mgmt", GF_LOG_ERROR, ENOTCONN,
+                                API_MSG_DICT_SET_FAILED, "remote-port=%d",
+                                server->port, NULL);
+                        errno = ENOTCONN;
+                        glfs_init_done(fs, -1);
+                        break;
+                    }
+
+                    ret = dict_set_str(rpc_trans->options, "remote-host",
+                                       server->volfile_server);
+                    if (ret != 0) {
+                        gf_smsg("glfs-mgmt", GF_LOG_ERROR, ENOTCONN,
+                                API_MSG_DICT_SET_FAILED, "remote-host=%s",
+                                server->volfile_server, NULL);
+                        errno = ENOTCONN;
+                        glfs_init_done(fs, -1);
+                        break;
+                    }
+                    /* delete the "transport.socket.connect-path"
+                     * key in case if it was set while looping
+                     * through list of volfile servers previously
+                     */
+                    dict_del(rpc_trans->options,
+                             "transport.socket.connect-path");
+                }
+
+                gf_smsg("glfs-mgmt", GF_LOG_INFO, 0, API_MSG_VOLFILE_CONNECTING,
+                        "server=%s", server->volfile_server, "port=%d",
+                        server->port, "transport=%s", server->transport, NULL);
+            }
+            break;
+        case RPC_CLNT_CONNECT:
+            ret = glfs_volfile_fetch(fs);
+            if (ret && (ctx->active == NULL)) {
+                /* Do it only for the first time */
+                /* Exit the process.. there are some wrong options */
+                gf_smsg("glfs-mgmt", GF_LOG_ERROR, EINVAL,
+                        API_MSG_GET_VOLFILE_FAILED, "key=%s",
+                        ctx->cmd_args.volfile_id, NULL);
+                errno = EINVAL;
+                glfs_init_done(fs, -1);
+            }
+
+            break;
+        default:
+            break;
+    }
 out:
-	return 0;
+    return 0;
 }
 
-
 int
-glusterfs_mgmt_notify (int32_t op, void *data, ...)
+glusterfs_mgmt_notify(int32_t op, void *data, ...)
 {
-	int ret = 0;
+    int ret = 0;
 
-	switch (op)
-	{
-		case GF_EN_DEFRAG_STATUS:
-			break;
+    switch (op) {
+        case GF_EN_DEFRAG_STATUS:
+            break;
 
-		default:
-			break;
-	}
+        default:
+            break;
+    }
 
-	return ret;
+    return ret;
 }
 
-
 int
-glfs_mgmt_init (struct glfs *fs)
+glfs_mgmt_init(struct glfs *fs)
 {
-	cmd_args_t		*cmd_args = NULL;
-	struct rpc_clnt		*rpc = NULL;
-	dict_t			*options = NULL;
-	int			ret = -1;
-	int			port = GF_DEFAULT_BASE_PORT;
-	char			*host = NULL;
-	glusterfs_ctx_t		*ctx = NULL;
-
-	ctx = fs->ctx;
-	cmd_args = &ctx->cmd_args;
-
-	if (ctx->mgmt)
-		return 0;
-
-	if (cmd_args->volfile_server_port)
-		port = cmd_args->volfile_server_port;
-
-	if (cmd_args->volfile_server) {
-		host = cmd_args->volfile_server;
-        } else if (cmd_args->volfile_server_transport &&
-                   !strcmp (cmd_args->volfile_server_transport, "unix")) {
-                host = DEFAULT_GLUSTERD_SOCKFILE;
-        } else {
-                host = "localhost";
-        }
-
-        if (!strcmp (cmd_args->volfile_server_transport, "unix")) {
-                ret = rpc_transport_unix_options_build (&options, host, 0);
-        } else {
-                ret = rpc_transport_inet_options_build (&options, host, port);
-        }
-
-	if (ret)
-		goto out;
-
-	rpc = rpc_clnt_new (options, THIS, THIS->name, 8);
-	if (!rpc) {
-		ret = -1;
-		gf_msg (THIS->name, GF_LOG_WARNING, 0,
-                        API_MSG_CREATE_RPC_CLIENT_FAILED,
-			"failed to create rpc clnt");
-		goto out;
-	}
-
-	ret = rpc_clnt_register_notify (rpc, mgmt_rpc_notify, THIS);
-	if (ret) {
-		gf_msg (THIS->name, GF_LOG_WARNING, 0,
-                        API_MSG_REG_NOTIFY_FUNC_FAILED,
-			"failed to register notify function");
-		goto out;
-	}
-
-	ret = rpcclnt_cbk_program_register (rpc, &mgmt_cbk_prog, THIS);
-	if (ret) {
-		gf_msg (THIS->name, GF_LOG_WARNING, 0,
-                        API_MSG_REG_CBK_FUNC_FAILED,
-			"failed to register callback function");
-		goto out;
-	}
-
-	ctx->notify = glusterfs_mgmt_notify;
-
-	/* This value should be set before doing the 'rpc_clnt_start()' as
-	   the notify function uses this variable */
-	ctx->mgmt = rpc;
-
-	ret = rpc_clnt_start (rpc);
+    cmd_args_t *cmd_args = NULL;
+    struct rpc_clnt *rpc = NULL;
+    dict_t *options = NULL;
+    int ret = -1;
+    int port = GF_DEFAULT_BASE_PORT;
+    char *host = NULL;
+    glusterfs_ctx_t *ctx = NULL;
+
+    ctx = fs->ctx;
+    cmd_args = &ctx->cmd_args;
+
+    if (ctx->mgmt)
+        return 0;
+
+    options = dict_new();
+    if (!options)
+        goto out;
+
+    if (cmd_args->volfile_server_port)
+        port = cmd_args->volfile_server_port;
+
+    if (cmd_args->volfile_server) {
+        host = cmd_args->volfile_server;
+    } else if (cmd_args->volfile_server_transport &&
+               !strcmp(cmd_args->volfile_server_transport, "unix")) {
+        host = DEFAULT_GLUSTERD_SOCKFILE;
+    } else {
+        host = "localhost";
+    }
+
+    if (cmd_args->volfile_server_transport &&
+        !strcmp(cmd_args->volfile_server_transport, "unix")) {
+        ret = rpc_transport_unix_options_build(options, host, 0);
+    } else {
+        xlator_cmdline_option_t *opt = find_xlator_option_in_cmd_args_t(
+            "address-family", cmd_args);
+        ret = rpc_transport_inet_options_build(options, host, port,
+                                               (opt ? opt->value : NULL));
+    }
+
+    if (ret)
+        goto out;
+
+    rpc = rpc_clnt_new(options, THIS, THIS->name, 8);
+    if (!rpc) {
+        ret = -1;
+        gf_smsg(THIS->name, GF_LOG_WARNING, 0, API_MSG_CREATE_RPC_CLIENT_FAILED,
+                NULL);
+        goto out;
+    }
+
+    ret = rpc_clnt_register_notify(rpc, mgmt_rpc_notify, THIS);
+    if (ret) {
+        gf_smsg(THIS->name, GF_LOG_WARNING, 0, API_MSG_REG_NOTIFY_FUNC_FAILED,
+                NULL);
+        goto out;
+    }
+
+    ret = rpcclnt_cbk_program_register(rpc, &mgmt_cbk_prog, THIS);
+    if (ret) {
+        gf_smsg(THIS->name, GF_LOG_WARNING, 0, API_MSG_REG_CBK_FUNC_FAILED,
+                NULL);
+        goto out;
+    }
+
+    ctx->notify = glusterfs_mgmt_notify;
+
+    /* This value should be set before doing the 'rpc_clnt_start()' as
+       the notify function uses this variable */
+    ctx->mgmt = rpc;
+
+    ret = rpc_clnt_start(rpc);
 out:
-	return ret;
+    if (options)
+        dict_unref(options);
+    return ret;
 }
diff --git a/api/src/glfs-resolve.c b/api/src/glfs-resolve.c
index 65fe590144d..8a393ecb464 100644
--- a/api/src/glfs-resolve.c
+++ b/api/src/glfs-resolve.c
@@ -1,5 +1,5 @@
 /*
-  Copyright (c) 2012 Red Hat, Inc. <http://www.redhat.com>
+  Copyright (c) 2012-2018 Red Hat, Inc. <http://www.redhat.com>
   This file is part of GlusterFS.
 
   This file is licensed to you under your choice of the GNU Lesser
@@ -8,7 +8,6 @@
   cases as published by the Free Software Foundation.
 */
 
-
 #include <unistd.h>
 #include <string.h>
 #include <stdlib.h>
@@ -16,59 +15,61 @@
 #include <inttypes.h>
 #include <limits.h>
 
-#include "glusterfs.h"
-#include "logging.h"
-#include "stack.h"
-#include "event.h"
+#include <glusterfs/glusterfs.h>
+#include <glusterfs/logging.h>
+#include <glusterfs/stack.h>
+#include <glusterfs/gf-event.h>
 #include "glfs-mem-types.h"
-#include "common-utils.h"
-#include "syncop.h"
-#include "call-stub.h"
+#include <glusterfs/common-utils.h>
+#include <glusterfs/syncop.h>
+#include <glusterfs/call-stub.h>
 #include "gfapi-messages.h"
-
+#include <glusterfs/inode.h>
 #include "glfs-internal.h"
 
-#define graphid_str(subvol) (uuid_utoa((unsigned char *)subvol->graph->graph_uuid))
-
-
+#define graphid_str(subvol)                                                    \
+    (uuid_utoa((unsigned char *)subvol->graph->graph_uuid))
 int
-glfs_first_lookup_safe (xlator_t *subvol)
+glfs_first_lookup_safe(xlator_t *subvol)
 {
-	loc_t  loc = {0, };
-	int    ret = -1;
+    loc_t loc = {
+        0,
+    };
+    int ret = -1;
 
-	loc.inode = subvol->itable->root;
-	memset (loc.gfid, 0, 16);
-	loc.gfid[15] = 1;
-	loc.path = "/";
-	loc.name = "";
+    loc.inode = subvol->itable->root;
+    memset(loc.gfid, 0, 16);
+    loc.gfid[15] = 1;
+    loc.path = "/";
+    loc.name = "";
 
-	ret = syncop_lookup (subvol, &loc, 0, 0, 0, 0);
-        DECODE_SYNCOP_ERR (ret);
+    ret = syncop_lookup(subvol, &loc, 0, 0, 0, 0);
+    DECODE_SYNCOP_ERR(ret);
 
-	gf_msg_debug (subvol->name, 0, "first lookup complete %d", ret);
+    gf_msg_debug(subvol->name, 0, "first lookup complete %d", ret);
 
-	return ret;
+    return ret;
 }
 
-
 int
-__glfs_first_lookup (struct glfs *fs, xlator_t *subvol)
+__glfs_first_lookup(struct glfs *fs, xlator_t *subvol)
 {
-	int ret = -1;
-
-	fs->migration_in_progress = 1;
-	pthread_mutex_unlock (&fs->mutex);
-	{
-		ret = glfs_first_lookup_safe (subvol);
-	}
-	pthread_mutex_lock (&fs->mutex);
-	fs->migration_in_progress = 0;
-	pthread_cond_broadcast (&fs->cond);
-
-	return ret;
-}
+    int ret = -1;
+
+    fs->migration_in_progress = 1;
+    pthread_mutex_unlock(&fs->mutex);
+    {
+        ret = glfs_first_lookup_safe(subvol);
+    }
+    pthread_mutex_lock(&fs->mutex);
+    fs->migration_in_progress = 0;
+    pthread_cond_broadcast(&fs->cond);
 
+    /* wake up other waiting tasks */
+    __GLFS_SYNCTASK_WAKE(fs);
+
+    return ret;
+}
 
 /**
  * We have to check if need_lookup flag is set in both old and the new inodes.
@@ -78,1006 +79,1121 @@ __glfs_first_lookup (struct glfs *fs, xlator_t *subvol)
  * below xlators can set their respective contexts.
  */
 inode_t *
-glfs_refresh_inode_safe (xlator_t *subvol, inode_t *oldinode,
-                         gf_boolean_t need_lookup)
+glfs_refresh_inode_safe(xlator_t *subvol, inode_t *oldinode,
+                        gf_boolean_t need_lookup)
 {
-	loc_t        loc = {0, };
-	int          ret = -1;
-	struct iatt  iatt = {0, };
-	inode_t     *newinode = NULL;
-        gf_boolean_t lookup_needed = _gf_false;
-
-
-	if (!oldinode)
-		return NULL;
-
-	if (!need_lookup && oldinode->table->xl == subvol)
-		return inode_ref (oldinode);
-
-	newinode = inode_find (subvol->itable, oldinode->gfid);
-	if (!need_lookup && newinode) {
-
-                lookup_needed = inode_needs_lookup (newinode, THIS);
-                if (!lookup_needed)
-                        return newinode;
-        }
-
-	gf_uuid_copy (loc.gfid, oldinode->gfid);
-        if (!newinode)
-                loc.inode = inode_new (subvol->itable);
-        else
-                loc.inode = newinode;
-
-	if (!loc.inode)
-		return NULL;
-
-	ret = syncop_lookup (subvol, &loc, &iatt, 0, 0, 0);
-        DECODE_SYNCOP_ERR (ret);
-
-	if (ret) {
-		gf_msg (subvol->name, GF_LOG_WARNING, errno,
-                        API_MSG_INODE_REFRESH_FAILED,
-			"inode refresh of %s failed: %s",
-			uuid_utoa (oldinode->gfid), strerror (errno));
-		loc_wipe (&loc);
-		return NULL;
-	}
-
-	newinode = inode_link (loc.inode, 0, 0, &iatt);
-	if (newinode)
-		inode_lookup (newinode);
-
-	loc_wipe (&loc);
-
-	return newinode;
+    loc_t loc = {
+        0,
+    };
+    int ret = -1;
+    struct iatt iatt = {
+        0,
+    };
+    inode_t *newinode = NULL;
+    gf_boolean_t lookup_needed = _gf_false;
+    uint64_t ctx_value = LOOKUP_NOT_NEEDED;
+
+    if (!oldinode)
+        return NULL;
+
+    if (!need_lookup && oldinode->table->xl == subvol)
+        return inode_ref(oldinode);
+
+    newinode = inode_find(subvol->itable, oldinode->gfid);
+    if (!need_lookup && newinode) {
+        lookup_needed = inode_needs_lookup(newinode, THIS);
+        if (!lookup_needed)
+            return newinode;
+    }
+
+    gf_uuid_copy(loc.gfid, oldinode->gfid);
+    if (!newinode)
+        loc.inode = inode_new(subvol->itable);
+    else
+        loc.inode = newinode;
+
+    if (!loc.inode)
+        return NULL;
+
+    ret = syncop_lookup(subvol, &loc, &iatt, 0, 0, 0);
+    DECODE_SYNCOP_ERR(ret);
+
+    if (ret) {
+        gf_smsg(subvol->name, GF_LOG_WARNING, errno,
+                API_MSG_INODE_REFRESH_FAILED, "gfid=%s",
+                uuid_utoa(oldinode->gfid), "err=%s", strerror(errno), NULL);
+        loc_wipe(&loc);
+        return NULL;
+    }
+
+    newinode = inode_link(loc.inode, 0, 0, &iatt);
+    if (newinode) {
+        if (newinode == loc.inode)
+            inode_ctx_set(newinode, THIS, &ctx_value);
+        inode_lookup(newinode);
+    } else {
+        gf_smsg(subvol->name, GF_LOG_WARNING, errno, API_MSG_INODE_LINK_FAILED,
+                "gfid=%s", uuid_utoa((unsigned char *)&iatt.ia_gfid), NULL);
+    }
+
+    loc_wipe(&loc);
+
+    return newinode;
 }
 
-
 inode_t *
-__glfs_refresh_inode (struct glfs *fs, xlator_t *subvol, inode_t *inode,
-                      gf_boolean_t need_lookup)
+__glfs_refresh_inode(struct glfs *fs, xlator_t *subvol, inode_t *inode,
+                     gf_boolean_t need_lookup)
 {
-	inode_t *newinode = NULL;
-
-	fs->migration_in_progress = 1;
-	pthread_mutex_unlock (&fs->mutex);
-	{
-		newinode = glfs_refresh_inode_safe (subvol, inode, need_lookup);
-	}
-	pthread_mutex_lock (&fs->mutex);
-	fs->migration_in_progress = 0;
-	pthread_cond_broadcast (&fs->cond);
-
-	return newinode;
+    inode_t *newinode = NULL;
+
+    fs->migration_in_progress = 1;
+    pthread_mutex_unlock(&fs->mutex);
+    {
+        newinode = glfs_refresh_inode_safe(subvol, inode, need_lookup);
+    }
+    pthread_mutex_lock(&fs->mutex);
+    fs->migration_in_progress = 0;
+    pthread_cond_broadcast(&fs->cond);
+
+    /* wake up other waiting tasks */
+    __GLFS_SYNCTASK_WAKE(fs);
+
+    return newinode;
 }
 
+GFAPI_SYMVER_PRIVATE_DEFAULT(glfs_loc_touchup, 3.4.0)
 int
-priv_glfs_loc_touchup (loc_t *loc)
+priv_glfs_loc_touchup(loc_t *loc)
 {
-        int     ret = 0;
+    int ret = 0;
 
-        ret = loc_touchup (loc, loc->name);
-        if (ret < 0) {
-                errno = -ret;
-                ret = -1;
-        }
+    ret = loc_touchup(loc, loc->name);
+    if (ret < 0) {
+        errno = -ret;
+        ret = -1;
+    }
 
-        return ret;
+    return ret;
 }
 
-GFAPI_SYMVER_PRIVATE_DEFAULT(glfs_loc_touchup, 3.4.0);
-
 int
-glfs_resolve_symlink (struct glfs *fs, xlator_t *subvol, inode_t *inode,
-		      char **lpath)
+glfs_resolve_symlink(struct glfs *fs, xlator_t *subvol, inode_t *inode,
+                     char **lpath)
 {
-	loc_t  loc = {0, };
-	char  *path = NULL;
-	char  *rpath = NULL;
-	int    ret = -1;
-
-	loc.inode = inode_ref (inode);
-	gf_uuid_copy (loc.gfid, inode->gfid);
-	ret = inode_path (inode, NULL, &rpath);
-	if (ret < 0)
-		goto out;
-	loc.path = rpath;
-
-	ret = syncop_readlink (subvol, &loc, &path, 4096, NULL, NULL);
-        DECODE_SYNCOP_ERR (ret);
-
-	if (ret < 0)
-		goto out;
-
-	if (lpath)
-		*lpath = path;
+    loc_t loc = {
+        0,
+    };
+    char *path = NULL;
+    char *rpath = NULL;
+    int ret = -1;
+
+    loc.inode = inode_ref(inode);
+    gf_uuid_copy(loc.gfid, inode->gfid);
+    ret = inode_path(inode, NULL, &rpath);
+    if (ret < 0)
+        goto out;
+    loc.path = rpath;
+
+    ret = syncop_readlink(subvol, &loc, &path, 4096, NULL, NULL);
+    DECODE_SYNCOP_ERR(ret);
+
+    if (ret < 0)
+        goto out;
+
+    if (lpath)
+        *lpath = path;
 out:
-	loc_wipe (&loc);
-	return ret;
+    loc_wipe(&loc);
+    return ret;
 }
 
-
 int
-glfs_resolve_base (struct glfs *fs, xlator_t *subvol, inode_t *inode,
-		   struct iatt *iatt)
+glfs_resolve_base(struct glfs *fs, xlator_t *subvol, inode_t *inode,
+                  struct iatt *iatt)
 {
-	loc_t       loc = {0, };
-	int         ret = -1;
-	char       *path = NULL;
-
-	loc.inode = inode_ref (inode);
-	gf_uuid_copy (loc.gfid, inode->gfid);
-
-	ret = inode_path (loc.inode, NULL, &path);
-	loc.path = path;
-	if (ret < 0)
-		goto out;
-
-	ret = syncop_lookup (subvol, &loc, iatt, NULL, NULL, NULL);
-        DECODE_SYNCOP_ERR (ret);
+    loc_t loc = {
+        0,
+    };
+    int ret = -1;
+    char *path = NULL;
+
+    loc.inode = inode_ref(inode);
+    gf_uuid_copy(loc.gfid, inode->gfid);
+
+    ret = inode_path(loc.inode, NULL, &path);
+    loc.path = path;
+    if (ret < 0)
+        goto out;
+
+    ret = syncop_lookup(subvol, &loc, iatt, NULL, NULL, NULL);
+    DECODE_SYNCOP_ERR(ret);
 out:
-	loc_wipe (&loc);
+    loc_wipe(&loc);
 
-	return ret;
+    return ret;
 }
+/*
+ * This function can be used to call named lookup on root.
+ * If you use glfs_resolve_base, that will be a nameless lookup.
+ */
+static int
+glfs_resolve_root(struct glfs *fs, xlator_t *subvol, inode_t *inode,
+                  struct iatt *iatt)
+{
+    loc_t loc = {
+        0,
+    };
+    int ret = -1;
+    char *path = NULL;
+
+    loc.inode = inode_ref(inode);
+
+    ret = inode_path(loc.inode, ".", &path);
+    loc.path = path;
+    loc.name = ".";
+    /* Having a value in loc.name will help to bypass md-cache check for
+     * nameless lookup.
+     * TODO: Re-visit on nameless lookup and md-cache.
+     * Github issue : https://github.com/gluster/glusterfs/issues/232
+     */
+    loc.parent = inode_ref(inode);
+    if (ret < 0)
+        goto out;
+
+    ret = syncop_lookup(subvol, &loc, iatt, NULL, NULL, NULL);
+    DECODE_SYNCOP_ERR(ret);
+out:
+    loc_wipe(&loc);
 
+    return ret;
+}
 
 inode_t *
-glfs_resolve_component (struct glfs *fs, xlator_t *subvol, inode_t *parent,
-			const char *component, struct iatt *iatt,
-			int force_lookup)
+glfs_resolve_component(struct glfs *fs, xlator_t *subvol, inode_t *parent,
+                       const char *component, struct iatt *iatt,
+                       int force_lookup)
 {
-	loc_t        loc = {0, };
-	inode_t     *inode = NULL;
-	int          reval = 0;
-	int          ret = -1;
-	int          glret = -1;
-	struct iatt  ciatt = {0, };
-	uuid_t       gfid;
-	dict_t      *xattr_req = NULL;
-
-	loc.name = component;
-
-	loc.parent = inode_ref (parent);
-	gf_uuid_copy (loc.pargfid, parent->gfid);
-
-        /* /.. and /. should point back to /
-           we lookup using inode and gfid of root
-           Fill loc.name so that we make use md-cache.
-           md-cache is not valid for nameless lookups.
-        */
-        if (__is_root_gfid (parent->gfid) &&
-            (strcmp (component, "..") == 0)) {
-                loc.inode = inode_ref (parent);
-                loc.name = ".";
+    loc_t loc = {
+        0,
+    };
+    inode_t *inode = NULL;
+    inode_t *temp_parent = NULL;
+    int reval = 0;
+    int ret = -1;
+    int glret = -1;
+    struct iatt ciatt = {
+        0,
+    };
+    uuid_t gfid;
+    dict_t *xattr_req = NULL;
+    uint64_t ctx_value = LOOKUP_NOT_NEEDED;
+
+    loc.parent = inode_ref(parent);
+    gf_uuid_copy(loc.pargfid, parent->gfid);
+
+    if (__is_root_gfid(parent->gfid) &&
+        ((strcmp(component, ".") == 0) || (strcmp(component, "..") == 0) ||
+         (strcmp(component, "") == 0))) {
+        if (!force_lookup) {
+            inode = inode_ref(parent);
         } else {
-                if (strcmp (component, ".") == 0)
-                        loc.inode = inode_ref (parent);
-                else if (strcmp (component, "..") == 0)
-                        loc.inode = inode_parent (parent, 0, 0);
-                else
-                        loc.inode = inode_grep (parent->table, parent,
-                                                component);
+            ret = glfs_resolve_root(fs, subvol, parent, &ciatt);
+            if (!ret)
+                inode = inode_ref(parent);
+        }
+        goto found;
+    }
+    /* *
+     * if the component name is either "." or "..", it will try to
+     * resolve that if inode has a proper parent (named lookup).
+     *
+     * Below condition works like this
+     *
+     * Example 1 :
+     * Path /out_dir/dir/in_dir/.
+     *     In put values :
+     *         parent = in_dir
+     *         component : "."
+     *
+     *      Out put values:
+     *         parent : dir
+     *         component : "in_dir"
+     *
+     * Example 2 :
+     * Path /out_dir/dir/in_dir/..
+     *     In put values :
+     *         parent = in_dir
+     *         component : ".."
+     *
+     *     Out put values:
+     *         parent : output_dir
+     *         component : "dir"
+     *
+     * In case of nameless lookup, both "." and ".." retained
+     */
+
+    if (strcmp(component, ".") == 0) {
+        loc.inode = inode_ref(parent);
+        temp_parent = inode_parent(loc.inode, 0, 0);
+        if (temp_parent) {
+            inode_unref(loc.parent);
+            loc.parent = temp_parent;
+            gf_uuid_copy(loc.pargfid, temp_parent->gfid);
+            inode_find_directory_name(loc.inode, &loc.name);
         }
 
+    } else if (strcmp(component, "..") == 0) {
+        loc.inode = inode_parent(parent, 0, 0);
+        if (loc.inode) {
+            temp_parent = inode_parent(loc.inode, 0, 0);
+            if (temp_parent) {
+                inode_unref(loc.parent);
+                loc.parent = temp_parent;
+                gf_uuid_copy(loc.pargfid, temp_parent->gfid);
+                inode_find_directory_name(loc.inode, &loc.name);
+            } else if (__is_root_gfid(loc.inode->gfid)) {
+                inode_unref(loc.parent);
+                loc.parent = inode_ref(loc.inode);
+                gf_uuid_copy(loc.pargfid, loc.inode->gfid);
+                loc.name = ".";
+            } else {
+                inode_unref(loc.inode);
+                loc.inode = NULL;
+            }
+        }
+    } else
+        loc.inode = inode_grep(parent->table, parent, component);
 
-	if (loc.inode) {
-		gf_uuid_copy (loc.gfid, loc.inode->gfid);
-		reval = 1;
-
-		if (!force_lookup) {
-			inode = inode_ref (loc.inode);
-			ciatt.ia_type = inode->ia_type;
-			goto found;
-		}
-	} else {
-		gf_uuid_generate (gfid);
-		loc.inode = inode_new (parent->table);
-                if (!loc.inode) {
-                        errno = ENOMEM;
-                        goto out;
-                }
+    if (!loc.name)
+        loc.name = component;
 
-                xattr_req = dict_new ();
-                if (!xattr_req) {
-                        errno = ENOMEM;
-                        goto out;
-                }
+    if (loc.inode) {
+        gf_uuid_copy(loc.gfid, loc.inode->gfid);
+        reval = 1;
 
-                ret = dict_set_static_bin (xattr_req, "gfid-req", gfid, 16);
-                if (ret) {
-                        errno = ENOMEM;
-                        goto out;
-                }
+        if (!(force_lookup || inode_needs_lookup(loc.inode, THIS))) {
+            inode = inode_ref(loc.inode);
+            goto found;
+        }
+    } else {
+        gf_uuid_generate(gfid);
+        loc.inode = inode_new(parent->table);
+        if (!loc.inode) {
+            errno = ENOMEM;
+            goto out;
+        }
+
+        xattr_req = dict_new();
+        if (!xattr_req) {
+            errno = ENOMEM;
+            goto out;
+        }
+
+        ret = dict_set_gfuuid(xattr_req, "gfid-req", gfid, true);
+        if (ret) {
+            errno = ENOMEM;
+            goto out;
+        }
+    }
+
+    glret = priv_glfs_loc_touchup(&loc);
+    if (glret < 0) {
+        ret = -1;
+        goto out;
+    }
+
+    ret = syncop_lookup(subvol, &loc, &ciatt, NULL, xattr_req, NULL);
+    if (ret && reval) {
+        /*
+         * A stale mapping might exist for a dentry/inode that has been
+         * removed from another client.
+         */
+        if (-ret == ENOENT) {
+            inode_unlink(loc.inode, loc.parent, loc.name);
+            if (!inode_has_dentry(loc.inode))
+                inode_forget(loc.inode, 0);
+        }
+
+        inode_unref(loc.inode);
+        gf_uuid_clear(loc.gfid);
+        loc.inode = inode_new(parent->table);
+        if (!loc.inode) {
+            errno = ENOMEM;
+            goto out;
+        }
 
-	}
-
-	glret = priv_glfs_loc_touchup (&loc);
-	if (glret < 0) {
-		ret = -1;
-		goto out;
-	}
-
-        ret = syncop_lookup (subvol, &loc, &ciatt, NULL, xattr_req, NULL);
-        if (ret && reval) {
-                /*
-                 * A stale mapping might exist for a dentry/inode that has been
-                 * removed from another client.
-                 */
-                if (-ret == ENOENT)
-                        inode_unlink(loc.inode, loc.parent,
-                                     loc.name);
-		inode_unref (loc.inode);
-		loc.inode = inode_new (parent->table);
-		if (!loc.inode) {
-			errno = ENOMEM;
-			goto out;
-		}
-
-		xattr_req = dict_new ();
-		if (!xattr_req) {
-			errno = ENOMEM;
-			goto out;
-		}
-
-		gf_uuid_generate (gfid);
-
-		ret = dict_set_static_bin (xattr_req, "gfid-req", gfid, 16);
-		if (ret) {
-			errno = ENOMEM;
-			goto out;
-		}
-
-		ret = syncop_lookup (subvol, &loc, &ciatt, NULL,
-				     xattr_req, NULL);
-	}
-        DECODE_SYNCOP_ERR (ret);
-	if (ret)
-		goto out;
-
-	inode = inode_link (loc.inode, loc.parent, component, &ciatt);
+        xattr_req = dict_new();
+        if (!xattr_req) {
+            errno = ENOMEM;
+            goto out;
+        }
+
+        gf_uuid_generate(gfid);
+
+        ret = dict_set_gfuuid(xattr_req, "gfid-req", gfid, true);
+        if (ret) {
+            errno = ENOMEM;
+            goto out;
+        }
+
+        ret = syncop_lookup(subvol, &loc, &ciatt, NULL, xattr_req, NULL);
+    }
+    DECODE_SYNCOP_ERR(ret);
+    if (ret)
+        goto out;
+
+    inode = inode_link(loc.inode, loc.parent, component, &ciatt);
+
+    if (!inode) {
+        gf_smsg(subvol->name, GF_LOG_WARNING, errno, API_MSG_INODE_LINK_FAILED,
+                "gfid=%s", uuid_utoa((unsigned char *)&ciatt.ia_gfid), NULL);
+        goto out;
+    } else if (inode == loc.inode)
+        inode_ctx_set(inode, THIS, &ctx_value);
 found:
-	if (inode)
-		inode_lookup (inode);
-	if (iatt)
-		*iatt = ciatt;
+    if (inode) {
+        ciatt.ia_type = inode->ia_type;
+        inode_lookup(inode);
+    }
+    if (iatt)
+        *iatt = ciatt;
 out:
-	if (xattr_req)
-		dict_unref (xattr_req);
+    if (xattr_req)
+        dict_unref(xattr_req);
+    loc_wipe(&loc);
 
-	loc_wipe (&loc);
-
-	return inode;
+    return inode;
 }
 
-
+GFAPI_SYMVER_PRIVATE_DEFAULT(glfs_resolve_at, 3.4.0)
 int
-priv_glfs_resolve_at (struct glfs *fs, xlator_t *subvol, inode_t *at,
-		 const char *origpath, loc_t *loc, struct iatt *iatt,
-		 int follow, int reval)
+priv_glfs_resolve_at(struct glfs *fs, xlator_t *subvol, inode_t *at,
+                     const char *origpath, loc_t *loc, struct iatt *iatt,
+                     int follow, int reval)
 {
-	inode_t    *inode = NULL;
-	inode_t    *parent = NULL;
-	char       *saveptr = NULL;
-	char       *path = NULL;
-	char       *component = NULL;
-	char       *next_component = NULL;
-	int         ret = -1;
-	struct iatt ciatt = {0, };
-
-	path = gf_strdup (origpath);
-	if (!path) {
-		errno = ENOMEM;
-		return -1;
-	}
-
-	parent = NULL;
-	if (at && path[0] != '/') {
-		/* A relative resolution of a path which starts with '/'
-		   is equal to an absolute path resolution.
-		*/
-		inode = inode_ref (at);
-	} else {
-		inode = inode_ref (subvol->itable->root);
-
-		if (strcmp (path, "/") == 0)
-			glfs_resolve_base (fs, subvol, inode, &ciatt);
-	}
-
-	for (component = strtok_r (path, "/", &saveptr);
-	     component; component = next_component) {
-
-		next_component = strtok_r (NULL, "/", &saveptr);
-
-		if (parent)
-			inode_unref (parent);
-
-		parent = inode;
-
-		inode = glfs_resolve_component (fs, subvol, parent,
-						component, &ciatt,
-						/* force hard lookup on the last
-						   component, as the caller
-						   wants proper iatt filled
-						*/
-						(reval || (!next_component &&
-						iatt)));
-		if (!inode) {
-                        ret = -1;
-			break;
-                }
+    inode_t *inode = NULL;
+    inode_t *parent = NULL;
+    char *saveptr = NULL;
+    char *path = NULL;
+    char *component = NULL;
+    char *next_component = NULL;
+    int ret = -1;
+    struct iatt ciatt = {
+        0,
+    };
+
+    DECLARE_OLD_THIS;
+    __GLFS_ENTRY_VALIDATE_FS(fs, invalid_fs);
+
+    if (origpath[0] == '\0') {
+        errno = EINVAL;
+        goto invalid_fs;
+    }
+
+    parent = NULL;
+    if (at && origpath[0] != '/') {
+        /* A relative resolution of a path which starts with '/'
+           is equal to an absolute path resolution.
+        */
+        inode = inode_ref(at);
+    } else {
+        inode = inode_ref(subvol->itable->root);
+
+        if (strcmp(origpath, "/") == 0)
+            glfs_resolve_root(fs, subvol, inode, &ciatt);
+    }
+
+    path = gf_strdup(origpath);
+    if (!path)
+        goto invalid_fs;
+
+    for (component = strtok_r(path, "/", &saveptr); component;
+         component = next_component) {
+        next_component = strtok_r(NULL, "/", &saveptr);
+
+        if (parent)
+            inode_unref(parent);
+        parent = inode;
+        inode = glfs_resolve_component(fs, subvol, parent, component, &ciatt,
+                                       /* force hard lookup on the last
+                                          component, as the caller
+                                          wants proper iatt filled
+                                       */
+                                       (reval || (!next_component && iatt)));
+        if (!inode) {
+            ret = -1;
+            break;
+        }
 
-		if (IA_ISLNK (ciatt.ia_type) && (next_component || follow)) {
-			/* If the component is not the last piece,
-			   then following it is necessary even if
-			   not requested by the caller
-			*/
-			char *lpath = NULL;
-			loc_t sym_loc = {0,};
-
-			if (follow > GLFS_SYMLINK_MAX_FOLLOW) {
-				errno = ELOOP;
-				ret = -1;
-				if (inode) {
-					inode_unref (inode);
-					inode = NULL;
-				}
-				break;
-			}
-
-			ret = glfs_resolve_symlink (fs, subvol, inode, &lpath);
-			inode_unref (inode);
-			inode = NULL;
-			if (ret < 0)
-				break;
-
-			ret = priv_glfs_resolve_at (fs, subvol, parent, lpath,
-					       &sym_loc,
-					       /* followed iatt becomes the
-						  component iatt
-					       */
-					       &ciatt,
-					       /* always recurisvely follow while
-						  following symlink
-					       */
-					       follow + 1, reval);
-			if (ret == 0)
-				inode = inode_ref (sym_loc.inode);
-			loc_wipe (&sym_loc);
-			GF_FREE (lpath);
-		}
-
-		if (!next_component)
-			break;
-
-		if (!IA_ISDIR (ciatt.ia_type)) {
-			/* next_component exists and this component is
-			   not a directory
-			*/
-			inode_unref (inode);
-			inode = NULL;
-			ret = -1;
-			errno = ENOTDIR;
-			break;
-		}
-	}
-
-	if (parent && next_component)
-		/* resolution failed mid-way */
-		goto out;
-
-	/* At this point, all components up to the last parent directory
-	   have been resolved successfully (@parent). Resolution of basename
-	   might have failed (@inode) if at all.
-	*/
-
-	loc->parent = parent;
-	if (parent) {
-		gf_uuid_copy (loc->pargfid, parent->gfid);
-		loc->name = component;
-	}
-
-	loc->inode = inode;
-	if (inode) {
-		gf_uuid_copy (loc->gfid, inode->gfid);
-		if (iatt)
-			*iatt = ciatt;
-		ret = 0;
-	}
-
-        if (priv_glfs_loc_touchup (loc) < 0) {
+        if (IA_ISLNK(ciatt.ia_type) && (next_component || follow)) {
+            /* If the component is not the last piece,
+               then following it is necessary even if
+               not requested by the caller
+            */
+            char *lpath = NULL;
+            loc_t sym_loc = {
+                0,
+            };
+
+            if (follow > GLFS_SYMLINK_MAX_FOLLOW) {
+                errno = ELOOP;
                 ret = -1;
+                if (inode) {
+                    inode_unref(inode);
+                    inode = NULL;
+                }
+                break;
+            }
+
+            ret = glfs_resolve_symlink(fs, subvol, inode, &lpath);
+            inode_unref(inode);
+            inode = NULL;
+            if (ret < 0)
+                break;
+
+            ret = priv_glfs_resolve_at(fs, subvol, parent, lpath, &sym_loc,
+                                       /* followed iatt becomes the
+                                          component iatt
+                                       */
+                                       &ciatt,
+                                       /* always recurisvely follow while
+                                          following symlink
+                                       */
+                                       follow + 1, reval);
+            if (ret == 0)
+                inode = inode_ref(sym_loc.inode);
+            loc_wipe(&sym_loc);
+            GF_FREE(lpath);
         }
-out:
-	GF_FREE (path);
 
-	/* do NOT loc_wipe here as only last component might be missing */
+        if (!next_component)
+            break;
+
+        if (!IA_ISDIR(ciatt.ia_type)) {
+            /* next_component exists and this component is
+               not a directory
+            */
+            inode_unref(inode);
+            inode = NULL;
+            ret = -1;
+            errno = ENOTDIR;
+            break;
+        }
+    }
+
+    if (parent && next_component)
+        /* resolution failed mid-way */
+        goto out;
+
+    /* At this point, all components up to the last parent directory
+       have been resolved successfully (@parent). Resolution of basename
+       might have failed (@inode) if at all.
+    */
+
+    loc->parent = parent;
+    if (parent) {
+        gf_uuid_copy(loc->pargfid, parent->gfid);
+        loc->name = component;
+    }
+
+    loc->inode = inode;
+    if (inode) {
+        gf_uuid_copy(loc->gfid, inode->gfid);
+        if (iatt)
+            *iatt = ciatt;
+        ret = 0;
+    }
+
+    if (priv_glfs_loc_touchup(loc) < 0) {
+        ret = -1;
+    }
+out:
+    GF_FREE(path);
+    __GLFS_EXIT_FS;
 
-	return ret;
+    /* do NOT loc_wipe here as only last component might be missing */
+invalid_fs:
+    return ret;
 }
 
-GFAPI_SYMVER_PRIVATE_DEFAULT(glfs_resolve_at, 3.4.0);
-
-
 int
-glfs_resolve_path (struct glfs *fs, xlator_t *subvol, const char *origpath,
-		   loc_t *loc, struct iatt *iatt, int follow, int reval)
+glfs_resolve_path(struct glfs *fs, xlator_t *subvol, const char *origpath,
+                  loc_t *loc, struct iatt *iatt, int follow, int reval)
 {
-	int ret = -1;
-	inode_t *cwd = NULL;
-
-	if (origpath[0] == '/')
-		return priv_glfs_resolve_at (fs, subvol, NULL, origpath, loc,
-                                             iatt, follow, reval);
-
-	cwd = glfs_cwd_get (fs);
-        if (NULL == cwd) {
-                gf_msg (subvol->name, GF_LOG_WARNING, EIO,
-                        API_MSG_GET_CWD_FAILED, "Failed to get cwd");
-                errno = EIO;
-                goto out;
-        }
+    int ret = -1;
+    inode_t *cwd = NULL;
 
-	ret = priv_glfs_resolve_at (fs, subvol, cwd, origpath, loc, iatt,
+    if (origpath[0] == '/')
+        return priv_glfs_resolve_at(fs, subvol, NULL, origpath, loc, iatt,
                                     follow, reval);
-	if (cwd)
-		inode_unref (cwd);
+
+    cwd = glfs_cwd_get(fs);
+    if (NULL == cwd) {
+        gf_smsg(subvol->name, GF_LOG_WARNING, EIO, API_MSG_GET_CWD_FAILED,
+                NULL);
+        errno = EIO;
+        goto out;
+    }
+
+    ret = priv_glfs_resolve_at(fs, subvol, cwd, origpath, loc, iatt, follow,
+                               reval);
+    if (cwd)
+        inode_unref(cwd);
 
 out:
-	return ret;
+    return ret;
 }
 
-
+GFAPI_SYMVER_PRIVATE_DEFAULT(glfs_resolve, 3.7.0)
 int
-priv_glfs_resolve (struct glfs *fs, xlator_t *subvol, const char *origpath,
-	      loc_t *loc, struct iatt *iatt, int reval)
+priv_glfs_resolve(struct glfs *fs, xlator_t *subvol, const char *origpath,
+                  loc_t *loc, struct iatt *iatt, int reval)
 {
-	int ret = -1;
+    int ret = -1;
 
-	ret = glfs_resolve_path (fs, subvol, origpath, loc, iatt, 1, reval);
+    ret = glfs_resolve_path(fs, subvol, origpath, loc, iatt, 1, reval);
 
-	return ret;
+    return ret;
 }
-GFAPI_SYMVER_PRIVATE_DEFAULT(glfs_resolve, 3.7.0);
 
 int
-glfs_lresolve (struct glfs *fs, xlator_t *subvol, const char *origpath,
-	       loc_t *loc, struct iatt *iatt, int reval)
+glfs_lresolve(struct glfs *fs, xlator_t *subvol, const char *origpath,
+              loc_t *loc, struct iatt *iatt, int reval)
 {
-	int ret = -1;
+    int ret = -1;
 
-	ret = glfs_resolve_path (fs, subvol, origpath, loc, iatt, 0, reval);
+    ret = glfs_resolve_path(fs, subvol, origpath, loc, iatt, 0, reval);
 
-	return ret;
+    return ret;
 }
 
-
 int
-glfs_migrate_fd_locks_safe (struct glfs *fs, xlator_t *oldsubvol, fd_t *oldfd,
-			    xlator_t *newsubvol, fd_t *newfd)
+glfs_migrate_fd_locks_safe(struct glfs *fs, xlator_t *oldsubvol, fd_t *oldfd,
+                           xlator_t *newsubvol, fd_t *newfd)
 {
-	dict_t *lockinfo = NULL;
-	int ret = 0;
-	char uuid1[64];
-
-	if (!oldfd->lk_ctx || fd_lk_ctx_empty (oldfd->lk_ctx))
-		return 0;
-
-	newfd->lk_ctx = fd_lk_ctx_ref (oldfd->lk_ctx);
-
-	ret = syncop_fgetxattr (oldsubvol, oldfd, &lockinfo,
-				GF_XATTR_LOCKINFO_KEY, NULL, NULL);
-        DECODE_SYNCOP_ERR (ret);
-	if (ret < 0) {
-		gf_msg (fs->volname, GF_LOG_WARNING, errno,
-                        API_MSG_FGETXATTR_FAILED,
-			"fgetxattr (%s) failed (%s) on graph %s (%d)",
-			uuid_utoa_r (oldfd->inode->gfid, uuid1),
-			strerror (errno),
-			graphid_str (oldsubvol), oldsubvol->graph->id);
-		goto out;
-	}
-
-	if (!dict_get (lockinfo, GF_XATTR_LOCKINFO_KEY)) {
-		gf_msg (fs->volname, GF_LOG_WARNING, 0,
-                        API_MSG_LOCKINFO_KEY_MISSING,
-			"missing lockinfo key (%s) on graph %s (%d)",
-			uuid_utoa_r (oldfd->inode->gfid, uuid1),
-			graphid_str (oldsubvol), oldsubvol->graph->id);
-		goto out;
-	}
-
-	ret = syncop_fsetxattr (newsubvol, newfd, lockinfo, 0, NULL, NULL);
-        DECODE_SYNCOP_ERR (ret);
-	if (ret < 0) {
-		gf_msg (fs->volname, GF_LOG_WARNING, 0,
-                        API_MSG_FSETXATTR_FAILED,
-			"fsetxattr (%s) failed (%s) on graph %s (%d)",
-			uuid_utoa_r (newfd->inode->gfid, uuid1),
-			strerror (errno),
-			graphid_str (newsubvol), newsubvol->graph->id);
-		goto out;
-	}
+    dict_t *lockinfo = NULL;
+    int ret = 0;
+    char uuid1[64];
+
+    if (!oldfd->lk_ctx || fd_lk_ctx_empty(oldfd->lk_ctx))
+        return 0;
+
+    newfd->lk_ctx = fd_lk_ctx_ref(oldfd->lk_ctx);
+
+    ret = syncop_fgetxattr(oldsubvol, oldfd, &lockinfo, GF_XATTR_LOCKINFO_KEY,
+                           NULL, NULL);
+    DECODE_SYNCOP_ERR(ret);
+    if (ret < 0) {
+        gf_smsg(fs->volname, GF_LOG_WARNING, errno, API_MSG_FGETXATTR_FAILED,
+                "gfid=%s", uuid_utoa_r(oldfd->inode->gfid, uuid1), "err=%s",
+                strerror(errno), "subvol=%s", graphid_str(oldsubvol), "id=%d",
+                oldsubvol->graph->id, NULL);
+        goto out;
+    }
+
+    if (!dict_get(lockinfo, GF_XATTR_LOCKINFO_KEY)) {
+        gf_smsg(fs->volname, GF_LOG_WARNING, 0, API_MSG_LOCKINFO_KEY_MISSING,
+                "gfid=%s", uuid_utoa_r(oldfd->inode->gfid, uuid1), "subvol=%s",
+                graphid_str(oldsubvol), "id=%d", oldsubvol->graph->id, NULL);
+        goto out;
+    }
+
+    ret = syncop_fsetxattr(newsubvol, newfd, lockinfo, 0, NULL, NULL);
+    DECODE_SYNCOP_ERR(ret);
+    if (ret < 0) {
+        gf_smsg(fs->volname, GF_LOG_WARNING, 0, API_MSG_FSETXATTR_FAILED,
+                "gfid=%s", uuid_utoa_r(newfd->inode->gfid, uuid1), "err=%s",
+                strerror(errno), "subvol=%s", graphid_str(newsubvol), "id=%d",
+                newsubvol->graph->id, NULL);
+        goto out;
+    }
 out:
-	if (lockinfo)
-		dict_unref (lockinfo);
-	return ret;
+    if (lockinfo)
+        dict_unref(lockinfo);
+    return ret;
 }
 
-
 fd_t *
-glfs_migrate_fd_safe (struct glfs *fs, xlator_t *newsubvol, fd_t *oldfd)
+glfs_migrate_fd_safe(struct glfs *fs, xlator_t *newsubvol, fd_t *oldfd)
 {
-	fd_t *newfd = NULL;
-	inode_t *oldinode = NULL;
-	inode_t *newinode = NULL;
-	xlator_t *oldsubvol = NULL;
-	int ret = -1;
-	loc_t loc = {0, };
-	char uuid1[64];
-
-
-	oldinode = oldfd->inode;
-	oldsubvol = oldinode->table->xl;
-
-	if (oldsubvol == newsubvol)
-		return fd_ref (oldfd);
-
-	if (!oldsubvol->switched) {
-		ret = syncop_fsync (oldsubvol, oldfd, 0, NULL, NULL);
-                DECODE_SYNCOP_ERR (ret);
-		if (ret) {
-			gf_msg (fs->volname, GF_LOG_WARNING, errno,
-                                API_MSG_FSYNC_FAILED, "fsync() failed "
-                                "(%s) on %s graph %s (%d)", strerror (errno),
-				uuid_utoa_r (oldfd->inode->gfid, uuid1),
-				graphid_str (oldsubvol), oldsubvol->graph->id);
-		}
-	}
-
-	newinode = glfs_refresh_inode_safe (newsubvol, oldinode, _gf_false);
-	if (!newinode) {
-		gf_msg (fs->volname, GF_LOG_WARNING, errno,
-                        API_MSG_INODE_REFRESH_FAILED,
-			"inode (%s) refresh failed (%s) on graph %s (%d)",
-			uuid_utoa_r (oldinode->gfid, uuid1),
-			strerror (errno),
-			graphid_str (newsubvol), newsubvol->graph->id);
-		goto out;
-	}
-
-	newfd = fd_create (newinode, getpid());
-	if (!newfd) {
-		gf_msg (fs->volname, GF_LOG_WARNING, errno,
-                        API_MSG_FDCREATE_FAILED,
-			"fd_create (%s) failed (%s) on graph %s (%d)",
-			uuid_utoa_r (newinode->gfid, uuid1),
-			strerror (errno),
-			graphid_str (newsubvol), newsubvol->graph->id);
-		goto out;
-	}
-
-	loc.inode = inode_ref (newinode);
-
-        ret = inode_path (oldfd->inode, NULL, (char **)&loc.path);
-        if (ret < 0) {
-                gf_msg (fs->volname, GF_LOG_INFO, 0, API_MSG_INODE_PATH_FAILED,
-                        "inode_path failed");
-                goto out;
+    fd_t *newfd = NULL;
+    inode_t *oldinode = NULL;
+    inode_t *newinode = NULL;
+    xlator_t *oldsubvol = NULL;
+    int ret = -1;
+    loc_t loc = {
+        0,
+    };
+    char uuid1[64];
+    dict_t *xdata = NULL;
+
+    oldinode = oldfd->inode;
+    oldsubvol = oldinode->table->xl;
+
+    if (oldsubvol == newsubvol)
+        return fd_ref(oldfd);
+
+    if (!oldsubvol->switched) {
+        xdata = dict_new();
+        if (!xdata || dict_set_int8(xdata, "last-fsync", 1)) {
+            gf_smsg(fs->volname, GF_LOG_WARNING, ENOMEM, API_MSG_FSYNC_FAILED,
+                    "err=%s", "last-fsync set failed", "gfid=%s",
+                    uuid_utoa_r(oldfd->inode->gfid, uuid1), "subvol=%s",
+                    graphid_str(oldsubvol), "id=%d", oldsubvol->graph->id,
+                    NULL);
         }
 
-        gf_uuid_copy (loc.gfid, oldinode->gfid);
-
-
-	if (IA_ISDIR (oldinode->ia_type))
-		ret = syncop_opendir (newsubvol, &loc, newfd, NULL, NULL);
-	else
-		ret = syncop_open (newsubvol, &loc,
-				   oldfd->flags & ~(O_TRUNC|O_EXCL|O_CREAT),
-				   newfd, NULL, NULL);
-        DECODE_SYNCOP_ERR (ret);
-	loc_wipe (&loc);
-
-	if (ret) {
-		gf_msg (fs->volname, GF_LOG_WARNING, errno,
-                        API_MSG_SYNCOP_OPEN_FAILED,
-			"syncop_open%s (%s) failed (%s) on graph %s (%d)",
-			IA_ISDIR (oldinode->ia_type) ? "dir" : "",
-			uuid_utoa_r (newinode->gfid, uuid1),
-			strerror (errno),
-			graphid_str (newsubvol), newsubvol->graph->id);
-		goto out;
-	}
-
-	ret = glfs_migrate_fd_locks_safe (fs, oldsubvol, oldfd, newsubvol,
-					  newfd);
-
-	if (ret) {
-		gf_msg (fs->volname, GF_LOG_WARNING, errno,
-                        API_MSG_LOCK_MIGRATE_FAILED,
-			"lock migration (%s) failed (%s) on graph %s (%d)",
-			uuid_utoa_r (newinode->gfid, uuid1),
-			strerror (errno),
-			graphid_str (newsubvol), newsubvol->graph->id);
-		goto out;
-	}
-
-        newfd->flags = oldfd->flags;
-	fd_bind (newfd);
+        ret = syncop_fsync(oldsubvol, oldfd, 0, NULL, NULL, xdata, NULL);
+        DECODE_SYNCOP_ERR(ret);
+        if (ret) {
+            gf_smsg(fs->volname, GF_LOG_WARNING, errno, API_MSG_FSYNC_FAILED,
+                    "err=%s", strerror(errno), "gfid=%s",
+                    uuid_utoa_r(oldfd->inode->gfid, uuid1), "subvol=%s",
+                    graphid_str(oldsubvol), "id=%d", oldsubvol->graph->id,
+                    NULL);
+        }
+    }
+
+    newinode = glfs_refresh_inode_safe(newsubvol, oldinode, _gf_false);
+    if (!newinode) {
+        gf_smsg(fs->volname, GF_LOG_WARNING, errno,
+                API_MSG_INODE_REFRESH_FAILED, "gfid=%s",
+                uuid_utoa_r(oldinode->gfid, uuid1), "err=%s", strerror(errno),
+                "subvol=%s", graphid_str(newsubvol), "id=%d",
+                newsubvol->graph->id, NULL);
+        goto out;
+    }
+
+    newfd = fd_create(newinode, getpid());
+    if (!newfd) {
+        gf_smsg(fs->volname, GF_LOG_WARNING, errno,
+                API_MSG_FDCREATE_FAILED_ON_GRAPH, "gfid=%s",
+                uuid_utoa_r(newinode->gfid, uuid1), "err=%s", strerror(errno),
+                "subvol=%s", graphid_str(newsubvol), "id=%d",
+                newsubvol->graph->id, NULL);
+        goto out;
+    }
+
+    loc.inode = inode_ref(newinode);
+
+    ret = inode_path(oldfd->inode, NULL, (char **)&loc.path);
+    if (ret < 0) {
+        gf_smsg(fs->volname, GF_LOG_INFO, 0, API_MSG_INODE_PATH_FAILED, NULL);
+        goto out;
+    }
+
+    gf_uuid_copy(loc.gfid, oldinode->gfid);
+
+    if (IA_ISDIR(oldinode->ia_type))
+        ret = syncop_opendir(newsubvol, &loc, newfd, NULL, NULL);
+    else
+        ret = syncop_open(newsubvol, &loc,
+                          oldfd->flags & ~(O_TRUNC | O_EXCL | O_CREAT), newfd,
+                          NULL, NULL);
+    DECODE_SYNCOP_ERR(ret);
+    loc_wipe(&loc);
+
+    if (ret) {
+        gf_smsg(fs->volname, GF_LOG_WARNING, errno, API_MSG_SYNCOP_OPEN_FAILED,
+                "type=%s", IA_ISDIR(oldinode->ia_type) ? "dir" : "", "gfid=%s",
+                uuid_utoa_r(newinode->gfid, uuid1), "err=%s", strerror(errno),
+                "subvol=%s", graphid_str(newsubvol), "id=%d",
+                newsubvol->graph->id, NULL);
+        goto out;
+    }
+
+    ret = glfs_migrate_fd_locks_safe(fs, oldsubvol, oldfd, newsubvol, newfd);
+
+    if (ret) {
+        gf_smsg(fs->volname, GF_LOG_WARNING, errno, API_MSG_LOCK_MIGRATE_FAILED,
+                "gfid=%s", uuid_utoa_r(newinode->gfid, uuid1), "err=%s",
+                strerror(errno), "subvol=%s", graphid_str(newsubvol), "id=%d",
+                newsubvol->graph->id, NULL);
+        goto out;
+    }
+
+    newfd->flags = oldfd->flags;
+    fd_bind(newfd);
 out:
-	if (newinode)
-		inode_unref (newinode);
+    if (newinode)
+        inode_unref(newinode);
 
-	if (ret) {
-		fd_unref (newfd);
-		newfd = NULL;
-	}
+    if (ret) {
+        fd_unref(newfd);
+        newfd = NULL;
+    }
 
-	return newfd;
-}
+    if (xdata)
+        dict_unref(xdata);
 
+    return newfd;
+}
 
 fd_t *
-__glfs_migrate_fd (struct glfs *fs, xlator_t *newsubvol, struct glfs_fd *glfd)
+__glfs_migrate_fd(struct glfs *fs, xlator_t *newsubvol, struct glfs_fd *glfd)
 {
-	fd_t *oldfd = NULL;
-	fd_t *newfd = NULL;
+    fd_t *oldfd = NULL;
+    fd_t *newfd = NULL;
 
-	oldfd = glfd->fd;
+    oldfd = glfd->fd;
 
-	fs->migration_in_progress = 1;
-	pthread_mutex_unlock (&fs->mutex);
-	{
-		newfd = glfs_migrate_fd_safe (fs, newsubvol, oldfd);
-	}
-	pthread_mutex_lock (&fs->mutex);
-	fs->migration_in_progress = 0;
-	pthread_cond_broadcast (&fs->cond);
+    fs->migration_in_progress = 1;
+    pthread_mutex_unlock(&fs->mutex);
+    {
+        newfd = glfs_migrate_fd_safe(fs, newsubvol, oldfd);
+    }
+    pthread_mutex_lock(&fs->mutex);
+    fs->migration_in_progress = 0;
+    pthread_cond_broadcast(&fs->cond);
 
-	return newfd;
-}
+    /* wake up other waiting tasks */
+    __GLFS_SYNCTASK_WAKE(fs);
 
+    return newfd;
+}
 
 fd_t *
-__glfs_resolve_fd (struct glfs *fs, xlator_t *subvol, struct glfs_fd *glfd)
+__glfs_resolve_fd(struct glfs *fs, xlator_t *subvol, struct glfs_fd *glfd)
 {
-	fd_t *fd = NULL;
+    fd_t *fd = NULL;
 
-	if (glfd->fd->inode->table->xl == subvol)
-		return fd_ref (glfd->fd);
+    if (glfd->fd->inode->table->xl == subvol)
+        return fd_ref(glfd->fd);
 
-	fd = __glfs_migrate_fd (fs, subvol, glfd);
-	if (!fd)
-		return NULL;
+    fd = __glfs_migrate_fd(fs, subvol, glfd);
+    if (!fd)
+        return NULL;
 
-	if (subvol == fs->active_subvol) {
-		fd_unref (glfd->fd);
-		glfd->fd = fd_ref (fd);
-	}
+    if (subvol == fs->active_subvol) {
+        fd_unref(glfd->fd);
+        glfd->fd = fd_ref(fd);
+    }
 
-	return fd;
+    return fd;
 }
 
-
 fd_t *
-glfs_resolve_fd (struct glfs *fs, xlator_t *subvol, struct glfs_fd *glfd)
+glfs_resolve_fd(struct glfs *fs, xlator_t *subvol, struct glfs_fd *glfd)
 {
-	fd_t *fd = NULL;
+    fd_t *fd = NULL;
 
-	glfs_lock (fs);
-	{
-		fd = __glfs_resolve_fd (fs, subvol, glfd);
-	}
-	glfs_unlock (fs);
+    glfs_lock(fs, _gf_true);
+    {
+        fd = __glfs_resolve_fd(fs, subvol, glfd);
+    }
+    glfs_unlock(fs);
 
-	return fd;
+    return fd;
 }
 
-
 void
-__glfs_migrate_openfds (struct glfs *fs, xlator_t *subvol)
+__glfs_migrate_openfds(struct glfs *fs, xlator_t *subvol)
 {
-	struct glfs_fd *glfd = NULL;
-	fd_t *fd = NULL;
-
-	list_for_each_entry (glfd, &fs->openfds, openfds) {
-		if (gf_uuid_is_null (glfd->fd->inode->gfid)) {
-			gf_msg (fs->volname, GF_LOG_INFO, 0,
-                                API_MSG_OPENFD_SKIPPED,
-				"skipping openfd %p/%p in graph %s (%d)",
-				glfd, glfd->fd,	graphid_str(subvol),
-				subvol->graph->id);
-			/* create in progress, defer */
-			continue;
-		}
-
-		fd = __glfs_migrate_fd (fs, subvol, glfd);
-		if (fd) {
-			fd_unref (glfd->fd);
-			glfd->fd = fd;
-		}
-	}
-}
+    struct glfs_fd *glfd = NULL;
+    fd_t *fd = NULL;
+
+    list_for_each_entry(glfd, &fs->openfds, openfds)
+    {
+        if (gf_uuid_is_null(glfd->fd->inode->gfid)) {
+            gf_smsg(fs->volname, GF_LOG_INFO, 0, API_MSG_OPENFD_SKIPPED,
+                    "glfd=%p", glfd, "glfd->fd=%p", glfd->fd, "subvol=%s",
+                    graphid_str(subvol), "id=%d", subvol->graph->id, NULL);
+            /* create in progress, defer */
+            continue;
+        }
 
+        fd = __glfs_migrate_fd(fs, subvol, glfd);
+        if (fd) {
+            fd_unref(glfd->fd);
+            glfd->fd = fd;
+        }
+    }
+}
 
+/* Note that though it appears that this function executes under fs->mutex,
+ * it is not fully executed under fs->mutex. i.e. there are functions like
+ * __glfs_first_lookup, __glfs_refresh_inode, __glfs_migrate_openfds which
+ * unlocks fs->mutex before sending any network fop, and reacquire fs->mutex
+ * once the fop is complete. Hence the variable read from fs at the start of the
+ * function need not have the same value by the end of the function.
+ */
 xlator_t *
-__glfs_active_subvol (struct glfs *fs)
+__glfs_active_subvol(struct glfs *fs)
 {
-	xlator_t      *new_subvol = NULL;
-	int            ret = -1;
-	inode_t       *new_cwd = NULL;
-
-	if (!fs->next_subvol)
-		return fs->active_subvol;
-
-	new_subvol = fs->next_subvol;
-
-	ret = __glfs_first_lookup (fs, new_subvol);
-	if (ret) {
-		gf_msg (fs->volname, GF_LOG_INFO, errno,
-                        API_MSG_FIRST_LOOKUP_GRAPH_FAILED,
-			"first lookup on graph %s (%d) failed (%s)",
-			graphid_str (new_subvol), new_subvol->graph->id,
-			strerror (errno));
-		return NULL;
-	}
-
-	if (fs->cwd) {
-		new_cwd = __glfs_refresh_inode (fs, new_subvol, fs->cwd,
-                                                _gf_false);
-
-		if (!new_cwd) {
-			char buf1[64];
-			gf_msg (fs->volname, GF_LOG_INFO, errno,
-                                API_MSG_CWD_GRAPH_REF_FAILED,
-				"cwd refresh of %s graph %s (%d) failed (%s)",
-				uuid_utoa_r (fs->cwd->gfid, buf1),
-				graphid_str (new_subvol),
-				new_subvol->graph->id, strerror (errno));
-			return NULL;
-		}
-	}
-
-	__glfs_migrate_openfds (fs, new_subvol);
-
-	/* switching @active_subvol and @cwd
-	   should be atomic
-	*/
-	fs->old_subvol = fs->active_subvol;
-	fs->active_subvol = fs->next_subvol;
-	fs->next_subvol = NULL;
-
-	if (new_cwd) {
-		__glfs_cwd_set (fs, new_cwd);
-		inode_unref (new_cwd);
-	}
-
-	gf_msg (fs->volname, GF_LOG_INFO, 0, API_MSG_SWITCHED_GRAPH,
-                "switched to graph %s (%d)",
-		graphid_str (new_subvol), new_subvol->graph->id);
-
-	return new_subvol;
-}
+    xlator_t *new_subvol = NULL;
+    int ret = -1;
+    inode_t *new_cwd = NULL;
+
+    if (!fs->next_subvol)
+        return fs->active_subvol;
+
+    new_subvol = fs->mip_subvol = fs->next_subvol;
+    fs->next_subvol = NULL;
+
+    ret = __glfs_first_lookup(fs, new_subvol);
+    if (ret) {
+        gf_smsg(fs->volname, GF_LOG_INFO, errno,
+                API_MSG_FIRST_LOOKUP_GRAPH_FAILED, "subvol=%s",
+                graphid_str(new_subvol), "id=%d", new_subvol->graph->id,
+                "err=%s", strerror(errno), NULL);
+        return NULL;
+    }
+
+    if (fs->cwd) {
+        new_cwd = __glfs_refresh_inode(fs, new_subvol, fs->cwd, _gf_false);
+
+        if (!new_cwd) {
+            char buf1[64];
+            gf_smsg(fs->volname, GF_LOG_INFO, errno,
+                    API_MSG_CWD_GRAPH_REF_FAILED, "buf=%s",
+                    uuid_utoa_r(fs->cwd->gfid, buf1), "subvol=%s",
+                    graphid_str(new_subvol), "id=%d", new_subvol->graph->id,
+                    "err=%s", strerror(errno), NULL);
+            return NULL;
+        }
+    }
+
+    __glfs_migrate_openfds(fs, new_subvol);
+    /* TODO: Migrate the fds and inodes which have leases to the new graph
+     * (issue #350)*/
 
+    /* switching @active_subvol and @cwd
+       should be atomic
+    */
+    fs->old_subvol = fs->active_subvol;
+    fs->active_subvol = fs->mip_subvol;
+    fs->mip_subvol = NULL;
 
+    if (new_cwd) {
+        __glfs_cwd_set(fs, new_cwd);
+        inode_unref(new_cwd);
+    }
+
+    gf_smsg(fs->volname, GF_LOG_INFO, 0, API_MSG_SWITCHED_GRAPH, "subvol=%s",
+            graphid_str(new_subvol), "id=%d", new_subvol->graph->id, NULL);
+
+    return new_subvol;
+}
+
+GFAPI_SYMVER_PRIVATE_DEFAULT(glfs_subvol_done, 3.4.0)
 void
-priv_glfs_subvol_done (struct glfs *fs, xlator_t *subvol)
+priv_glfs_subvol_done(struct glfs *fs, xlator_t *subvol)
 {
-	int ref = 0;
-	xlator_t *active_subvol = NULL;
-
-	if (!subvol)
-		return;
-
-	glfs_lock (fs);
-	{
-		ref = (--subvol->winds);
-		active_subvol = fs->active_subvol;
-	}
-	glfs_unlock (fs);
-
-	if (ref == 0) {
-		assert (subvol != active_subvol);
-		xlator_notify (subvol, GF_EVENT_PARENT_DOWN, subvol, NULL);
-	}
+    int ref = 0;
+    xlator_t *active_subvol = NULL;
+
+    if (!subvol)
+        return;
+
+    /* For decrementing subvol->wind ref count we need not check/wait for
+     * migration-in-progress flag.
+     * Also glfs_subvol_done is called in call-back path therefore waiting
+     * for migration-in-progress flag can lead to dead-lock.
+     */
+    glfs_lock(fs, _gf_false);
+    {
+        ref = (--subvol->winds);
+        active_subvol = fs->active_subvol;
+    }
+    glfs_unlock(fs);
+
+    if (ref == 0) {
+        assert(subvol != active_subvol);
+        xlator_notify(subvol, GF_EVENT_PARENT_DOWN, subvol, NULL);
+    }
 }
 
-GFAPI_SYMVER_PRIVATE_DEFAULT(glfs_subvol_done, 3.4.0);
-
-
+GFAPI_SYMVER_PRIVATE_DEFAULT(glfs_active_subvol, 3.4.0)
 xlator_t *
-priv_glfs_active_subvol (struct glfs *fs)
+priv_glfs_active_subvol(struct glfs *fs)
 {
-	xlator_t      *subvol = NULL;
-	xlator_t      *old_subvol = NULL;
+    xlator_t *subvol = NULL;
+    xlator_t *old_subvol = NULL;
 
-	glfs_lock (fs);
-	{
-		subvol = __glfs_active_subvol (fs);
+    glfs_lock(fs, _gf_true);
+    {
+        subvol = __glfs_active_subvol(fs);
 
-		if (subvol)
-			subvol->winds++;
+        if (subvol)
+            subvol->winds++;
 
-		if (fs->old_subvol) {
-			old_subvol = fs->old_subvol;
-			fs->old_subvol = NULL;
-			old_subvol->switched = 1;
-		}
-	}
-	glfs_unlock (fs);
+        if (fs->old_subvol) {
+            old_subvol = fs->old_subvol;
+            fs->old_subvol = NULL;
+            old_subvol->switched = 1;
+        }
+    }
+    glfs_unlock(fs);
 
-	if (old_subvol)
-		priv_glfs_subvol_done (fs, old_subvol);
+    if (old_subvol)
+        priv_glfs_subvol_done(fs, old_subvol);
 
-	return subvol;
+    return subvol;
 }
 
-GFAPI_SYMVER_PRIVATE_DEFAULT(glfs_active_subvol, 3.4.0);
-
 int
-__glfs_cwd_set (struct glfs *fs, inode_t *inode)
+__glfs_cwd_set(struct glfs *fs, inode_t *inode)
 {
-	if (inode->table->xl != fs->active_subvol) {
-		inode = __glfs_refresh_inode (fs, fs->active_subvol, inode,
-                                              _gf_false);
-		if (!inode)
-			return -1;
-	} else {
-		inode_ref (inode);
-	}
+    if (inode->table->xl != fs->active_subvol) {
+        inode = __glfs_refresh_inode(fs, fs->active_subvol, inode, _gf_false);
+        if (!inode)
+            return -1;
+    } else {
+        inode_ref(inode);
+    }
 
-	if (fs->cwd)
-		inode_unref (fs->cwd);
+    if (fs->cwd)
+        inode_unref(fs->cwd);
 
-	fs->cwd = inode;
+    fs->cwd = inode;
 
-	return 0;
+    return 0;
 }
 
-
 int
-glfs_cwd_set (struct glfs *fs, inode_t *inode)
+glfs_cwd_set(struct glfs *fs, inode_t *inode)
 {
-	int ret = 0;
+    int ret = 0;
 
-	glfs_lock (fs);
-	{
-		ret = __glfs_cwd_set (fs, inode);
-	}
-	glfs_unlock (fs);
+    glfs_lock(fs, _gf_true);
+    {
+        ret = __glfs_cwd_set(fs, inode);
+    }
+    glfs_unlock(fs);
 
-	return ret;
+    return ret;
 }
 
-
 inode_t *
-__glfs_cwd_get (struct glfs *fs)
+__glfs_cwd_get(struct glfs *fs)
 {
-	inode_t *cwd = NULL;
+    inode_t *cwd = NULL;
 
-	if (!fs->cwd)
-		return NULL;
+    if (!fs->cwd)
+        return NULL;
 
-	if (fs->cwd->table->xl == fs->active_subvol) {
-		cwd = inode_ref (fs->cwd);
-		return cwd;
-	}
+    if (fs->cwd->table->xl == fs->active_subvol) {
+        cwd = inode_ref(fs->cwd);
+        return cwd;
+    }
 
-	cwd = __glfs_refresh_inode (fs, fs->active_subvol, fs->cwd, _gf_false);
+    cwd = __glfs_refresh_inode(fs, fs->active_subvol, fs->cwd, _gf_false);
 
-	return cwd;
+    return cwd;
 }
 
 inode_t *
-glfs_cwd_get (struct glfs *fs)
+glfs_cwd_get(struct glfs *fs)
 {
-	inode_t *cwd = NULL;
+    inode_t *cwd = NULL;
 
-	glfs_lock (fs);
-	{
-		cwd = __glfs_cwd_get (fs);
-	}
-	glfs_unlock (fs);
+    glfs_lock(fs, _gf_true);
+    {
+        cwd = __glfs_cwd_get(fs);
+    }
+    glfs_unlock(fs);
 
-	return cwd;
+    return cwd;
 }
 
 inode_t *
-__glfs_resolve_inode (struct glfs *fs, xlator_t *subvol,
-		    struct glfs_object *object)
+__glfs_resolve_inode(struct glfs *fs, xlator_t *subvol,
+                     struct glfs_object *object)
 {
-	inode_t *inode = NULL;
-        gf_boolean_t lookup_needed = _gf_false;
+    inode_t *inode = NULL;
+    gf_boolean_t lookup_needed = _gf_false;
 
-        lookup_needed = inode_needs_lookup (object->inode, THIS);
+    lookup_needed = inode_needs_lookup(object->inode, THIS);
 
-	if (!lookup_needed && object->inode->table->xl == subvol)
-		return inode_ref (object->inode);
+    if (!lookup_needed && object->inode->table->xl == subvol)
+        return inode_ref(object->inode);
 
-	inode = __glfs_refresh_inode (fs, fs->active_subvol,
-                                      object->inode, lookup_needed);
-	if (!inode)
-		return NULL;
+    inode = __glfs_refresh_inode(fs, fs->active_subvol, object->inode,
+                                 lookup_needed);
+    if (!inode)
+        return NULL;
 
-	if (subvol == fs->active_subvol) {
-		inode_unref (object->inode);
-		object->inode = inode_ref (inode);
-	}
+    if (subvol == fs->active_subvol) {
+        inode_unref(object->inode);
+        object->inode = inode_ref(inode);
+    }
 
-	return inode;
+    return inode;
 }
 
 inode_t *
-glfs_resolve_inode (struct glfs *fs, xlator_t *subvol,
-		    struct glfs_object *object)
+glfs_resolve_inode(struct glfs *fs, xlator_t *subvol,
+                   struct glfs_object *object)
 {
-	inode_t *inode = NULL;
+    inode_t *inode = NULL;
 
-	glfs_lock (fs);
-	{
-		inode = __glfs_resolve_inode(fs, subvol, object);
-	}
-	glfs_unlock (fs);
+    glfs_lock(fs, _gf_true);
+    {
+        inode = __glfs_resolve_inode(fs, subvol, object);
+    }
+    glfs_unlock(fs);
 
-	return inode;
+    return inode;
 }
 
 int
-glfs_create_object (loc_t *loc, struct glfs_object **retobject)
+glfs_create_object(loc_t *loc, struct glfs_object **retobject)
 {
-	struct glfs_object *object = NULL;
+    struct glfs_object *object = NULL;
 
-	object = GF_CALLOC (1, sizeof(struct glfs_object),
-			    glfs_mt_glfs_object_t);
-	if (object == NULL) {
-		errno = ENOMEM;
-		return -1;
-	}
+    object = GF_CALLOC(1, sizeof(struct glfs_object), glfs_mt_glfs_object_t);
+    if (object == NULL) {
+        errno = ENOMEM;
+        return -1;
+    }
 
-	object->inode = loc->inode;
-	gf_uuid_copy (object->gfid, object->inode->gfid);
+    object->inode = loc->inode;
+    gf_uuid_copy(object->gfid, object->inode->gfid);
 
-	/* we hold the reference */
-	loc->inode = NULL;
+    /* we hold the reference */
+    loc->inode = NULL;
 
-	*retobject = object;
+    *retobject = object;
 
-	return 0;
+    return 0;
 }
 
 struct glfs_object *
-glfs_h_resolve_symlink (struct glfs *fs, struct glfs_object *object)
+glfs_h_resolve_symlink(struct glfs *fs, struct glfs_object *object)
 {
-
-        xlator_t                *subvol         = NULL;
-        loc_t                   sym_loc         = {0,};
-        struct iatt             iatt            = {0,};
-        char                    *lpath          = NULL;
-        int                     ret             = 0;
-        struct glfs_object      *target_object  = NULL;
-
-        subvol = glfs_active_subvol (fs);
-        if (!subvol) {
-                ret = -1;
-                errno = EIO;
-                goto out;
-        }
-
-        ret = glfs_resolve_symlink (fs, subvol, object->inode, &lpath);
-        if (ret < 0)
-                goto out;
-
-        ret = glfs_resolve_at (fs, subvol, NULL, lpath,
-                               &sym_loc, &iatt,
-                              /* always recurisvely follow while
-                                following symlink
-                              */
-                               1, 0);
-        if (ret == 0)
-                ret = glfs_create_object (&sym_loc, &target_object);
+    xlator_t *subvol = NULL;
+    loc_t sym_loc = {
+        0,
+    };
+    struct iatt iatt = {
+        0,
+    };
+    char *lpath = NULL;
+    int ret = 0;
+    struct glfs_object *target_object = NULL;
+
+    subvol = glfs_active_subvol(fs);
+    if (!subvol) {
+        ret = -1;
+        errno = EIO;
+        goto out;
+    }
+
+    ret = glfs_resolve_symlink(fs, subvol, object->inode, &lpath);
+    if (ret < 0)
+        goto out;
+
+    ret = glfs_resolve_at(fs, subvol, NULL, lpath, &sym_loc, &iatt,
+                          /* always recurisvely follow while
+                            following symlink
+                          */
+                          1, 0);
+    if (ret == 0)
+        ret = glfs_create_object(&sym_loc, &target_object);
 
 out:
-        loc_wipe (&sym_loc);
-        GF_FREE (lpath);
-        return target_object;
+    loc_wipe(&sym_loc);
+    GF_FREE(lpath);
+    return target_object;
 }
diff --git a/api/src/glfs.c b/api/src/glfs.c
index 18f5c8f913f..b4bf1423f6d 100644
--- a/api/src/glfs.c
+++ b/api/src/glfs.c
@@ -1,5 +1,5 @@
 /*
-  Copyright (c) 2012 Red Hat, Inc. <http://www.redhat.com>
+  Copyright (c) 2012-2018 Red Hat, Inc. <http://www.redhat.com>
   This file is part of GlusterFS.
 
   This file is licensed to you under your choice of the GNU Lesser
@@ -8,10 +8,8 @@
   cases as published by the Free Software Foundation.
 */
 
-
 /*
   TODO:
-  - merge locks in glfs_posix_lock for lock self-healing
   - set proper pid/lk_owner to call frames (currently buried in syncop)
   - fix logging.c/h to store logfp and loglevel in glusterfs_ctx_t and
     reach it via THIS.
@@ -32,1226 +30,1777 @@
 #include <sys/types.h>
 #include <unistd.h>
 #include <limits.h>
-
-#include "glusterfs.h"
-#include "logging.h"
-#include "stack.h"
-#include "event.h"
+#ifdef GF_LINUX_HOST_OS
+#include <sys/prctl.h>
+#endif
+
+#include <glusterfs/glusterfs.h>
+#include <glusterfs/logging.h>
+#include <glusterfs/stack.h>
+#include <glusterfs/gf-event.h>
 #include "glfs-mem-types.h"
-#include "common-utils.h"
-#include "syncop.h"
-#include "call-stub.h"
-#include "gfapi-messages.h"
+#include <glusterfs/common-utils.h>
+#include <glusterfs/syncop.h>
+#include <glusterfs/call-stub.h>
+#include <glusterfs/hashfn.h>
+#include "rpc-clnt.h"
+#include <glusterfs/statedump.h>
+#include <glusterfs/syscall.h>
 
+#include "gfapi-messages.h"
 #include "glfs.h"
 #include "glfs-internal.h"
-#include "hashfn.h"
-#include "rpc-clnt.h"
-
 
 static gf_boolean_t
-vol_assigned (cmd_args_t *args)
+vol_assigned(cmd_args_t *args)
 {
-	return args->volfile || args->volfile_server;
+    return args->volfile || args->volfile_server;
 }
 
-
 static int
-glusterfs_ctx_defaults_init (glusterfs_ctx_t *ctx)
+glusterfs_ctx_defaults_init(glusterfs_ctx_t *ctx)
 {
-	call_pool_t   *pool = NULL;
-	int	       ret = -1;
+    call_pool_t *pool = NULL;
+    int ret = -1;
 
-	if (!ctx) {
-		goto err;
-        }
-
-        ret = xlator_mem_acct_init (THIS, glfs_mt_end + 1);
-        if (ret != 0) {
-                gf_msg (THIS->name, GF_LOG_ERROR, ENOMEM,
-                        API_MSG_MEM_ACCT_INIT_FAILED,
-                        "Memory accounting init failed");
-                return ret;
-        }
+    if (!ctx) {
+        goto err;
+    }
 
-        /* reset ret to -1 so that we don't need to explicitly
-         * set it in all error paths before "goto err"
-         */
-
-        ret = -1;
-
-	ctx->process_uuid = generate_glusterfs_ctx_id ();
-	if (!ctx->process_uuid) {
-		goto err;
-	}
-
-	ctx->page_size	= 128 * GF_UNIT_KB;
-
-	ctx->iobuf_pool = iobuf_pool_new ();
-	if (!ctx->iobuf_pool) {
-		goto err;
-	}
-
-	ctx->event_pool = event_pool_new (DEFAULT_EVENT_POOL_SIZE,
-                                          STARTING_EVENT_THREADS);
-	if (!ctx->event_pool) {
-		goto err;
-	}
-
-	ctx->env = syncenv_new (0, 0, 0);
-	if (!ctx->env) {
-		goto err;
-	}
-
-	pool = GF_CALLOC (1, sizeof (call_pool_t),
-			  glfs_mt_call_pool_t);
-	if (!pool) {
-		goto err;
-	}
-
-	/* frame_mem_pool size 112 * 4k */
-	pool->frame_mem_pool = mem_pool_new (call_frame_t, 4096);
-	if (!pool->frame_mem_pool) {
-		goto err;
-	}
-	/* stack_mem_pool size 256 * 1024 */
-	pool->stack_mem_pool = mem_pool_new (call_stack_t, 1024);
-	if (!pool->stack_mem_pool) {
-		goto err;
-	}
-
-	ctx->stub_mem_pool = mem_pool_new (call_stub_t, 1024);
-	if (!ctx->stub_mem_pool) {
-		goto err;
-	}
-
-	ctx->dict_pool = mem_pool_new (dict_t, GF_MEMPOOL_COUNT_OF_DICT_T);
-	if (!ctx->dict_pool)
-		goto err;
-
-	ctx->dict_pair_pool = mem_pool_new (data_pair_t,
-					    GF_MEMPOOL_COUNT_OF_DATA_PAIR_T);
-	if (!ctx->dict_pair_pool)
-		goto err;
-
-	ctx->dict_data_pool = mem_pool_new (data_t, GF_MEMPOOL_COUNT_OF_DATA_T);
-	if (!ctx->dict_data_pool)
-		goto err;
-
-        ctx->logbuf_pool = mem_pool_new (log_buf_t,
-                                         GF_MEMPOOL_COUNT_OF_LRU_BUF_T);
-        if (!ctx->logbuf_pool)
-                goto err;
-
-	INIT_LIST_HEAD (&pool->all_frames);
-	INIT_LIST_HEAD (&ctx->cmd_args.xlator_options);
-        INIT_LIST_HEAD (&ctx->cmd_args.volfile_servers);
-
-	LOCK_INIT (&pool->lock);
-	ctx->pool = pool;
-
-	pthread_mutex_init (&(ctx->lock), NULL);
-
-	ret = 0;
+    ret = xlator_mem_acct_init(THIS, glfs_mt_end + 1);
+    if (ret != 0) {
+        gf_smsg(THIS->name, GF_LOG_ERROR, ENOMEM, API_MSG_MEM_ACCT_INIT_FAILED,
+                NULL);
+        return ret;
+    }
+
+    /* reset ret to -1 so that we don't need to explicitly
+     * set it in all error paths before "goto err"
+     */
+
+    ret = -1;
+
+    ctx->process_uuid = generate_glusterfs_ctx_id();
+    if (!ctx->process_uuid) {
+        goto err;
+    }
+
+    ctx->page_size = 128 * GF_UNIT_KB;
+
+    ctx->iobuf_pool = iobuf_pool_new();
+    if (!ctx->iobuf_pool) {
+        goto err;
+    }
+
+    ctx->event_pool = gf_event_pool_new(DEFAULT_EVENT_POOL_SIZE,
+                                        STARTING_EVENT_THREADS);
+    if (!ctx->event_pool) {
+        goto err;
+    }
+
+    ctx->env = syncenv_new(0, 0, 0);
+    if (!ctx->env) {
+        goto err;
+    }
+
+    pool = GF_CALLOC(1, sizeof(call_pool_t), glfs_mt_call_pool_t);
+    if (!pool) {
+        goto err;
+    }
+
+    /* frame_mem_pool size 112 * 4k */
+    pool->frame_mem_pool = mem_pool_new(call_frame_t, 4096);
+    if (!pool->frame_mem_pool) {
+        goto err;
+    }
+    /* stack_mem_pool size 256 * 1024 */
+    pool->stack_mem_pool = mem_pool_new(call_stack_t, 1024);
+    if (!pool->stack_mem_pool) {
+        goto err;
+    }
+
+    ctx->stub_mem_pool = mem_pool_new(call_stub_t, 1024);
+    if (!ctx->stub_mem_pool) {
+        goto err;
+    }
+
+    ctx->dict_pool = mem_pool_new(dict_t, GF_MEMPOOL_COUNT_OF_DICT_T);
+    if (!ctx->dict_pool)
+        goto err;
+
+    ctx->dict_pair_pool = mem_pool_new(data_pair_t,
+                                       GF_MEMPOOL_COUNT_OF_DATA_PAIR_T);
+    if (!ctx->dict_pair_pool)
+        goto err;
+
+    ctx->dict_data_pool = mem_pool_new(data_t, GF_MEMPOOL_COUNT_OF_DATA_T);
+    if (!ctx->dict_data_pool)
+        goto err;
+
+    ctx->logbuf_pool = mem_pool_new(log_buf_t, GF_MEMPOOL_COUNT_OF_LRU_BUF_T);
+    if (!ctx->logbuf_pool)
+        goto err;
+
+    INIT_LIST_HEAD(&pool->all_frames);
+    INIT_LIST_HEAD(&ctx->cmd_args.xlator_options);
+    INIT_LIST_HEAD(&ctx->cmd_args.volfile_servers);
+
+    LOCK_INIT(&pool->lock);
+    ctx->pool = pool;
+
+    ret = 0;
 err:
-	if (ret && pool) {
-		if (pool->frame_mem_pool)
-			mem_pool_destroy (pool->frame_mem_pool);
-		if (pool->stack_mem_pool)
-			mem_pool_destroy (pool->stack_mem_pool);
-		GF_FREE (pool);
-	}
-
-	if (ret && ctx) {
-		if (ctx->stub_mem_pool)
-			mem_pool_destroy (ctx->stub_mem_pool);
-		if (ctx->dict_pool)
-			mem_pool_destroy (ctx->dict_pool);
-		if (ctx->dict_data_pool)
-			mem_pool_destroy (ctx->dict_data_pool);
-		if (ctx->dict_pair_pool)
-			mem_pool_destroy (ctx->dict_pair_pool);
-                if (ctx->logbuf_pool)
-                        mem_pool_destroy (ctx->logbuf_pool);
-	}
-
-	return ret;
-}
+    if (ret && pool) {
+        if (pool->frame_mem_pool)
+            mem_pool_destroy(pool->frame_mem_pool);
+        if (pool->stack_mem_pool)
+            mem_pool_destroy(pool->stack_mem_pool);
+        GF_FREE(pool);
+    }
+
+    if (ret && ctx) {
+        if (ctx->stub_mem_pool)
+            mem_pool_destroy(ctx->stub_mem_pool);
+        if (ctx->dict_pool)
+            mem_pool_destroy(ctx->dict_pool);
+        if (ctx->dict_data_pool)
+            mem_pool_destroy(ctx->dict_data_pool);
+        if (ctx->dict_pair_pool)
+            mem_pool_destroy(ctx->dict_pair_pool);
+        if (ctx->logbuf_pool)
+            mem_pool_destroy(ctx->logbuf_pool);
+    }
 
+    return ret;
+}
 
 static int
-create_master (struct glfs *fs)
+create_master(struct glfs *fs)
 {
-	int		 ret = 0;
-	xlator_t	*master = NULL;
-
-	master = GF_CALLOC (1, sizeof (*master),
-			    glfs_mt_xlator_t);
-	if (!master)
-		goto err;
+    int ret = 0;
+    xlator_t *master = NULL;
 
-	master->name = gf_strdup ("gfapi");
-	if (!master->name)
-		goto err;
+    master = GF_CALLOC(1, sizeof(*master), glfs_mt_xlator_t);
+    if (!master)
+        goto err;
 
-	if (xlator_set_type (master, "mount/api") == -1) {
-		gf_msg ("glfs", GF_LOG_ERROR, 0,
-                        API_MSG_MASTER_XLATOR_INIT_FAILED, "master xlator "
-                        "for %s initialization failed", fs->volname);
-		goto err;
-	}
+    master->name = gf_strdup("gfapi");
+    if (!master->name)
+        goto err;
 
-	master->ctx	 = fs->ctx;
-	master->private	 = fs;
-	master->options	 = get_new_dict ();
-	if (!master->options)
-		goto err;
+    if (xlator_set_type(master, "mount/api") == -1) {
+        gf_smsg("glfs", GF_LOG_ERROR, 0, API_MSG_MASTER_XLATOR_INIT_FAILED,
+                "name=%s", fs->volname, NULL);
+        goto err;
+    }
 
+    master->ctx = fs->ctx;
+    master->private = fs;
+    master->options = dict_new();
+    if (!master->options)
+        goto err;
 
-	ret = xlator_init (master);
-	if (ret) {
-		gf_msg ("glfs", GF_LOG_ERROR, 0,
-                        API_MSG_GFAPI_XLATOR_INIT_FAILED,
-			"failed to initialize gfapi translator");
-		goto err;
-	}
+    ret = xlator_init(master);
+    if (ret) {
+        gf_smsg("glfs", GF_LOG_ERROR, 0, API_MSG_GFAPI_XLATOR_INIT_FAILED,
+                NULL);
+        goto err;
+    }
 
-	fs->ctx->master = master;
-	THIS = master;
+    fs->ctx->master = master;
+    THIS = master;
 
-	return 0;
+    return 0;
 
 err:
-	if (master) {
-		xlator_destroy (master);
-	}
+    if (master) {
+        xlator_destroy(master);
+    }
 
-	return -1;
+    return -1;
 }
 
-
 static FILE *
-get_volfp (struct glfs *fs)
+get_volfp(struct glfs *fs)
 {
-	cmd_args_t  *cmd_args = NULL;
-	FILE	    *specfp = NULL;
+    cmd_args_t *cmd_args = NULL;
+    FILE *specfp = NULL;
 
-	cmd_args = &fs->ctx->cmd_args;
+    cmd_args = &fs->ctx->cmd_args;
 
-	if ((specfp = fopen (cmd_args->volfile, "r")) == NULL) {
-		gf_msg ("glfs", GF_LOG_ERROR, errno,
-                        API_MSG_VOLFILE_OPEN_FAILED,
-			"volume file %s open failed: %s",
-			cmd_args->volfile,
-			strerror (errno));
-		return NULL;
-	}
+    if ((specfp = fopen(cmd_args->volfile, "r")) == NULL) {
+        gf_smsg("glfs", GF_LOG_ERROR, errno, API_MSG_VOLFILE_OPEN_FAILED,
+                "file=%s", cmd_args->volfile, "err=%s", strerror(errno), NULL);
+        return NULL;
+    }
 
-	gf_msg_debug ("glfs", 0, "loading volume file %s", cmd_args->volfile);
+    gf_msg_debug("glfs", 0, "loading volume file %s", cmd_args->volfile);
 
-	return specfp;
+    return specfp;
 }
 
-
 int
-glfs_volumes_init (struct glfs *fs)
+glfs_volumes_init(struct glfs *fs)
 {
-	FILE		   *fp = NULL;
-	cmd_args_t	   *cmd_args = NULL;
-	int		    ret = 0;
+    FILE *fp = NULL;
+    cmd_args_t *cmd_args = NULL;
+    int ret = 0;
+
+    cmd_args = &fs->ctx->cmd_args;
 
-	cmd_args = &fs->ctx->cmd_args;
+    if (!vol_assigned(cmd_args))
+        return -1;
 
-	if (!vol_assigned (cmd_args))
-		return -1;
+    if (sys_access(SECURE_ACCESS_FILE, F_OK) == 0) {
+        fs->ctx->secure_mgmt = 1;
+        fs->ctx->ssl_cert_depth = glusterfs_read_secure_access_file();
+    }
 
-	if (cmd_args->volfile_server) {
-		ret = glfs_mgmt_init (fs);
-		goto out;
-	}
+    if (cmd_args->volfile_server) {
+        ret = glfs_mgmt_init(fs);
+        goto out;
+    }
 
-	fp = get_volfp (fs);
+    fp = get_volfp(fs);
 
-	if (!fp) {
-		gf_msg ("glfs", GF_LOG_ERROR, ENOENT,
-                        API_MSG_VOL_SPEC_FILE_ERROR,
-			"Cannot reach volume specification file");
-		ret = -1;
-		goto out;
-	}
+    if (!fp) {
+        gf_smsg("glfs", GF_LOG_ERROR, ENOENT, API_MSG_VOL_SPEC_FILE_ERROR,
+                NULL);
+        ret = -1;
+        goto out;
+    }
 
-	ret = glfs_process_volfp (fs, fp);
-	if (ret)
-		goto out;
+    ret = glfs_process_volfp(fs, fp);
+    if (ret)
+        goto out;
 
 out:
-	return ret;
+    return ret;
 }
 
-
 ///////////////////////////////////////////////////////////////////////////////
 
-
+GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_set_xlator_option, 3.4.0)
 int
-pub_glfs_set_xlator_option (struct glfs *fs, const char *xlator,
-                            const char *key, const char *value)
+pub_glfs_set_xlator_option(struct glfs *fs, const char *xlator, const char *key,
+                           const char *value)
 {
-	xlator_cmdline_option_t *option = NULL;
+    xlator_cmdline_option_t *option = NULL;
 
-        DECLARE_OLD_THIS;
-        __GLFS_ENTRY_VALIDATE_FS (fs, invalid_fs);
+    DECLARE_OLD_THIS;
+    __GLFS_ENTRY_VALIDATE_FS(fs, invalid_fs);
 
-	option = GF_CALLOC (1, sizeof (*option),
-			    glfs_mt_xlator_cmdline_option_t);
-	if (!option)
-		goto enomem;
+    option = GF_CALLOC(1, sizeof(*option), glfs_mt_xlator_cmdline_option_t);
+    if (!option)
+        goto enomem;
 
-	INIT_LIST_HEAD (&option->cmd_args);
+    INIT_LIST_HEAD(&option->cmd_args);
 
-	option->volume = gf_strdup (xlator);
-	if (!option->volume)
-		goto enomem;
-	option->key = gf_strdup (key);
-	if (!option->key)
-		goto enomem;
-	option->value = gf_strdup (value);
-	if (!option->value)
-		goto enomem;
+    option->volume = gf_strdup(xlator);
+    if (!option->volume)
+        goto enomem;
+    option->key = gf_strdup(key);
+    if (!option->key)
+        goto enomem;
+    option->value = gf_strdup(value);
+    if (!option->value)
+        goto enomem;
 
-	list_add (&option->cmd_args, &fs->ctx->cmd_args.xlator_options);
+    list_add(&option->cmd_args, &fs->ctx->cmd_args.xlator_options);
 
-        __GLFS_EXIT_FS;
+    __GLFS_EXIT_FS;
 
-	return 0;
+    return 0;
 enomem:
-	errno = ENOMEM;
+    errno = ENOMEM;
 
-	if (!option) {
-                __GLFS_EXIT_FS;
-		return -1;
-        }
+    if (!option) {
+        __GLFS_EXIT_FS;
+        return -1;
+    }
 
-	GF_FREE (option->volume);
-	GF_FREE (option->key);
-	GF_FREE (option->value);
-	GF_FREE (option);
+    GF_FREE(option->volume);
+    GF_FREE(option->key);
+    GF_FREE(option->value);
+    GF_FREE(option);
 
-        __GLFS_EXIT_FS;
+    __GLFS_EXIT_FS;
 
 invalid_fs:
-	return -1;
+    return -1;
 }
 
-GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_set_xlator_option, 3.4.0);
-
-
+GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_unset_volfile_server, 3.5.1)
 int
-pub_glfs_unset_volfile_server (struct glfs *fs, const char *transport,
-                               const char *host, const int port)
+pub_glfs_unset_volfile_server(struct glfs *fs, const char *transport,
+                              const char *host, const int port)
 {
-        cmd_args_t       *cmd_args = NULL;
-        server_cmdline_t *server = NULL;
-        server_cmdline_t *tmp = NULL;
-        char             *transport_val = NULL;
-        int               port_val = 0;
-        int               ret = -1;
-
-        if (!fs || !host) {
-                errno = EINVAL;
-                return ret;
-        }
-
-        DECLARE_OLD_THIS;
-        __GLFS_ENTRY_VALIDATE_FS (fs, invalid_fs);
+    cmd_args_t *cmd_args = NULL;
+    server_cmdline_t *server = NULL;
+    server_cmdline_t *tmp = NULL;
+    char *transport_val = NULL;
+    int port_val = 0;
+    int ret = -1;
+
+    if (!fs || !host) {
+        errno = EINVAL;
+        return ret;
+    }
 
-        cmd_args = &fs->ctx->cmd_args;
+    DECLARE_OLD_THIS;
+    __GLFS_ENTRY_VALIDATE_FS(fs, invalid_fs);
 
-        if (transport) {
-                transport_val = gf_strdup (transport);
-        } else {
-                transport_val = gf_strdup (GF_DEFAULT_VOLFILE_TRANSPORT);
-        }
+    cmd_args = &fs->ctx->cmd_args;
 
-        if (!transport_val) {
-                errno = ENOMEM;
-                goto out;
-        }
+    if (transport) {
+        transport_val = gf_strdup(transport);
+    } else {
+        transport_val = gf_strdup(GF_DEFAULT_VOLFILE_TRANSPORT);
+    }
 
-        if (port) {
-                port_val = port;
-        } else {
-                port_val = GF_DEFAULT_BASE_PORT;
-        }
-
-        list_for_each_entry_safe (server, tmp,
-                                  &cmd_args->curr_server->list,
-                                  list) {
-                if ((!strcmp(server->volfile_server, host) &&
-                     !strcmp(server->transport, transport_val) &&
-                     (server->port == port_val))) {
-                        list_del (&server->list);
-                        ret = 0;
-                        goto out;
-                }
+    if (!transport_val) {
+        errno = ENOMEM;
+        goto out;
+    }
+
+    if (port) {
+        port_val = port;
+    } else {
+        port_val = GF_DEFAULT_BASE_PORT;
+    }
+
+    list_for_each_entry_safe(server, tmp, &cmd_args->curr_server->list, list)
+    {
+        if (!server->volfile_server || !server->transport)
+            continue;
+        if ((!strcmp(server->volfile_server, host) &&
+             !strcmp(server->transport, transport_val) &&
+             (server->port == port_val))) {
+            list_del(&server->list);
+            ret = 0;
+            goto out;
         }
+    }
 
 out:
-        GF_FREE (transport_val);
-        __GLFS_EXIT_FS;
+    GF_FREE(transport_val);
+    __GLFS_EXIT_FS;
 
 invalid_fs:
-        return ret;
+    return ret;
 }
 
-GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_unset_volfile_server, 3.5.1);
-
-
+GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_set_volfile_server, 3.4.0)
 int
-pub_glfs_set_volfile_server (struct glfs *fs, const char *transport,
-                             const char *host, int port)
+pub_glfs_set_volfile_server(struct glfs *fs, const char *transport,
+                            const char *host, int port)
 {
-        cmd_args_t            *cmd_args = NULL;
-        server_cmdline_t      *server = NULL;
-        server_cmdline_t      *tmp = NULL;
-        int                    ret = -1;
-
-        if (!fs || !host) {
-                errno = EINVAL;
-                return ret;
-        }
-
-        DECLARE_OLD_THIS;
-        __GLFS_ENTRY_VALIDATE_FS (fs, invalid_fs);
+    cmd_args_t *cmd_args = NULL;
+    int ret = -1;
+    char *server_host = NULL;
+    char *server_transport = NULL;
 
-        cmd_args = &fs->ctx->cmd_args;
-
-        cmd_args->max_connect_attempts = 1;
+    if (!fs || !host) {
+        errno = EINVAL;
+        return ret;
+    }
 
-        server = GF_CALLOC (1, sizeof (server_cmdline_t),
-                            glfs_mt_server_cmdline_t);
+    DECLARE_OLD_THIS;
+    __GLFS_ENTRY_VALIDATE_FS(fs, invalid_fs);
 
-        if (!server) {
-                errno = ENOMEM;
-                goto out;
-        }
+    cmd_args = &fs->ctx->cmd_args;
+    cmd_args->max_connect_attempts = 1;
 
-        INIT_LIST_HEAD (&server->list);
-
-        server->volfile_server = gf_strdup (host);
-        if (!server->volfile_server) {
-                errno = ENOMEM;
-                goto out;
-        }
-
-        if (transport) {
-                server->transport = gf_strdup (transport);
+    server_host = gf_strdup(host);
+    if (!server_host) {
+        errno = ENOMEM;
+        goto out;
+    }
+
+    if (transport) {
+        /* volfile fetch support over tcp|unix only */
+        if (!strcmp(transport, "tcp") || !strcmp(transport, "unix")) {
+            server_transport = gf_strdup(transport);
+        } else if (!strcmp(transport, "rdma")) {
+            server_transport = gf_strdup(GF_DEFAULT_VOLFILE_TRANSPORT);
+            gf_smsg("glfs", GF_LOG_WARNING, EINVAL, API_MSG_TRANS_RDMA_DEP,
+                    NULL);
         } else {
-                server->transport = gf_strdup (GF_DEFAULT_VOLFILE_TRANSPORT);
+            gf_smsg("glfs", GF_LOG_TRACE, EINVAL, API_MSG_TRANS_NOT_SUPPORTED,
+                    "transport=%s", transport, NULL);
+            goto out;
         }
+    } else {
+        server_transport = gf_strdup(GF_DEFAULT_VOLFILE_TRANSPORT);
+    }
 
-        if (!server->transport) {
-                errno = ENOMEM;
-                goto out;
-        }
-
-        if (port) {
-                server->port = port;
-        } else {
-                server->port = GF_DEFAULT_BASE_PORT;
-        }
+    if (!server_transport) {
+        errno = ENOMEM;
+        goto out;
+    }
 
-        if (!cmd_args->volfile_server) {
-                cmd_args->volfile_server = server->volfile_server;
-                cmd_args->volfile_server_transport = server->transport;
-                cmd_args->volfile_server_port = server->port;
-                cmd_args->curr_server = server;
-        }
+    if (!port) {
+        port = GF_DEFAULT_BASE_PORT;
+    }
 
-        list_for_each_entry(tmp, &cmd_args->volfile_servers, list) {
-                if ((!strcmp(tmp->volfile_server, server->volfile_server) &&
-                     !strcmp(tmp->transport, server->transport) &&
-                     (tmp->port == server->port))) {
-                        errno = EEXIST;
-                        ret = -1;
-                        goto out;
-                }
-        }
+    if (!strcmp(server_transport, "unix")) {
+        port = 0;
+    }
 
-        list_add_tail (&server->list, &cmd_args->volfile_servers);
+    ret = gf_set_volfile_server_common(cmd_args, server_host, server_transport,
+                                       port);
+    if (ret) {
+        gf_log("glfs", GF_LOG_ERROR, "failed to set volfile server: %s",
+               strerror(errno));
+    }
 
-        ret = 0;
 out:
-        if (ret == -1) {
-                if (server) {
-                        GF_FREE (server->volfile_server);
-                        GF_FREE (server->transport);
-                        GF_FREE (server);
-                }
-        }
+    if (server_host) {
+        GF_FREE(server_host);
+    }
 
-        __GLFS_EXIT_FS;
+    if (server_transport) {
+        GF_FREE(server_transport);
+    }
+
+    __GLFS_EXIT_FS;
 
 invalid_fs:
-        return ret;
+    return ret;
 }
 
-GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_set_volfile_server, 3.4.0);
+/* *
+ * Used to free the arguments allocated by glfs_set_volfile_server()
+ */
+static void
+glfs_free_volfile_servers(cmd_args_t *cmd_args)
+{
+    server_cmdline_t *server = NULL;
+    server_cmdline_t *tmp = NULL;
+
+    GF_VALIDATE_OR_GOTO(THIS->name, cmd_args, out);
+
+    list_for_each_entry_safe(server, tmp, &cmd_args->volfile_servers, list)
+    {
+        list_del_init(&server->list);
+        GF_FREE(server->volfile_server);
+        GF_FREE(server->transport);
+        GF_FREE(server);
+    }
+    cmd_args->curr_server = NULL;
+out:
+    return;
+}
 
+static void
+glfs_free_xlator_options(cmd_args_t *cmd_args)
+{
+    xlator_cmdline_option_t *xo = NULL;
+    xlator_cmdline_option_t *tmp_xo = NULL;
+
+    if (!&(cmd_args->xlator_options))
+        return;
+
+    list_for_each_entry_safe(xo, tmp_xo, &cmd_args->xlator_options, cmd_args)
+    {
+        list_del_init(&xo->cmd_args);
+        GF_FREE(xo->volume);
+        GF_FREE(xo->key);
+        GF_FREE(xo->value);
+        GF_FREE(xo);
+    }
+}
 
+GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_setfsuid, 3.4.2)
 int
-pub_glfs_setfsuid (uid_t fsuid)
+pub_glfs_setfsuid(uid_t fsuid)
 {
-         /* TODO:
-         * - Set the THIS and restore it appropriately
-         */
-	return syncopctx_setfsuid (&fsuid);
+    /* TODO:
+     * - Set the THIS and restore it appropriately
+     */
+    return syncopctx_setfsuid(&fsuid);
 }
 
-GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_setfsuid, 3.4.2);
-
+GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_setfsgid, 3.4.2)
+int
+pub_glfs_setfsgid(gid_t fsgid)
+{
+    /* TODO:
+     * - Set the THIS and restore it appropriately
+     */
+    return syncopctx_setfsgid(&fsgid);
+}
 
+GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_setfsgroups, 3.4.2)
 int
-pub_glfs_setfsgid (gid_t fsgid)
+pub_glfs_setfsgroups(size_t size, const gid_t *list)
 {
-         /* TODO:
-         * - Set the THIS and restore it appropriately
-         */
-	return syncopctx_setfsgid (&fsgid);
+    /* TODO:
+     * - Set the THIS and restore it appropriately
+     */
+    return syncopctx_setfsgroups(size, list);
 }
 
-GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_setfsgid, 3.4.2);
+GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_setfsleaseid, 4.0.0)
+int
+pub_glfs_setfsleaseid(glfs_leaseid_t leaseid)
+{
+    int ret = -1;
+    char *gleaseid = NULL;
+
+    gleaseid = gf_leaseid_get();
+    if (gleaseid) {
+        if (leaseid)
+            memcpy(gleaseid, leaseid, LEASE_ID_SIZE);
+        else /* reset leaseid */
+            memset(gleaseid, 0, LEASE_ID_SIZE);
+        ret = 0;
+    }
 
+    if (ret)
+        gf_log("glfs", GF_LOG_ERROR, "failed to set leaseid: %s",
+               strerror(errno));
+    return ret;
+}
 
 int
-pub_glfs_setfsgroups (size_t size, const gid_t *list)
+get_fop_attr_glfd(dict_t **fop_attr, struct glfs_fd *glfd)
 {
-         /* TODO:
-         * - Set the THIS and restore it appropriately
-         */
-	return syncopctx_setfsgroups(size, list);
+    char *leaseid = NULL;
+    int ret = 0;
+    gf_boolean_t dict_create = _gf_false;
+
+    leaseid = GF_MALLOC(LEASE_ID_SIZE, gf_common_mt_char);
+    GF_CHECK_ALLOC_AND_LOG("gfapi", leaseid, ret, "lease id alloc failed", out);
+    memcpy(leaseid, glfd->lease_id, LEASE_ID_SIZE);
+    if (*fop_attr == NULL) {
+        *fop_attr = dict_new();
+        dict_create = _gf_true;
+    }
+    GF_CHECK_ALLOC_AND_LOG("gfapi", *fop_attr, ret, "dict_new failed", out);
+    ret = dict_set_bin(*fop_attr, "lease-id", leaseid, LEASE_ID_SIZE);
+out:
+    if (ret) {
+        GF_FREE(leaseid);
+        if (dict_create) {
+            if (*fop_attr)
+                dict_unref(*fop_attr);
+            *fop_attr = NULL;
+        }
+    }
+    return ret;
 }
 
-GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_setfsgroups, 3.4.2);
+int
+set_fop_attr_glfd(struct glfs_fd *glfd)
+{
+    char *lease_id = NULL;
+    int ret = -1;
 
+    lease_id = gf_existing_leaseid();
+    if (lease_id) {
+        memcpy(glfd->lease_id, lease_id, LEASE_ID_SIZE);
+        ret = 0;
+    }
+    return ret;
+}
 
-struct glfs *
-pub_glfs_from_glfd (struct glfs_fd *glfd)
+int
+get_fop_attr_thrd_key(dict_t **fop_attr)
 {
-	return glfd->fs;
+    char *existing_leaseid = NULL, *leaseid = NULL;
+    int ret = 0;
+    gf_boolean_t dict_create = _gf_false;
+
+    existing_leaseid = gf_existing_leaseid();
+    if (existing_leaseid) {
+        leaseid = GF_MALLOC(LEASE_ID_SIZE, gf_common_mt_char);
+        GF_CHECK_ALLOC_AND_LOG("gfapi", leaseid, ret, "lease id alloc failed",
+                               out);
+        memcpy(leaseid, existing_leaseid, LEASE_ID_SIZE);
+        if (*fop_attr == NULL) {
+            *fop_attr = dict_new();
+            dict_create = _gf_true;
+        }
+        GF_CHECK_ALLOC_AND_LOG("gfapi", *fop_attr, ret, "dict_new failed", out);
+        ret = dict_set_bin(*fop_attr, "lease-id", leaseid, LEASE_ID_SIZE);
+    }
+
+out:
+    if (ret) {
+        GF_FREE(leaseid);
+        if (dict_create) {
+            if (*fop_attr)
+                dict_unref(*fop_attr);
+            *fop_attr = NULL;
+        }
+    }
+    return ret;
 }
 
-GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_from_glfd, 3.4.0);
+void
+unset_fop_attr(dict_t **fop_attr)
+{
+    char *lease_id = NULL;
+    lease_id = gf_existing_leaseid();
+    if (lease_id)
+        memset(lease_id, 0, LEASE_ID_SIZE);
+    if (*fop_attr) {
+        dict_unref(*fop_attr);
+        *fop_attr = NULL;
+    }
+}
 
+GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_from_glfd, 3.4.0)
+struct glfs *
+pub_glfs_from_glfd(struct glfs_fd *glfd)
+{
+    if (glfd == NULL) {
+        errno = EBADF;
+        return NULL;
+    }
 
-struct glfs_fd *
-glfs_fd_new (struct glfs *fs)
+    return glfd->fs;
+}
+
+static void
+glfs_fd_destroy(struct glfs_fd *glfd)
 {
-	struct glfs_fd  *glfd = NULL;
+    if (!glfd)
+        return;
 
-	glfd = GF_CALLOC (1, sizeof (*glfd), glfs_mt_glfs_fd_t);
-	if (!glfd)
-		return NULL;
+    glfs_lock(glfd->fs, _gf_true);
+    {
+        list_del_init(&glfd->openfds);
+    }
+    glfs_unlock(glfd->fs);
 
-	glfd->fs = fs;
+    if (glfd->fd) {
+        fd_unref(glfd->fd);
+        glfd->fd = NULL;
+    }
 
-	INIT_LIST_HEAD (&glfd->openfds);
+    GF_FREE(glfd->readdirbuf);
 
-	return glfd;
+    GF_FREE(glfd);
 }
 
-
-void
-glfs_fd_bind (struct glfs_fd *glfd)
+struct glfs_fd *
+glfs_fd_new(struct glfs *fs)
 {
-	struct glfs *fs = NULL;
+    struct glfs_fd *glfd = NULL;
 
-	fs = glfd->fs;
+    glfd = GF_CALLOC(1, sizeof(*glfd), glfs_mt_glfs_fd_t);
+    if (!glfd)
+        return NULL;
+
+    glfd->fs = fs;
+
+    INIT_LIST_HEAD(&glfd->openfds);
 
-	glfs_lock (fs);
-	{
-		list_add_tail (&glfd->openfds, &fs->openfds);
-	}
-	glfs_unlock (fs);
+    GF_REF_INIT(glfd, glfs_fd_destroy);
+
+    return glfd;
 }
 
 void
-glfs_fd_destroy (struct glfs_fd *glfd)
+glfs_fd_bind(struct glfs_fd *glfd)
 {
-	if (!glfd)
-		return;
-
-	glfs_lock (glfd->fs);
-	{
-		list_del_init (&glfd->openfds);
-	}
-	glfs_unlock (glfd->fs);
-
-        if (glfd->fd) {
-                fd_unref (glfd->fd);
-                glfd->fd = NULL;
-        }
+    struct glfs *fs = NULL;
 
-	GF_FREE (glfd->readdirbuf);
+    fs = glfd->fs;
 
-	GF_FREE (glfd);
+    glfs_lock(fs, _gf_true);
+    {
+        list_add_tail(&glfd->openfds, &fs->openfds);
+    }
+    glfs_unlock(fs);
 }
 
-
 static void *
-glfs_poller (void *data)
+glfs_poller(void *data)
 {
-	struct glfs  *fs = NULL;
+    struct glfs *fs = NULL;
 
-	fs = data;
+    fs = data;
 
-	event_dispatch (fs->ctx->event_pool);
+    gf_event_dispatch(fs->ctx->event_pool);
 
-	return NULL;
+    return NULL;
 }
 
 static struct glfs *
-glfs_new_fs (const char *volname)
+glfs_new_fs(const char *volname)
 {
-        struct glfs     *fs             = NULL;
+    struct glfs *fs = NULL;
 
-        fs = CALLOC (1, sizeof (*fs));
-        if (!fs)
-                return NULL;
+    fs = CALLOC(1, sizeof(*fs));
+    if (!fs)
+        return NULL;
 
-        INIT_LIST_HEAD (&fs->openfds);
-        INIT_LIST_HEAD (&fs->upcall_list);
+    INIT_LIST_HEAD(&fs->openfds);
+    INIT_LIST_HEAD(&fs->upcall_list);
+    INIT_LIST_HEAD(&fs->waitq);
 
-        PTHREAD_MUTEX_INIT (&fs->mutex, NULL, fs->pthread_flags,
-                            GLFS_INIT_MUTEX, err);
+    PTHREAD_MUTEX_INIT(&fs->mutex, NULL, fs->pthread_flags, GLFS_INIT_MUTEX,
+                       err);
 
-        PTHREAD_COND_INIT (&fs->cond, NULL, fs->pthread_flags,
-                           GLFS_INIT_COND, err);
+    PTHREAD_COND_INIT(&fs->cond, NULL, fs->pthread_flags, GLFS_INIT_COND, err);
 
-        PTHREAD_COND_INIT (&fs->child_down_cond, NULL, fs->pthread_flags,
-                           GLFS_INIT_COND_CHILD, err);
+    PTHREAD_COND_INIT(&fs->child_down_cond, NULL, fs->pthread_flags,
+                      GLFS_INIT_COND_CHILD, err);
 
-        PTHREAD_MUTEX_INIT (&fs->upcall_list_mutex, NULL, fs->pthread_flags,
-                            GLFS_INIT_MUTEX_UPCALL, err);
+    PTHREAD_MUTEX_INIT(&fs->upcall_list_mutex, NULL, fs->pthread_flags,
+                       GLFS_INIT_MUTEX_UPCALL, err);
 
-        fs->volname = strdup (volname);
-        if (!fs->volname)
-                goto err;
+    fs->volname = strdup(volname);
+    if (!fs->volname)
+        goto err;
 
-        fs->pin_refcnt = 0;
+    fs->pin_refcnt = 0;
+    fs->upcall_events = 0;
+    fs->up_cbk = NULL;
+    fs->up_data = NULL;
 
-        return fs;
+    return fs;
 
 err:
-        glfs_free_from_ctx (fs);
-        return NULL;
+    glfs_free_from_ctx(fs);
+    return NULL;
 }
 
+extern xlator_t global_xlator;
+extern glusterfs_ctx_t *global_ctx;
+extern pthread_mutex_t global_ctx_mutex;
+
+static int
+glfs_init_global_ctx()
+{
+    int ret = 0;
+    glusterfs_ctx_t *ctx = NULL;
+
+    pthread_mutex_lock(&global_ctx_mutex);
+    {
+        if (global_xlator.ctx)
+            goto unlock;
+
+        ctx = glusterfs_ctx_new();
+        if (!ctx) {
+            ret = -1;
+            goto unlock;
+        }
+
+        gf_log_globals_init(ctx, GF_LOG_NONE);
+
+        global_ctx = ctx;
+        global_xlator.ctx = global_ctx;
+
+        ret = glusterfs_ctx_defaults_init(ctx);
+        if (ret) {
+            global_ctx = NULL;
+            global_xlator.ctx = NULL;
+            goto unlock;
+        }
+    }
+unlock:
+    pthread_mutex_unlock(&global_ctx_mutex);
+
+    if (ret)
+        FREE(ctx);
+
+    return ret;
+}
+
+GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_new, 3.4.0)
 struct glfs *
-pub_glfs_new (const char *volname)
+pub_glfs_new(const char *volname)
 {
-	struct glfs     *fs             = NULL;
-	int              ret            = -1;
-	glusterfs_ctx_t *ctx            = NULL;
-        xlator_t        *old_THIS       = NULL;
-
-        if (!volname) {
-                errno = EINVAL;
-                return NULL;
+    if (!volname) {
+        errno = EINVAL;
+        return NULL;
+    }
+
+    struct glfs *fs = NULL;
+    int i = 0;
+    int ret = -1;
+    glusterfs_ctx_t *ctx = NULL;
+    xlator_t *old_THIS = NULL;
+    char pname[16] = "";
+    char msg[32] = "";
+
+    if (volname[0] == '/' || volname[0] == '-') {
+        if (strncmp(volname, "/snaps/", 7) == 0) {
+            goto label;
         }
+        errno = EINVAL;
+        return NULL;
+    }
 
-        fs = glfs_new_fs (volname);
-        if (!fs)
-                return NULL;
+    for (i = 0; i < strlen(volname); i++) {
+        if (!isalnum(volname[i]) && (volname[i] != '_') &&
+            (volname[i] != '-')) {
+            errno = EINVAL;
+            return NULL;
+        }
+    }
 
-        ctx = glusterfs_ctx_new ();
-        if (!ctx)
-                goto fini;
+label:
+    /*
+     * Do this as soon as possible in case something else depends on
+     * pool allocations.
+     */
+    mem_pools_init();
 
-        /* first globals init, for gf_mem_acct_enable_set () */
+    fs = glfs_new_fs(volname);
+    if (!fs)
+        goto out;
 
-        ret = glusterfs_globals_init (ctx);
-        if (ret)
-                goto fini;
+    ctx = glusterfs_ctx_new();
+    if (!ctx)
+        goto out;
 
-        old_THIS = THIS;
-        /* THIS is set to NULL so that we do not modify the caller xlators'
-         * ctx, instead we set the global_xlator->ctx
-         */
-        THIS = NULL;
-        THIS->ctx = ctx;
+    /* first globals init, for gf_mem_acct_enable_set () */
 
-        /* then ctx_defaults_init, for xlator_mem_acct_init(THIS) */
+    ret = glusterfs_globals_init(ctx);
+    if (ret)
+        goto out;
 
-        ret = glusterfs_ctx_defaults_init (ctx);
-        if (ret)
-                goto fini;
+    old_THIS = THIS;
+    ret = glfs_init_global_ctx();
+    if (ret)
+        goto out;
 
-        fs->ctx = ctx;
+    /* then ctx_defaults_init, for xlator_mem_acct_init(THIS) */
 
-        ret = glfs_set_logging (fs, "/dev/null", 0);
-        if (ret)
-                goto fini;
+    ret = glusterfs_ctx_defaults_init(ctx);
+    if (ret)
+        goto out;
 
-        fs->ctx->cmd_args.volfile_id = gf_strdup (volname);
-        if (!(fs->ctx->cmd_args.volfile_id))
-                goto fini;
+    fs->ctx = ctx;
+    fs->ctx->process_mode = GF_CLIENT_PROCESS;
 
+    ret = glfs_set_logging(fs, "/dev/null", 0);
+    if (ret)
         goto out;
 
-fini:
-         glfs_fini (fs);
-         fs = NULL;
-out:
-        if (old_THIS)
-                THIS = old_THIS;
+    fs->ctx->cmd_args.volfile_id = gf_strdup(volname);
+    if (!(fs->ctx->cmd_args.volfile_id)) {
+        ret = -1;
+        goto out;
+    }
+
+    ret = -1;
+#ifdef GF_LINUX_HOST_OS
+    ret = prctl(PR_GET_NAME, (unsigned long)pname, 0, 0, 0);
+#endif
+    if (ret)
+        fs->ctx->cmd_args.process_name = gf_strdup("gfapi");
+    else {
+        snprintf(msg, sizeof(msg), "gfapi.%s", pname);
+        fs->ctx->cmd_args.process_name = gf_strdup(msg);
+    }
+    ret = 0;
 
-        return fs;
-}
+out:
+    if (ret) {
+        if (fs) {
+            glfs_fini(fs);
+            fs = NULL;
+        } else {
+            /* glfs_fini() calls mem_pools_fini() too */
+            mem_pools_fini();
+        }
+    }
 
-GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_new, 3.4.0);
+    if (old_THIS)
+        THIS = old_THIS;
 
+    return fs;
+}
 
+GFAPI_SYMVER_PRIVATE_DEFAULT(glfs_new_from_ctx, 3.7.0)
 struct glfs *
-priv_glfs_new_from_ctx (glusterfs_ctx_t *ctx)
+priv_glfs_new_from_ctx(glusterfs_ctx_t *ctx)
 {
-        struct glfs    *fs = NULL;
+    struct glfs *fs = NULL;
 
-        if (!ctx)
-                goto out;
+    if (!ctx)
+        goto out;
 
-        fs = glfs_new_fs ("");
-        if (!fs)
-                goto out;
+    fs = glfs_new_fs("");
+    if (!fs)
+        goto out;
 
-        fs->ctx = ctx;
+    fs->ctx = ctx;
 
 out:
-        return fs;
+    return fs;
 }
 
-GFAPI_SYMVER_PRIVATE_DEFAULT(glfs_new_from_ctx, 3.7.0);
-
-
+GFAPI_SYMVER_PRIVATE_DEFAULT(glfs_free_from_ctx, 3.7.0)
 void
-priv_glfs_free_from_ctx (struct glfs *fs)
+priv_glfs_free_from_ctx(struct glfs *fs)
 {
-        upcall_entry       *u_list   = NULL;
-        upcall_entry       *tmp      = NULL;
-
-        if (!fs)
-                return;
-
-        /* cleanup upcall structures */
-        list_for_each_entry_safe (u_list, tmp,
-                                  &fs->upcall_list,
-                                  upcall_list) {
-                list_del_init (&u_list->upcall_list);
-                GF_FREE (u_list->upcall_data.data);
-                GF_FREE (u_list);
-        }
+    upcall_entry *u_list = NULL;
+    upcall_entry *tmp = NULL;
 
-        PTHREAD_MUTEX_DESTROY (&fs->mutex, fs->pthread_flags, GLFS_INIT_MUTEX);
+    if (!fs)
+        return;
 
-        PTHREAD_COND_DESTROY (&fs->cond, fs->pthread_flags, GLFS_INIT_COND);
+    /* cleanup upcall structures */
+    list_for_each_entry_safe(u_list, tmp, &fs->upcall_list, upcall_list)
+    {
+        list_del_init(&u_list->upcall_list);
+        GF_FREE(u_list->upcall_data.data);
+        GF_FREE(u_list);
+    }
 
-        PTHREAD_COND_DESTROY (&fs->child_down_cond, fs->pthread_flags,
-                              GLFS_INIT_COND_CHILD);
+    PTHREAD_MUTEX_DESTROY(&fs->mutex, fs->pthread_flags, GLFS_INIT_MUTEX);
 
-        PTHREAD_MUTEX_DESTROY (&fs->upcall_list_mutex, fs->pthread_flags,
-                               GLFS_INIT_MUTEX_UPCALL);
+    PTHREAD_COND_DESTROY(&fs->cond, fs->pthread_flags, GLFS_INIT_COND);
 
-        FREE (fs->volname);
+    PTHREAD_COND_DESTROY(&fs->child_down_cond, fs->pthread_flags,
+                         GLFS_INIT_COND_CHILD);
 
-        FREE (fs);
-}
+    PTHREAD_MUTEX_DESTROY(&fs->upcall_list_mutex, fs->pthread_flags,
+                          GLFS_INIT_MUTEX_UPCALL);
 
-GFAPI_SYMVER_PRIVATE_DEFAULT(glfs_free_from_ctx, 3.7.0);
+    if (fs->oldvolfile)
+        FREE(fs->oldvolfile);
 
+    FREE(fs->volname);
 
+    FREE(fs);
+}
+
+GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_set_volfile, 3.4.0)
 int
-pub_glfs_set_volfile (struct glfs *fs, const char *volfile)
+pub_glfs_set_volfile(struct glfs *fs, const char *volfile)
 {
-	cmd_args_t  *cmd_args = NULL;
+    cmd_args_t *cmd_args = NULL;
 
-	cmd_args = &fs->ctx->cmd_args;
+    cmd_args = &fs->ctx->cmd_args;
 
-	if (vol_assigned (cmd_args))
-		return -1;
+    if (vol_assigned(cmd_args))
+        return -1;
 
-	cmd_args->volfile = gf_strdup (volfile);
-        if (!cmd_args->volfile)
-                return -1;
-	return 0;
+    cmd_args->volfile = gf_strdup(volfile);
+    if (!cmd_args->volfile)
+        return -1;
+    return 0;
 }
 
-GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_set_volfile, 3.4.0);
-
-
+GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_set_logging, 3.4.0)
 int
-pub_glfs_set_logging (struct glfs *fs, const char *logfile, int loglevel)
+pub_glfs_set_logging(struct glfs *fs, const char *logfile, int loglevel)
 {
-        int  ret = -1;
-        char *tmplog = NULL;
-
-        DECLARE_OLD_THIS;
-        __GLFS_ENTRY_VALIDATE_FS (fs, invalid_fs);
-
-        if (!logfile) {
-                ret = gf_set_log_file_path (&fs->ctx->cmd_args);
-                if (ret)
-                        goto out;
-                tmplog = fs->ctx->cmd_args.log_file;
-        } else {
-                tmplog = (char *)logfile;
-        }
+    int ret = -1;
+    char *tmplog = NULL;
 
-        /* finish log set parameters before init */
-        if (loglevel >= 0)
-                gf_log_set_loglevel (loglevel);
+    DECLARE_OLD_THIS;
+    __GLFS_ENTRY_VALIDATE_FS(fs, invalid_fs);
 
-        ret = gf_log_init (fs->ctx, tmplog, NULL);
+    if (!logfile) {
+        ret = gf_set_log_file_path(&fs->ctx->cmd_args, fs->ctx);
         if (ret)
-                goto out;
+            goto out;
+        tmplog = fs->ctx->cmd_args.log_file;
+    } else {
+        tmplog = (char *)logfile;
+    }
+
+    /* finish log set parameters before init */
+    if (loglevel >= 0)
+        gf_log_set_loglevel(fs->ctx, loglevel);
+
+    ret = gf_log_init(fs->ctx, tmplog, NULL);
+    if (ret)
+        goto out;
 
-        ret = gf_log_inject_timer_event (fs->ctx);
-        if (ret)
-                goto out;
+    ret = gf_log_inject_timer_event(fs->ctx);
+    if (ret)
+        goto out;
 
 out:
-        __GLFS_EXIT_FS;
+    __GLFS_EXIT_FS;
 
 invalid_fs:
-        return ret;
+    return ret;
 }
 
-GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_set_logging, 3.4.0);
-
-
 int
-glfs_init_wait (struct glfs *fs)
+glfs_init_wait(struct glfs *fs)
 {
-	int   ret = -1;
-
-	/* Always a top-down call, use glfs_lock() */
-	glfs_lock (fs);
-	{
-		while (!fs->init)
-			pthread_cond_wait (&fs->cond,
-					   &fs->mutex);
-		ret = fs->ret;
-		errno = fs->err;
-	}
-	glfs_unlock (fs);
-
-	return ret;
+    int ret = -1;
+
+    /* Always a top-down call, use glfs_lock() */
+    glfs_lock(fs, _gf_true);
+    {
+        while (!fs->init)
+            pthread_cond_wait(&fs->cond, &fs->mutex);
+        ret = fs->ret;
+        errno = fs->err;
+    }
+    glfs_unlock(fs);
+
+    return ret;
 }
 
-
+GFAPI_SYMVER_PRIVATE_DEFAULT(glfs_init_done, 3.4.0)
 void
-priv_glfs_init_done (struct glfs *fs, int ret)
+priv_glfs_init_done(struct glfs *fs, int ret)
 {
-	glfs_init_cbk init_cbk;
-
-	if (!fs) {
-		gf_msg ("glfs", GF_LOG_ERROR, EINVAL, API_MSG_GLFS_FSOBJ_NULL,
-			"fs is NULL");
-		goto out;
-	}
-
-	init_cbk = fs->init_cbk;
-
-	/* Always a bottom-up call, use mutex_lock() */
-	pthread_mutex_lock (&fs->mutex);
-	{
-		fs->init = 1;
-		fs->ret = ret;
-		fs->err = errno;
-
-		if (!init_cbk)
-			pthread_cond_broadcast (&fs->cond);
-	}
-	pthread_mutex_unlock (&fs->mutex);
-
-	if (init_cbk)
-		init_cbk (fs, ret);
+    glfs_init_cbk init_cbk;
+
+    if (!fs) {
+        gf_smsg("glfs", GF_LOG_ERROR, EINVAL, API_MSG_GLFS_FSOBJ_NULL, NULL);
+        goto out;
+    }
+
+    init_cbk = fs->init_cbk;
+
+    /* Always a bottom-up call, use mutex_lock() */
+    pthread_mutex_lock(&fs->mutex);
+    {
+        fs->init = 1;
+        fs->ret = ret;
+        fs->err = errno;
+
+        if (!init_cbk)
+            pthread_cond_broadcast(&fs->cond);
+    }
+    pthread_mutex_unlock(&fs->mutex);
+
+    if (init_cbk)
+        init_cbk(fs, ret);
 out:
-	return;
+    return;
 }
 
-GFAPI_SYMVER_PRIVATE_DEFAULT(glfs_init_done, 3.4.0);
+int
+glfs_init_common(struct glfs *fs)
+{
+    int ret = -1;
+
+    ret = create_master(fs);
+    if (ret)
+        return ret;
+
+    ret = gf_thread_create(&fs->poller, NULL, glfs_poller, fs, "glfspoll");
+    if (ret)
+        return ret;
+
+    ret = glfs_volumes_init(fs);
+    if (ret)
+        return ret;
 
+    fs->dev_id = gf_dm_hashfn(fs->volname, strlen(fs->volname));
+    return ret;
+}
 
 int
-glfs_init_common (struct glfs *fs)
+glfs_init_async(struct glfs *fs, glfs_init_cbk cbk)
 {
-	int  ret = -1;
+    int ret = -1;
 
-	ret = create_master (fs);
-	if (ret)
-		return ret;
+    if (!fs || !fs->ctx) {
+        gf_smsg("glfs", GF_LOG_ERROR, EINVAL, API_MSG_FS_NOT_INIT, NULL);
+        errno = EINVAL;
+        return ret;
+    }
 
-	ret = gf_thread_create (&fs->poller, NULL, glfs_poller, fs);
-	if (ret)
-		return ret;
+    fs->init_cbk = cbk;
 
-	ret = glfs_volumes_init (fs);
-	if (ret)
-		return ret;
+    ret = glfs_init_common(fs);
 
-	fs->dev_id = gf_dm_hashfn (fs->volname, strlen (fs->volname));
-	return ret;
+    return ret;
 }
 
-
+GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_init, 3.4.0)
 int
-glfs_init_async (struct glfs *fs, glfs_init_cbk cbk)
+pub_glfs_init(struct glfs *fs)
 {
-	int  ret = -1;
+    int ret = -1;
 
-	if (!fs || !fs->ctx) {
-		gf_msg ("glfs", GF_LOG_ERROR, EINVAL, API_MSG_INVALID_ENTRY,
-			"fs is not properly initialized.");
-		errno = EINVAL;
-		return ret;
-	}
+    DECLARE_OLD_THIS;
+
+    if (!fs || !fs->ctx) {
+        gf_smsg("glfs", GF_LOG_ERROR, EINVAL, API_MSG_FS_NOT_INIT, NULL);
+        errno = EINVAL;
+        return ret;
+    }
+
+    __GLFS_ENTRY_VALIDATE_FS(fs, invalid_fs);
+
+    ret = glfs_init_common(fs);
+    if (ret)
+        goto out;
 
-	fs->init_cbk = cbk;
+    ret = glfs_init_wait(fs);
+out:
+    __GLFS_EXIT_FS;
 
-	ret = glfs_init_common (fs);
+    /* Set the initial current working directory to "/" */
+    if (ret >= 0) {
+        ret = glfs_chdir(fs, "/");
+    }
 
-	return ret;
+invalid_fs:
+    return ret;
 }
 
+static int
+glusterfs_ctx_destroy(glusterfs_ctx_t *ctx)
+{
+    call_pool_t *pool = NULL;
+    int ret = 0;
+    glusterfs_graph_t *trav_graph = NULL;
+    glusterfs_graph_t *tmp = NULL;
+
+    if (ctx == NULL)
+        return 0;
+
+    if (ctx->cmd_args.curr_server)
+        glfs_free_volfile_servers(&ctx->cmd_args);
+
+    glfs_free_xlator_options(&ctx->cmd_args);
+
+    /* For all the graphs, crawl through the xlator_t structs and free
+     * all its members except for the mem_acct member,
+     * as GF_FREE will be referencing it.
+     */
+    list_for_each_entry_safe(trav_graph, tmp, &ctx->graphs, list)
+    {
+        xlator_tree_free_members(trav_graph->first);
+    }
+
+    /* Free the memory pool */
+    if (ctx->stub_mem_pool)
+        mem_pool_destroy(ctx->stub_mem_pool);
+    if (ctx->dict_pool)
+        mem_pool_destroy(ctx->dict_pool);
+    if (ctx->dict_data_pool)
+        mem_pool_destroy(ctx->dict_data_pool);
+    if (ctx->dict_pair_pool)
+        mem_pool_destroy(ctx->dict_pair_pool);
+    if (ctx->logbuf_pool)
+        mem_pool_destroy(ctx->logbuf_pool);
+
+    pool = ctx->pool;
+    if (pool) {
+        if (pool->frame_mem_pool)
+            mem_pool_destroy(pool->frame_mem_pool);
+        if (pool->stack_mem_pool)
+            mem_pool_destroy(pool->stack_mem_pool);
+        LOCK_DESTROY(&pool->lock);
+        GF_FREE(pool);
+    }
+
+    /* Free the event pool */
+    ret = gf_event_pool_destroy(ctx->event_pool);
+
+    /* Free the iobuf pool */
+    iobuf_pool_destroy(ctx->iobuf_pool);
+
+    GF_FREE(ctx->process_uuid);
+    GF_FREE(ctx->cmd_args.volfile_id);
+    GF_FREE(ctx->cmd_args.process_name);
+
+    LOCK_DESTROY(&ctx->lock);
+    pthread_mutex_destroy(&ctx->notify_lock);
+    pthread_cond_destroy(&ctx->notify_cond);
+
+    /* Free all the graph structs and its containing xlator_t structs
+     * from this point there should be no reference to GF_FREE/GF_CALLOC
+     * as it will try to access mem_acct and the below function would
+     * have freed the same.
+     */
+    list_for_each_entry_safe(trav_graph, tmp, &ctx->graphs, list)
+    {
+        glusterfs_graph_destroy_residual(trav_graph);
+    }
+
+    GF_FREE(ctx->statedump_path);
+    FREE(ctx);
+
+    return ret;
+}
 
+GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_fini, 3.4.0)
 int
-pub_glfs_init (struct glfs *fs)
+pub_glfs_fini(struct glfs *fs)
 {
-	int  ret = -1;
+    int ret = -1;
+    int countdown = 100;
+    xlator_t *subvol = NULL;
+    glusterfs_ctx_t *ctx = NULL;
+    glusterfs_graph_t *graph = NULL;
+    call_pool_t *call_pool = NULL;
+    int fs_init = 0;
+    int err = -1;
+    struct synctask *waittask = NULL;
+
+    DECLARE_OLD_THIS;
+
+    if (!fs) {
+        errno = EINVAL;
+        goto invalid_fs;
+    }
+
+    ctx = fs->ctx;
+    if (!ctx) {
+        goto free_fs;
+    }
+
+    THIS = fs->ctx->master;
+
+    if (ctx->mgmt) {
+        rpc_clnt_disable(ctx->mgmt);
+    }
+
+    call_pool = fs->ctx->pool;
+
+    /* Wake up any suspended synctasks */
+    while (!list_empty(&fs->waitq)) {
+        waittask = list_entry(fs->waitq.next, struct synctask, waitq);
+        list_del_init(&waittask->waitq);
+        synctask_wake(waittask);
+    }
+
+    while (countdown--) {
+        /* give some time for background frames to finish */
+        pthread_mutex_lock(&fs->mutex);
+        {
+            /* Do we need to increase countdown? */
+            if ((!call_pool->cnt) && (!fs->pin_refcnt)) {
+                gf_msg_trace("glfs", 0,
+                             "call_pool_cnt - %" PRId64
+                             ","
+                             "pin_refcnt - %d",
+                             call_pool->cnt, fs->pin_refcnt);
+
+                ctx->cleanup_started = 1;
+                pthread_mutex_unlock(&fs->mutex);
+                break;
+            }
+        }
+        pthread_mutex_unlock(&fs->mutex);
+        gf_nanosleep(100000 * GF_US_IN_NS);
+    }
+
+    /* leaked frames may exist, we ignore */
+
+    /*We deem glfs_fini as successful if there are no pending frames in the call
+     *pool*/
+    ret = (call_pool->cnt == 0) ? 0 : -1;
+
+    pthread_mutex_lock(&fs->mutex);
+    {
+        fs_init = fs->init;
+    }
+    pthread_mutex_unlock(&fs->mutex);
+
+    if (fs_init != 0) {
+        subvol = glfs_active_subvol(fs);
+        if (subvol) {
+            /* PARENT_DOWN within glfs_subvol_done() is issued
+               only on graph switch (new graph should activiate
+               and decrement the extra @winds count taken in
+               glfs_graph_setup()
+
+               Since we are explicitly destroying,
+               PARENT_DOWN is necessary
+            */
+            xlator_notify(subvol, GF_EVENT_PARENT_DOWN, subvol, 0);
+            /* Here we wait for GF_EVENT_CHILD_DOWN before exiting,
+               in case of asynchrnous cleanup
+            */
+            graph = subvol->graph;
+            err = pthread_mutex_lock(&fs->mutex);
+            if (err != 0) {
+                gf_smsg("glfs", GF_LOG_ERROR, err, API_MSG_FSMUTEX_LOCK_FAILED,
+                        "error=%s", strerror(err), NULL);
+                goto fail;
+            }
+            /* check and wait for CHILD_DOWN for active subvol*/
+            {
+                while (graph->used) {
+                    err = pthread_cond_wait(&fs->child_down_cond, &fs->mutex);
+                    if (err != 0)
+                        gf_smsg("glfs", GF_LOG_INFO, err,
+                                API_MSG_COND_WAIT_FAILED, "name=%s",
+                                subvol->name, "err=%s", strerror(err), NULL);
+                }
+            }
+
+            err = pthread_mutex_unlock(&fs->mutex);
+            if (err != 0) {
+                gf_smsg("glfs", GF_LOG_ERROR, err,
+                        API_MSG_FSMUTEX_UNLOCK_FAILED, "error=%s",
+                        strerror(err), NULL);
+                goto fail;
+            }
+        }
+        glfs_subvol_done(fs, subvol);
+    }
+
+    ctx->cleanup_started = 1;
+
+    if (fs_init != 0) {
+        /* Destroy all the inode tables of all the graphs.
+         * NOTE:
+         * - inode objects should be destroyed before calling fini()
+         *   of each xlator, as fini() and forget() of the xlators
+         *   can share few common locks or data structures, calling
+         *   fini first might destroy those required by forget
+         *   ( eg: in quick-read)
+         * - The call to inode_table_destroy_all is not required when
+         *   the cleanup during graph switch is implemented to perform
+         *   inode table destroy.
+         */
+        inode_table_destroy_all(ctx);
+
+        /* Call fini() of all the xlators in the active graph
+         * NOTE:
+         * - xlator fini() should be called before destroying any of
+         *   the threads. (eg: fini() in protocol-client uses timer
+         *   thread) */
+        glusterfs_graph_deactivate(ctx->active);
+
+        /* Join the syncenv_processor threads and cleanup
+         * syncenv resources*/
+        syncenv_destroy(ctx->env);
+
+        /* Join the poller thread */
+        if (gf_event_dispatch_destroy(ctx->event_pool) < 0)
+            ret = -1;
+    }
+
+    /* Avoid dispatching events to mgmt after freed,
+     * unreference mgmt after the event_dispatch_destroy */
+    if (ctx->mgmt) {
+        rpc_clnt_unref(ctx->mgmt);
+        ctx->mgmt = NULL;
+    }
+
+    /* log infra has to be brought down before destroying
+     * timer registry, as logging uses timer infra
+     */
+    if (gf_log_fini(ctx) != 0)
+        ret = -1;
 
-        DECLARE_OLD_THIS;
+    /* Join the timer thread */
+    if (fs_init != 0) {
+        gf_timer_registry_destroy(ctx);
+    }
 
-	if (!fs || !fs->ctx) {
-		gf_msg ("glfs", GF_LOG_ERROR, EINVAL, API_MSG_INVALID_ENTRY,
-			"fs is not properly initialized.");
-		errno = EINVAL;
-		return ret;
-	}
+    /* Destroy the context and the global pools */
+    if (glusterfs_ctx_destroy(ctx) != 0)
+        ret = -1;
 
-        __GLFS_ENTRY_VALIDATE_FS (fs, invalid_fs);
+free_fs:
+    glfs_free_from_ctx(fs);
 
-	ret = glfs_init_common (fs);
-	if (ret)
-		goto out;
+    /*
+     * Do this as late as possible in case anything else has (or
+     * grows) a dependency on mem-pool allocations.
+     */
+    mem_pools_fini();
 
-	ret = glfs_init_wait (fs);
-out:
-        __GLFS_EXIT_FS;
+fail:
+    if (!ret)
+        ret = err;
 
-        /* Set the initial current working directory to "/" */
-        if (ret >= 0) {
-                ret = glfs_chdir (fs, "/");
-        }
+    __GLFS_EXIT_FS;
 
 invalid_fs:
-	return ret;
+    return ret;
 }
 
-GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_init, 3.4.0);
-
-static int
-glusterfs_ctx_destroy (glusterfs_ctx_t *ctx)
+GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_get_volfile, 3.6.0)
+ssize_t
+pub_glfs_get_volfile(struct glfs *fs, void *buf, size_t len)
 {
-        call_pool_t       *pool            = NULL;
-        int               ret              = 0;
-        glusterfs_graph_t *trav_graph      = NULL;
-        glusterfs_graph_t *tmp             = NULL;
+    ssize_t res = -1;
 
-        if (ctx == NULL)
-                return 0;
+    DECLARE_OLD_THIS;
+    __GLFS_ENTRY_VALIDATE_FS(fs, invalid_fs);
 
-        /* For all the graphs, crawl through the xlator_t structs and free
-         * all its members except for the mem_acct member,
-         * as GF_FREE will be referencing it.
-         */
-        list_for_each_entry_safe (trav_graph, tmp, &ctx->graphs, list) {
-                xlator_tree_free_members (trav_graph->first);
-        }
+    glfs_lock(fs, _gf_true);
+    if (len >= fs->oldvollen) {
+        gf_msg_trace("glfs", 0, "copying %zu to %p", len, buf);
+        memcpy(buf, fs->oldvolfile, len);
+        res = len;
+    } else {
+        res = len - fs->oldvollen;
+        gf_msg_trace("glfs", 0, "buffer is %zd too short", -res);
+    }
+    glfs_unlock(fs);
 
-        /* Free the memory pool */
-        if (ctx->stub_mem_pool)
-                mem_pool_destroy (ctx->stub_mem_pool);
-        if (ctx->dict_pool)
-                mem_pool_destroy (ctx->dict_pool);
-        if (ctx->dict_data_pool)
-                mem_pool_destroy (ctx->dict_data_pool);
-        if (ctx->dict_pair_pool)
-                mem_pool_destroy (ctx->dict_pair_pool);
-        if (ctx->logbuf_pool)
-                mem_pool_destroy (ctx->logbuf_pool);
-
-        pool = ctx->pool;
-        if (pool) {
-                if (pool->frame_mem_pool)
-                        mem_pool_destroy (pool->frame_mem_pool);
-                if (pool->stack_mem_pool)
-                        mem_pool_destroy (pool->stack_mem_pool);
-                LOCK_DESTROY (&pool->lock);
-                GF_FREE (pool);
-        }
+    __GLFS_EXIT_FS;
 
-        /* Free the event pool */
-        ret = event_pool_destroy (ctx->event_pool);
+invalid_fs:
+    return res;
+}
 
-        /* Free the iobuf pool */
-        iobuf_pool_destroy (ctx->iobuf_pool);
+GFAPI_SYMVER_PRIVATE_DEFAULT(glfs_ipc, 3.12.0)
+int
+priv_glfs_ipc(struct glfs *fs, int opcode, void *xd_in, void **xd_out)
+{
+    xlator_t *subvol = NULL;
+    int ret = -1;
 
-        GF_FREE (ctx->process_uuid);
-        GF_FREE (ctx->cmd_args.volfile_id);
+    DECLARE_OLD_THIS;
+    __GLFS_ENTRY_VALIDATE_FS(fs, invalid_fs);
 
-        pthread_mutex_destroy (&(ctx->lock));
-        pthread_mutex_destroy (&(ctx->notify_lock));
-        pthread_cond_destroy (&(ctx->notify_cond));
+    subvol = glfs_active_subvol(fs);
+    if (!subvol) {
+        ret = -1;
+        errno = EIO;
+        goto out;
+    }
 
-        /* Free all the graph structs and its containing xlator_t structs
-         * from this point there should be no reference to GF_FREE/GF_CALLOC
-         * as it will try to access mem_acct and the below funtion would
-         * have freed the same.
-         */
-        list_for_each_entry_safe (trav_graph, tmp, &ctx->graphs, list) {
-                glusterfs_graph_destroy_residual (trav_graph);
-        }
+    ret = syncop_ipc(subvol, opcode, (dict_t *)xd_in, (dict_t **)xd_out);
+    DECODE_SYNCOP_ERR(ret);
 
-        FREE (ctx);
+out:
+    glfs_subvol_done(fs, subvol);
+    __GLFS_EXIT_FS;
 
-        return ret;
+invalid_fs:
+    return ret;
 }
 
+GFAPI_SYMVER_PRIVATE_DEFAULT(glfs_setfspid, 6.1)
 int
-pub_glfs_fini (struct glfs *fs)
+priv_glfs_setfspid(struct glfs *fs, pid_t pid)
 {
-        int                ret = -1;
-        int                countdown = 100;
-        xlator_t           *subvol = NULL;
-        glusterfs_ctx_t    *ctx = NULL;
-        glusterfs_graph_t  *graph = NULL;
-        call_pool_t        *call_pool = NULL;
-        int                fs_init = 0;
-        int                err = -1;
+    cmd_args_t *cmd_args = NULL;
+    int ret = 0;
 
-        DECLARE_OLD_THIS;
+    cmd_args = &fs->ctx->cmd_args;
+    cmd_args->client_pid = pid;
+    cmd_args->client_pid_set = 1;
+    ret = syncopctx_setfspid(&pid);
 
-        __GLFS_ENTRY_VALIDATE_FS (fs, invalid_fs);
+    return ret;
+}
 
-        ctx = fs->ctx;
-        if (!ctx) {
-                goto free_fs;
-        }
+GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_free, 3.7.16)
+void
+pub_glfs_free(void *ptr)
+{
+    GLFS_FREE(ptr);
+}
 
-        if (ctx->mgmt) {
-                rpc_clnt_disable (ctx->mgmt);
-                ctx->mgmt = NULL;
-        }
+GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_upcall_get_fs, 3.7.16)
+struct glfs *
+pub_glfs_upcall_get_fs(struct glfs_upcall *arg)
+{
+    return arg->fs;
+}
 
-        call_pool = fs->ctx->pool;
-
-        while (countdown--) {
-                /* give some time for background frames to finish */
-                pthread_mutex_lock (&fs->mutex);
-                {
-                        /* Do we need to increase countdown? */
-                        if ((!call_pool->cnt) && (!fs->pin_refcnt)) {
-                                gf_msg_trace ("glfs", 0,
-                                        "call_pool_cnt - %ld,"
-                                        "pin_refcnt - %d",
-                                        call_pool->cnt, fs->pin_refcnt);
-
-                                ctx->cleanup_started = 1;
-                                pthread_mutex_unlock (&fs->mutex);
-                                break;
-                        }
-                }
-                pthread_mutex_unlock (&fs->mutex);
-                usleep (100000);
-        }
+GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_upcall_get_reason, 3.7.16)
+enum glfs_upcall_reason
+pub_glfs_upcall_get_reason(struct glfs_upcall *arg)
+{
+    return arg->reason;
+}
 
-        /* leaked frames may exist, we ignore */
+GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_upcall_get_event, 3.7.16)
+void *
+pub_glfs_upcall_get_event(struct glfs_upcall *arg)
+{
+    return arg->event;
+}
 
-        /*We deem glfs_fini as successful if there are no pending frames in the call
-         *pool*/
-        ret = (call_pool->cnt == 0)? 0: -1;
+GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_upcall_inode_get_object, 3.7.16)
+struct glfs_object *
+pub_glfs_upcall_inode_get_object(struct glfs_upcall_inode *arg)
+{
+    return arg->object;
+}
 
-        pthread_mutex_lock (&fs->mutex);
-        {
-                fs_init = fs->init;
-        }
-        pthread_mutex_unlock (&fs->mutex);
-
-        if (fs_init != 0) {
-                subvol = glfs_active_subvol (fs);
-                if (subvol) {
-                        /* PARENT_DOWN within glfs_subvol_done() is issued
-                           only on graph switch (new graph should activiate
-                           and decrement the extra @winds count taken in
-                           glfs_graph_setup()
-
-                           Since we are explicitly destroying,
-                           PARENT_DOWN is necessary
-                        */
-                        xlator_notify (subvol, GF_EVENT_PARENT_DOWN, subvol, 0);
-                        /* Here we wait for GF_EVENT_CHILD_DOWN before exiting,
-                           in case of asynchrnous cleanup
-                        */
-                        graph = subvol->graph;
-                        err = pthread_mutex_lock (&fs->mutex);
-                        if (err != 0) {
-                                gf_msg ("glfs", GF_LOG_ERROR, err,
-                                        API_MSG_FSMUTEX_LOCK_FAILED,
-                                        "pthread lock on glfs mutex, "
-                                        "returned error: (%s)", strerror (err));
-                                goto fail;
-                        }
-                        /* check and wait for CHILD_DOWN for active subvol*/
-                        {
-                                while (graph->used) {
-                                        err = pthread_cond_wait (&fs->child_down_cond,
-                                                                 &fs->mutex);
-                                        if (err != 0)
-                                                gf_msg ("glfs", GF_LOG_INFO, err,
-                                                        API_MSG_COND_WAIT_FAILED,
-                                                        "%s cond wait failed %s",
-                                                        subvol->name,
-                                                        strerror (err));
-                                }
-                        }
-
-                        err = pthread_mutex_unlock (&fs->mutex);
-                        if (err != 0) {
-                                gf_msg ("glfs", GF_LOG_ERROR, err,
-                                        API_MSG_FSMUTEX_UNLOCK_FAILED,
-                                        "pthread unlock on glfs mutex, "
-                                        "returned error: (%s)", strerror (err));
-                                goto fail;
-                        }
-                }
-                glfs_subvol_done (fs, subvol);
-        }
+GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_upcall_inode_get_flags, 3.7.16)
+uint64_t
+pub_glfs_upcall_inode_get_flags(struct glfs_upcall_inode *arg)
+{
+    return arg->flags;
+}
 
-        ctx->cleanup_started = 1;
-
-        if (fs_init != 0) {
-                /* Destroy all the inode tables of all the graphs.
-                 * NOTE:
-                 * - inode objects should be destroyed before calling fini()
-                 *   of each xlator, as fini() and forget() of the xlators
-                 *   can share few common locks or data structures, calling
-                 *   fini first might destroy those required by forget
-                 *   ( eg: in quick-read)
-                 * - The call to inode_table_destroy_all is not required when
-                 *   the cleanup during graph switch is implemented to perform
-                 *   inode table destroy.
-                 */
-                inode_table_destroy_all (ctx);
-
-                /* Call fini() of all the xlators in the active graph
-                 * NOTE:
-                 * - xlator fini() should be called before destroying any of
-                 *   the threads. (eg: fini() in protocol-client uses timer
-                 *   thread) */
-                glusterfs_graph_deactivate (ctx->active);
-
-                /* Join the syncenv_processor threads and cleanup
-                 * syncenv resources*/
-                syncenv_destroy (ctx->env);
-
-                /* Join the poller thread */
-                if (event_dispatch_destroy (ctx->event_pool) != 0)
-                        ret = -1;
-        }
+GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_upcall_inode_get_stat, 3.7.16)
+struct stat *
+pub_glfs_upcall_inode_get_stat(struct glfs_upcall_inode *arg)
+{
+    return &arg->buf;
+}
 
-        /* log infra has to be brought down before destroying
-         * timer registry, as logging uses timer infra
-         */
-        if (gf_log_fini (ctx) != 0)
-                ret = -1;
+GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_upcall_inode_get_expire, 3.7.16)
+uint64_t
+pub_glfs_upcall_inode_get_expire(struct glfs_upcall_inode *arg)
+{
+    return arg->expire_time_attr;
+}
 
-        /* Join the timer thread */
-        if (fs_init != 0) {
-                gf_timer_registry_destroy (ctx);
-        }
+GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_upcall_inode_get_pobject, 3.7.16)
+struct glfs_object *
+pub_glfs_upcall_inode_get_pobject(struct glfs_upcall_inode *arg)
+{
+    return arg->p_object;
+}
 
-        /* Destroy the context and the global pools */
-        if (glusterfs_ctx_destroy (ctx) != 0)
-                ret = -1;
+GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_upcall_inode_get_pstat, 3.7.16)
+struct stat *
+pub_glfs_upcall_inode_get_pstat(struct glfs_upcall_inode *arg)
+{
+    return &arg->p_buf;
+}
 
-free_fs:
-        glfs_free_from_ctx (fs);
+GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_upcall_inode_get_oldpobject, 3.7.16)
+struct glfs_object *
+pub_glfs_upcall_inode_get_oldpobject(struct glfs_upcall_inode *arg)
+{
+    return arg->oldp_object;
+}
 
-fail:
-        if (!ret)
-                ret = err;
+GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_upcall_inode_get_oldpstat, 3.7.16)
+struct stat *
+pub_glfs_upcall_inode_get_oldpstat(struct glfs_upcall_inode *arg)
+{
+    return &arg->oldp_buf;
+}
 
-        __GLFS_EXIT_FS;
+GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_upcall_lease_get_object, 4.1.6)
+struct glfs_object *
+pub_glfs_upcall_lease_get_object(struct glfs_upcall_lease *arg)
+{
+    return arg->object;
+}
 
-invalid_fs:
-        return ret;
+GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_upcall_lease_get_lease_type, 4.1.6)
+uint32_t
+pub_glfs_upcall_lease_get_lease_type(struct glfs_upcall_lease *arg)
+{
+    return arg->lease_type;
 }
 
-GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_fini, 3.4.0);
+/* definitions of the GLFS_SYSRQ_* chars are in glfs.h */
+static struct glfs_sysrq_help {
+    char sysrq;
+    char *msg;
+} glfs_sysrq_help[] = {{GLFS_SYSRQ_HELP, "(H)elp"},
+                       {GLFS_SYSRQ_STATEDUMP, "(S)tatedump"},
+                       {0, NULL}};
 
+GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_sysrq, 3.10.0)
+int
+pub_glfs_sysrq(struct glfs *fs, char sysrq)
+{
+    glusterfs_ctx_t *ctx = NULL;
+    int ret = 0;
+    int msg_len;
+    char msg[1024] = {
+        0,
+    }; /* should not exceed 1024 chars */
+
+    if (!fs || !fs->ctx) {
+        ret = -1;
+        errno = EINVAL;
+        goto out;
+    }
 
-ssize_t
-pub_glfs_get_volfile (struct glfs *fs, void *buf, size_t len)
+    ctx = fs->ctx;
+
+    switch (sysrq) {
+        case GLFS_SYSRQ_HELP: {
+            struct glfs_sysrq_help *usage = NULL;
+
+            for (usage = glfs_sysrq_help; usage->sysrq; usage++) {
+                msg_len = strlen(msg);
+                snprintf(msg + msg_len, /* append to msg */
+                         sizeof(msg) - msg_len - 2,
+                         /* - 2 for the " " + terminating \0 */
+                         " %s", usage->msg);
+            }
+
+            /* not really an 'error', but make sure it gets logged */
+            gf_log("glfs", GF_LOG_ERROR, "available events: %s", msg);
+
+            break;
+        }
+        case GLFS_SYSRQ_STATEDUMP:
+            gf_proc_dump_info(SIGUSR1, ctx);
+            break;
+        default:
+            gf_smsg("glfs", GF_LOG_ERROR, ENOTSUP, API_MSG_INVALID_SYSRQ,
+                    "sysrq=%c", sysrq, NULL);
+            errno = ENOTSUP;
+            ret = -1;
+    }
+out:
+    return ret;
+}
+
+GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_upcall_register, 3.13.0)
+int
+pub_glfs_upcall_register(struct glfs *fs, uint32_t event_list,
+                         glfs_upcall_cbk cbk, void *data)
 {
-        ssize_t         res = -1;
+    int ret = 0;
+
+    /* list of supported upcall events */
+    uint32_t up_events = (GLFS_EVENT_INODE_INVALIDATE |
+                          GLFS_EVENT_RECALL_LEASE);
 
-        DECLARE_OLD_THIS;
-        __GLFS_ENTRY_VALIDATE_FS (fs, invalid_fs);
+    DECLARE_OLD_THIS;
+    __GLFS_ENTRY_VALIDATE_FS(fs, invalid_fs);
 
-        glfs_lock(fs);
-        if (len >= fs->oldvollen) {
-                gf_msg_trace ("glfs", 0, "copying %zu to %p", len, buf);
-                memcpy(buf,fs->oldvolfile,len);
-                res = len;
+    GF_VALIDATE_OR_GOTO(THIS->name, cbk, out);
+
+    /* Event list should be either GLFS_EVENT_ANY
+     * or list of supported individual events (up_events)
+     */
+    if ((event_list != GLFS_EVENT_ANY) && (event_list & ~up_events)) {
+        errno = EINVAL;
+        ret = -1;
+        gf_smsg(THIS->name, GF_LOG_ERROR, errno, API_MSG_INVALID_ARG,
+                "event_list=(0x%08x)", event_list, NULL);
+        goto out;
+    }
+
+    /* in case other thread does unregister */
+    pthread_mutex_lock(&fs->mutex);
+    {
+        if (event_list & GLFS_EVENT_INODE_INVALIDATE) {
+            /* @todo: Check if features.cache-invalidation is
+             * enabled.
+             */
+            fs->upcall_events |= GF_UPCALL_CACHE_INVALIDATION;
+            ret |= GLFS_EVENT_INODE_INVALIDATE;
         }
-        else {
-                res = len - fs->oldvollen;
-                gf_msg_trace ("glfs", 0, "buffer is %zd too short", -res);
+        if (event_list & GLFS_EVENT_RECALL_LEASE) {
+            /* @todo: Check if features.leases is enabled */
+            fs->upcall_events |= GF_UPCALL_RECALL_LEASE;
+            ret |= GLFS_EVENT_RECALL_LEASE;
         }
-        glfs_unlock(fs);
+        /* Override cbk function if existing */
+        fs->up_cbk = cbk;
+        fs->up_data = data;
+        fs->cache_upcalls = _gf_true;
+    }
+    pthread_mutex_unlock(&fs->mutex);
 
-        __GLFS_EXIT_FS;
+out:
+    __GLFS_EXIT_FS;
 
 invalid_fs:
-        return res;
+    return ret;
 }
 
-GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_get_volfile, 3.6.0);
+GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_upcall_unregister, 3.13.0)
+int
+pub_glfs_upcall_unregister(struct glfs *fs, uint32_t event_list)
+{
+    int ret = 0;
+    /* list of supported upcall events */
+    uint32_t up_events = (GLFS_EVENT_INODE_INVALIDATE |
+                          GLFS_EVENT_RECALL_LEASE);
+
+    DECLARE_OLD_THIS;
+    __GLFS_ENTRY_VALIDATE_FS(fs, invalid_fs);
+
+    /* Event list should be either GLFS_EVENT_ANY
+     * or list of supported individual events (up_events)
+     */
+    if ((event_list != GLFS_EVENT_ANY) && (event_list & ~up_events)) {
+        errno = EINVAL;
+        ret = -1;
+        gf_smsg(THIS->name, GF_LOG_ERROR, errno, API_MSG_INVALID_ARG,
+                "event_list=(0x%08x)", event_list, NULL);
+        goto out;
+    }
+
+    pthread_mutex_lock(&fs->mutex);
+    {
+        /* We already checked if event_list contains list of supported
+         * upcall events. No other specific checks needed as of now for
+         * unregister */
+        fs->upcall_events &= ~(event_list);
+        ret |= ((event_list == GLFS_EVENT_ANY) ? up_events : event_list);
+
+        /* If there are no upcall events registered, reset cbk */
+        if (fs->upcall_events == 0) {
+            fs->up_cbk = NULL;
+            fs->up_data = NULL;
+            fs->cache_upcalls = _gf_false;
+        }
+    }
+    pthread_mutex_unlock(&fs->mutex);
+
+out:
+    __GLFS_EXIT_FS;
+
+invalid_fs:
+    return ret;
+}
 
+GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_set_statedump_path, 7.0)
 int
-pub_glfs_ipc (struct glfs *fs, int opcode)
+pub_glfs_set_statedump_path(struct glfs *fs, const char *path)
 {
-	xlator_t        *subvol = NULL;
-        int             ret = -1;
+    struct stat st;
+    int ret;
+    DECLARE_OLD_THIS;
+    __GLFS_ENTRY_VALIDATE_FS(fs, invalid_fs);
+
+    if (!path) {
+        gf_log("glfs", GF_LOG_ERROR, "path is NULL");
+        errno = EINVAL;
+        goto err;
+    }
+
+    /* If path is not present OR, if it is directory AND has enough permission
+     * to create files, then proceed */
+    ret = sys_stat(path, &st);
+    if (ret && errno != ENOENT) {
+        gf_log("glfs", GF_LOG_ERROR, "%s: not a valid path (%s)", path,
+               strerror(errno));
+        errno = EINVAL;
+        goto err;
+    }
+
+    if (!ret) {
+        /* file is present, now check other things */
+        if (!S_ISDIR(st.st_mode)) {
+            gf_log("glfs", GF_LOG_ERROR, "%s: path is not directory", path);
+            errno = EINVAL;
+            goto err;
+        }
+        if (sys_access(path, W_OK | X_OK) < 0) {
+            gf_log("glfs", GF_LOG_ERROR,
+                   "%s: path doesn't have write permission", path);
+            errno = EPERM;
+            goto err;
+        }
+    }
 
-	DECLARE_OLD_THIS;
-        __GLFS_ENTRY_VALIDATE_FS (fs, invalid_fs);
+    /* If set, it needs to be freed, so we don't have leak */
+    GF_FREE(fs->ctx->statedump_path);
 
-	subvol = glfs_active_subvol (fs);
-	if (!subvol) {
-		ret = -1;
-		errno = EIO;
-		goto out;
-	}
+    fs->ctx->statedump_path = gf_strdup(path);
+    if (!fs->ctx->statedump_path) {
+        gf_log("glfs", GF_LOG_ERROR,
+               "%s: failed to set statedump path, no memory", path);
+        errno = ENOMEM;
+        goto err;
+    }
 
-	ret = syncop_ipc (subvol, opcode, NULL, NULL);
-        DECODE_SYNCOP_ERR (ret);
+    __GLFS_EXIT_FS;
 
-out:
-        glfs_subvol_done (fs, subvol);
-        __GLFS_EXIT_FS;
+    return 0;
+err:
+    __GLFS_EXIT_FS;
 
 invalid_fs:
-        return ret;
+    return -1;
 }
-
-GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_ipc, 3.7.0);
diff --git a/api/src/glfs.h b/api/src/glfs.h
index d3bf1b4cdcc..279d11d58ee 100644
--- a/api/src/glfs.h
+++ b/api/src/glfs.h
@@ -1,5 +1,5 @@
 /*
-  Copyright (c) 2012 Red Hat, Inc. <http://www.redhat.com>
+  Copyright (c) 2012-2018 Red Hat, Inc. <http://www.redhat.com>
   This file is part of GlusterFS.
 
   This file is licensed to you under your choice of the GNU Lesser
@@ -8,7 +8,6 @@
   cases as published by the Free Software Foundation.
 */
 
-
 #ifndef _GLFS_H
 #define _GLFS_H
 
@@ -21,6 +20,17 @@
   both the library and the application.
 */
 
+/* Values for valid flags to be used when using XXXsetattr, to set multiple
+ attribute values passed via the related stat structure.
+ */
+
+#define GFAPI_SET_ATTR_MODE 0x1
+#define GFAPI_SET_ATTR_UID 0x2
+#define GFAPI_SET_ATTR_GID 0x4
+#define GFAPI_SET_ATTR_SIZE 0x8
+#define GFAPI_SET_ATTR_ATIME 0x10
+#define GFAPI_SET_ATTR_MTIME 0x20
+
 #ifndef _FILE_OFFSET_BITS
 #define _FILE_OFFSET_BITS 64
 #endif
@@ -41,6 +51,40 @@
 #include <sys/cdefs.h>
 #include <dirent.h>
 #include <sys/statvfs.h>
+#include <stdint.h>
+#include <sys/time.h>
+
+/*
+ * For off64_t to be defined, we need both
+ * __USE_LARGEFILE64 to be true and __off64_t_defnined to be
+ * false. But, making __USE_LARGEFILE64 true causes other issues
+ * such as redinition of stat and fstat to stat64 and fstat64
+ * respectively which again causes compilation issues.
+ * Without off64_t being defined, this will not compile as
+ * copy_file_range uses off64_t. Hence define it here. First
+ * check whether __off64_t_defined is true or not. <unistd.h>
+ * sets that flag when it defines off64_t. If __off64_t_defined
+ * is false and __USE_FILE_OFFSET64 is true, then go on to define
+ * off64_t using __off64_t.
+ */
+#ifndef GF_BSD_HOST_OS
+#if defined(__USE_FILE_OFFSET64) && !defined(__off64_t_defined)
+typedef __off64_t off64_t;
+#endif /* defined(__USE_FILE_OFFSET64) && !defined(__off64_t_defined) */
+#else
+#include <stdio.h>
+#ifndef _OFF64_T_DECLARED
+/*
+ * Including <stdio.h> (done above) should actually define
+ * _OFF64_T_DECLARED with off64_t data type being available
+ * for consumption. But, off64_t data type is not recognizable
+ * for FreeBSD versions less than 11. Hence, int64_t is typedefed
+ * to off64_t.
+ */
+#define _OFF64_T_DECLARED
+typedef int64_t off64_t;
+#endif /* _OFF64_T_DECLARED */
+#endif /* GF_BSD_HOST_OS */
 
 #if defined(HAVE_SYS_ACL_H) || (defined(USE_POSIX_ACLS) && USE_POSIX_ACLS)
 #include <sys/acl.h>
@@ -51,19 +95,20 @@ typedef int acl_type_t;
 
 /* Portability non glibc c++ build systems */
 #ifndef __THROW
-# if defined __cplusplus
-#  define __THROW       throw ()
-# else
-#  define __THROW
-# endif
+#if defined __cplusplus
+#define __THROW throw()
+#else
+#define __THROW
+#endif
 #endif
 
 #ifndef GF_DARWIN_HOST_OS
-#define GFAPI_PUBLIC(sym, ver) /**/
+#define GFAPI_PUBLIC(sym, ver)  /**/
 #define GFAPI_PRIVATE(sym, ver) /**/
 #else
 #define GFAPI_PUBLIC(sym, ver) __asm("_" __STRING(sym) "$GFAPI_" __STRING(ver))
-#define GFAPI_PRIVATE(sym, ver) __asm("_" __STRING(sym) "$GFAPI_PRIVATE_" __STRING(ver))
+#define GFAPI_PRIVATE(sym, ver)                                                \
+    __asm("_" __STRING(sym) "$GFAPI_PRIVATE_" __STRING(ver))
 #endif
 
 __BEGIN_DECLS
@@ -72,7 +117,6 @@ __BEGIN_DECLS
 struct glfs;
 typedef struct glfs glfs_t;
 
-
 /*
   SYNOPSIS
 
@@ -93,8 +137,8 @@ typedef struct glfs glfs_t;
 
   @volname: Name of the volume. This identifies the server-side volume and
             the fetched volfile (equivalent of --volfile-id command line
-	    parameter to glusterfsd). When used with glfs_set_volfile() the
-	    @volname has no effect (except for appearing in log messages).
+            parameter to glusterfsd). When used with glfs_set_volfile() the
+            @volname has no effect (except for appearing in log messages).
 
   RETURN VALUES
 
@@ -103,9 +147,8 @@ typedef struct glfs glfs_t;
 
 */
 
-glfs_t *glfs_new (const char *volname) __THROW
-        GFAPI_PUBLIC(glfs_new, 3.4.0);
-
+glfs_t *
+glfs_new(const char *volname) __THROW GFAPI_PUBLIC(glfs_new, 3.4.0);
 
 /*
   SYNOPSIS
@@ -134,9 +177,9 @@ glfs_t *glfs_new (const char *volname) __THROW
 
 */
 
-int glfs_set_volfile (glfs_t *fs, const char *volfile) __THROW
-        GFAPI_PUBLIC(glfs_set_volfile, 3.4.0);
-
+int
+glfs_set_volfile(glfs_t *fs, const char *volfile) __THROW
+    GFAPI_PUBLIC(glfs_set_volfile, 3.4.0);
 
 /*
   SYNOPSIS
@@ -161,10 +204,9 @@ int glfs_set_volfile (glfs_t *fs, const char *volfile) __THROW
        specification file.
 
   @transport: String specifying the transport used to connect to the
-              management daemon. Specifying NULL will result in the usage
-              of the default (tcp) transport type. Permitted values
-              are those what you specify as transport-type in a volume
-              specification file (e.g "tcp", "rdma", "unix" etc.)
+              management daemon. Specifying NULL will result in the
+              usage of the default (tcp) transport type. Permitted
+              values are "tcp" or "unix".
 
   @host:      String specifying the address where to find the management daemon.
               Socket path, while using Unix domain socket as transport type.
@@ -188,12 +230,15 @@ int glfs_set_volfile (glfs_t *fs, const char *volfile) __THROW
 
 */
 
-int glfs_set_volfile_server (glfs_t *fs, const char *transport,
-                             const char *host, int port) __THROW
-        GFAPI_PUBLIC(glfs_set_volfile_server, 3.4.0);
-int glfs_unset_volfile_server (glfs_t *fs, const char *transport,
-                               const char *host, int port) __THROW
-        GFAPI_PUBLIC(glfs_unset_volfile_server, 3.5.1);
+int
+glfs_set_volfile_server(glfs_t *fs, const char *transport, const char *host,
+                        int port) __THROW
+    GFAPI_PUBLIC(glfs_set_volfile_server, 3.4.0);
+
+int
+glfs_unset_volfile_server(glfs_t *fs, const char *transport, const char *host,
+                          int port) __THROW
+    GFAPI_PUBLIC(glfs_unset_volfile_server, 3.5.1);
 /*
   SYNOPSIS
 
@@ -223,9 +268,9 @@ int glfs_unset_volfile_server (glfs_t *fs, const char *transport,
 
 */
 
-int glfs_set_logging (glfs_t *fs, const char *logfile, int loglevel) __THROW
-        GFAPI_PUBLIC(glfs_set_logging, 3.4.0);
-
+int
+glfs_set_logging(glfs_t *fs, const char *logfile, int loglevel) __THROW
+    GFAPI_PUBLIC(glfs_set_logging, 3.4.0);
 
 /*
   SYNOPSIS
@@ -251,9 +296,8 @@ int glfs_set_logging (glfs_t *fs, const char *logfile, int loglevel) __THROW
 
 */
 
-int glfs_init (glfs_t *fs) __THROW
-        GFAPI_PUBLIC(glfs_init, 3.4.0);
-
+int
+glfs_init(glfs_t *fs) __THROW GFAPI_PUBLIC(glfs_init, 3.4.0);
 
 /*
   SYNOPSIS
@@ -285,8 +329,8 @@ int glfs_init (glfs_t *fs) __THROW
    0 : Success.
 */
 
-int glfs_fini (glfs_t *fs) __THROW
-        GFAPI_PUBLIC(glfs_fini, 3.4.0);
+int
+glfs_fini(glfs_t *fs) __THROW GFAPI_PUBLIC(glfs_fini, 3.4.0);
 
 /*
   SYNOPSIS
@@ -316,9 +360,9 @@ int glfs_fini (glfs_t *fs) __THROW
       <0: volfile length exceeds @len by N bytes (@buf unchanged)
 */
 
-ssize_t glfs_get_volfile (glfs_t *fs, void *buf, size_t len) __THROW
-        GFAPI_PUBLIC(glfs_get_volfile, 3.6.0);
-
+ssize_t
+glfs_get_volfile(glfs_t *fs, void *buf, size_t len) __THROW
+    GFAPI_PUBLIC(glfs_get_volfile, 3.6.0);
 
 /*
   SYNOPSIS
@@ -331,9 +375,9 @@ ssize_t glfs_get_volfile (glfs_t *fs, void *buf, size_t len) __THROW
        the management server (glusterd) to fetch volume uuid and stores it
        in the glusterfs_context linked to the glfs object fs which can be used
        in the subsequent calls. Later it parses that UUID to convert it from
-       cannonical string format into an opaque byte array and copy it into
-       the volid array. Incase if either of the input parameters, volid or size,
-       is NULL, number of bytes required to copy the volume UUID is returned.
+       canonical string format into an opaque byte array and copy it into
+       the volid array. In case if either of the input parameters, volid or
+  size, is NULL, number of bytes required to copy the volume UUID is returned.
 
   PARAMETERS
 
@@ -348,9 +392,9 @@ ssize_t glfs_get_volfile (glfs_t *fs, void *buf, size_t len) __THROW
         Others : length of the volume UUID stored.
 */
 
-int glfs_get_volumeid (struct glfs *fs, char *volid, size_t size) __THROW
-        GFAPI_PUBLIC(glfs_get_volumeid, 3.5.0);
-
+int
+glfs_get_volumeid(glfs_t *fs, char *volid, size_t size) __THROW
+    GFAPI_PUBLIC(glfs_get_volumeid, 3.5.0);
 
 /*
  * FILE OPERATION
@@ -372,6 +416,119 @@ struct glfs_fd;
 typedef struct glfs_fd glfs_fd_t;
 
 /*
+ * Mask for request/result items in the struct glfs_stat.
+ *
+ * Query request/result mask for glfs_stat() (family of functions) and
+ * struct glfs_stat::glfs_st_mask.
+ *
+ * These bits should be set in the mask argument of glfs_stat() (family of
+ * functions) to request particular items when calling glfs_stat().
+ *
+ * NOTE: Lower order 32 bits are used to reflect statx(2) bits. For Gluster
+ * specific attrs/extensions, use higher order 32 bits.
+ *
+ */
+#define GLFS_STAT_TYPE 0x0000000000000001U   /* Want/got stx_mode & S_IFMT */
+#define GLFS_STAT_MODE 0x0000000000000002U   /* Want/got stx_mode & ~S_IFMT */
+#define GLFS_STAT_NLINK 0x0000000000000004U  /* Want/got stx_nlink */
+#define GLFS_STAT_UID 0x0000000000000008U    /* Want/got stx_uid */
+#define GLFS_STAT_GID 0x0000000000000010U    /* Want/got stx_gid */
+#define GLFS_STAT_ATIME 0x0000000000000020U  /* Want/got stx_atime */
+#define GLFS_STAT_MTIME 0x0000000000000040U  /* Want/got stx_mtime */
+#define GLFS_STAT_CTIME 0x0000000000000080U  /* Want/got stx_ctime */
+#define GLFS_STAT_INO 0x0000000000000100U    /* Want/got stx_ino */
+#define GLFS_STAT_SIZE 0x0000000000000200U   /* Want/got stx_size */
+#define GLFS_STAT_BLOCKS 0x0000000000000400U /* Want/got stx_blocks */
+#define GLFS_STAT_BASIC_STATS                                                  \
+    0x00000000000007ffU /* Items in the normal stat struct */
+#define GLFS_STAT_BTIME 0x0000000000000800U /* Want/got stx_btime */
+#define GLFS_STAT_ALL 0x0000000000000fffU   /* All currently supported flags */
+#define GLFS_STAT_RESERVED                                                     \
+    0x8000000000000000U /* Reserved to denote future expansion */
+
+/* Macros for checking validity of struct glfs_stat members.*/
+#define GLFS_STAT_TYPE_VALID(stmask) (stmask & GLFS_STAT_TYPE)
+#define GLFS_STAT_MODE_VALID(stmask) (stmask & GLFS_STAT_MODE)
+#define GLFS_STAT_NLINK_VALID(stmask) (stmask & GLFS_STAT_NLINK)
+#define GLFS_STAT_UID_VALID(stmask) (stmask & GLFS_STAT_UID)
+#define GLFS_STAT_GID_VALID(stmask) (stmask & GLFS_STAT_GID)
+#define GLFS_STAT_ATIME_VALID(stmask) (stmask & GLFS_STAT_ATIME)
+#define GLFS_STAT_MTIME_VALID(stmask) (stmask & GLFS_STAT_MTIME)
+#define GLFS_STAT_CTIME_VALID(stmask) (stmask & GLFS_STAT_CTIME)
+#define GLFS_STAT_INO_VALID(stmask) (stmask & GLFS_STAT_INO)
+#define GLFS_STAT_SIZE_VALID(stmask) (stmask & GLFS_STAT_SIZE)
+#define GLFS_STAT_BLOCKS_VALID(stmask) (stmask & GLFS_STAT_BLOCKS)
+#define GLFS_STAT_BTIME_VALID(stmask) (stmask & GLFS_STAT_BTIME)
+#define GLFS_STAT_GFID_VALID(stmask) (stmask & GLFS_STAT_GFID)
+
+/*
+ * Attributes to be found in glfs_st_attributes and masked in
+ * glfs_st_attributes_mask.
+ *
+ * These give information about the features or the state of a file that might
+ * be of use to programs.
+ *
+ * NOTE: Lower order 32 bits are used to reflect statx(2) attribute bits. For
+ * Gluster specific attrs, use higher order 32 bits.
+ *
+ * NOTE: We do not support any file attributes or state as yet!
+ */
+#define GLFS_STAT_ATTR_RESERVED                                                \
+    0x8000000000000000U /* Reserved to denote future expansion */
+
+/* Extended file attribute structure.
+ *
+ * The caller passes a mask of what they're specifically interested in as a
+ * parameter to glfs_stat().  What glfs_stat() actually got will be indicated
+ * in glfs_st_mask upon return.
+ *
+ * For each bit in the mask argument:
+ *
+ * - if the datum is not supported:
+ *
+ *   - the bit will be cleared, and
+ *
+ *   - the datum value is undefined
+ *
+ * - otherwise, if explicitly requested:
+ *
+ *   - the field will be filled in and the bit will be set;
+ *
+ * - otherwise, if not requested, but available in, it will be filled in
+ * anyway, and the bit will be set upon return;
+ *
+ * - otherwise the field and the bit will be cleared before returning.
+ *
+ */
+
+struct glfs_stat {
+    uint64_t glfs_st_mask;       /* What results were written [uncond] */
+    uint64_t glfs_st_attributes; /* Flags conveying information about the file
+                                    [uncond] */
+    uint64_t glfs_st_attributes_mask; /* Mask to show what's supported in
+                                         st_attributes [ucond] */
+    struct timespec glfs_st_atime;    /* Last access time */
+    struct timespec glfs_st_btime;    /* File creation time */
+    struct timespec glfs_st_ctime;    /* Last attribute change time */
+    struct timespec glfs_st_mtime;    /* Last data modification time */
+    ino_t glfs_st_ino;                /* Inode number */
+    off_t glfs_st_size;               /* File size */
+    blkcnt_t glfs_st_blocks;          /* Number of 512-byte blocks allocated */
+    uint32_t glfs_st_rdev_major; /* Device ID of special file [if bdev/cdev] */
+    uint32_t glfs_st_rdev_minor;
+    uint32_t glfs_st_dev_major; /* ID of device containing file [uncond] */
+    uint32_t glfs_st_dev_minor;
+    blksize_t glfs_st_blksize; /* Preferred general I/O size [uncond] */
+    nlink_t glfs_st_nlink;     /* Number of hard links */
+    uid_t glfs_st_uid;         /* User ID of owner */
+    gid_t glfs_st_gid;         /* Group ID of owner */
+    mode_t glfs_st_mode;       /* File mode */
+};
+
+#define GLFS_LEASE_ID_SIZE 16 /* 128bits */
+typedef char glfs_leaseid_t[GLFS_LEASE_ID_SIZE];
+
+/*
  * PER THREAD IDENTITY MODIFIERS
  *
  * The following operations enable to set a per thread identity context
@@ -388,18 +545,27 @@ typedef struct glfs_fd glfs_fd_t;
  *    caller
  *  - The groups once set, need to be unset by setting the size to 0 (in which
  *    case the list argument is a do not care)
+ *  - In case of leases feature enables, setfsleaseid is used to set and reset
+ *    leaseid before and after every I/O operation.
  *  - Once a process for a thread of operation choses to set the IDs, all glfs
  *    calls made from that thread would default to the IDs set for the thread.
  *    As a result use these APIs with care and ensure that the set IDs are
  *    reverted to global process defaults as required.
  *
  */
-int glfs_setfsuid (uid_t fsuid) __THROW
-        GFAPI_PUBLIC(glfs_setfsuid, 3.4.2);
-int glfs_setfsgid (gid_t fsgid) __THROW
-        GFAPI_PUBLIC(glfs_setfsgid, 3.4.2);
-int glfs_setfsgroups (size_t size, const gid_t *list) __THROW
-        GFAPI_PUBLIC(glfs_setfsgroups, 3.4.2);
+int
+glfs_setfsuid(uid_t fsuid) __THROW GFAPI_PUBLIC(glfs_setfsuid, 3.4.2);
+
+int
+glfs_setfsgid(gid_t fsgid) __THROW GFAPI_PUBLIC(glfs_setfsgid, 3.4.2);
+
+int
+glfs_setfsgroups(size_t size, const gid_t *list) __THROW
+    GFAPI_PUBLIC(glfs_setfsgroups, 3.4.2);
+
+int
+glfs_setfsleaseid(glfs_leaseid_t leaseid) __THROW
+    GFAPI_PUBLIC(glfs_setfsleaseid, 4.0.0);
 
 /*
   SYNOPSIS
@@ -426,9 +592,9 @@ int glfs_setfsgroups (size_t size, const gid_t *list) __THROW
 
  */
 
-glfs_fd_t *glfs_open (glfs_t *fs, const char *path, int flags) __THROW
-        GFAPI_PUBLIC(glfs_open, 3.4.0);
-
+glfs_fd_t *
+glfs_open(glfs_t *fs, const char *path, int flags) __THROW
+    GFAPI_PUBLIC(glfs_open, 3.4.0);
 
 /*
   SYNOPSIS
@@ -456,19 +622,20 @@ glfs_fd_t *glfs_open (glfs_t *fs, const char *path, int flags) __THROW
 
  */
 
-glfs_fd_t *glfs_creat (glfs_t *fs, const char *path, int flags,
-		       mode_t mode) __THROW
-        GFAPI_PUBLIC(glfs_creat, 3.4.0);
+glfs_fd_t *
+glfs_creat(glfs_t *fs, const char *path, int flags, mode_t mode) __THROW
+    GFAPI_PUBLIC(glfs_creat, 3.4.0);
 
-int glfs_close (glfs_fd_t *fd) __THROW
-        GFAPI_PUBLIC(glfs_close, 3.4.0);
+int
+glfs_close(glfs_fd_t *fd) __THROW GFAPI_PUBLIC(glfs_close, 3.4.0);
 
-glfs_t *glfs_from_glfd (glfs_fd_t *fd) __THROW
-        GFAPI_PUBLIC(glfs_from_glfd, 3.4.0);
+glfs_t *
+glfs_from_glfd(glfs_fd_t *fd) __THROW GFAPI_PUBLIC(glfs_from_glfd, 3.4.0);
 
-int glfs_set_xlator_option (glfs_t *fs, const char *xlator, const char *key,
-			    const char *value) __THROW
-        GFAPI_PUBLIC(glfs_set_xlator_options, 3.4.0);
+int
+glfs_set_xlator_option(glfs_t *fs, const char *xlator, const char *key,
+                       const char *value) __THROW
+    GFAPI_PUBLIC(glfs_set_xlator_option, 3.4.0);
 
 /*
 
@@ -491,292 +658,828 @@ int glfs_set_xlator_option (glfs_t *fs, const char *xlator, const char *key,
   time of issuing the async IO call. This can be used by the
   caller to differentiate different instances of the async requests
   in a common callback function.
+
+  @prestat and @poststat are allocated on the stack, that are auto destroyed
+  post the callback function returns.
 */
 
-typedef void (*glfs_io_cbk) (glfs_fd_t *fd, ssize_t ret, void *data);
+typedef void (*glfs_io_cbk)(glfs_fd_t *fd, ssize_t ret,
+                            struct glfs_stat *prestat,
+                            struct glfs_stat *poststat, void *data);
 
 // glfs_{read,write}[_async]
 
-ssize_t glfs_read (glfs_fd_t *fd, void *buf,
-                   size_t count, int flags) __THROW
-        GFAPI_PUBLIC(glfs_read, 3.4.0);
-ssize_t glfs_write (glfs_fd_t *fd, const void *buf,
-                    size_t count, int flags) __THROW
-        GFAPI_PUBLIC(glfs_write, 3.4.0);
-int glfs_read_async (glfs_fd_t *fd, void *buf, size_t count, int flags,
-		     glfs_io_cbk fn, void *data) __THROW
-        GFAPI_PUBLIC(glfs_read_async, 3.4.0);
-int glfs_write_async (glfs_fd_t *fd, const void *buf, size_t count, int flags,
-		      glfs_io_cbk fn, void *data) __THROW
-        GFAPI_PUBLIC(glfs_write_async, 3.4.0);
+ssize_t
+glfs_read(glfs_fd_t *fd, void *buf, size_t count, int flags) __THROW
+    GFAPI_PUBLIC(glfs_read, 3.4.0);
+
+ssize_t
+glfs_write(glfs_fd_t *fd, const void *buf, size_t count, int flags) __THROW
+    GFAPI_PUBLIC(glfs_write, 3.4.0);
+
+int
+glfs_read_async(glfs_fd_t *fd, void *buf, size_t count, int flags,
+                glfs_io_cbk fn, void *data) __THROW
+    GFAPI_PUBLIC(glfs_read_async, 6.0);
+
+int
+glfs_write_async(glfs_fd_t *fd, const void *buf, size_t count, int flags,
+                 glfs_io_cbk fn, void *data) __THROW
+    GFAPI_PUBLIC(glfs_write_async, 6.0);
 
 // glfs_{read,write}v[_async]
 
-ssize_t glfs_readv (glfs_fd_t *fd, const struct iovec *iov, int iovcnt,
-		    int flags) __THROW
-        GFAPI_PUBLIC(glfs_readv, 3.4.0);
-ssize_t glfs_writev (glfs_fd_t *fd, const struct iovec *iov, int iovcnt,
-		     int flags) __THROW
-        GFAPI_PUBLIC(glfs_writev, 3.4.0);
-int glfs_readv_async (glfs_fd_t *fd, const struct iovec *iov, int count,
-		      int flags, glfs_io_cbk fn, void *data) __THROW
-        GFAPI_PUBLIC(glfs_readv_async, 3.4.0);
-int glfs_writev_async (glfs_fd_t *fd, const struct iovec *iov, int count,
-		       int flags, glfs_io_cbk fn, void *data) __THROW
-        GFAPI_PUBLIC(glfs_writev_async, 3.4.0);
+ssize_t
+glfs_readv(glfs_fd_t *fd, const struct iovec *iov, int iovcnt,
+           int flags) __THROW GFAPI_PUBLIC(glfs_readv, 3.4.0);
+
+ssize_t
+glfs_writev(glfs_fd_t *fd, const struct iovec *iov, int iovcnt,
+            int flags) __THROW GFAPI_PUBLIC(glfs_writev, 3.4.0);
+
+int
+glfs_readv_async(glfs_fd_t *fd, const struct iovec *iov, int count, int flags,
+                 glfs_io_cbk fn, void *data) __THROW
+    GFAPI_PUBLIC(glfs_readv_async, 6.0);
+
+int
+glfs_writev_async(glfs_fd_t *fd, const struct iovec *iov, int count, int flags,
+                  glfs_io_cbk fn, void *data) __THROW
+    GFAPI_PUBLIC(glfs_writev_async, 6.0);
 
 // glfs_p{read,write}[_async]
 
-ssize_t glfs_pread (glfs_fd_t *fd, void *buf, size_t count, off_t offset,
-		    int flags) __THROW
-        GFAPI_PUBLIC(glfs_pread, 3.4.0);
-ssize_t glfs_pwrite (glfs_fd_t *fd, const void *buf, size_t count,
-		     off_t offset, int flags) __THROW
-        GFAPI_PUBLIC(glfs_pwrite, 3.4.0);
-int glfs_pread_async (glfs_fd_t *fd, void *buf, size_t count, off_t offset,
-		      int flags, glfs_io_cbk fn, void *data) __THROW
-        GFAPI_PUBLIC(glfs_pread_async, 3.4.0);
-int glfs_pwrite_async (glfs_fd_t *fd, const void *buf, int count, off_t offset,
-		       int flags, glfs_io_cbk fn, void *data) __THROW
-        GFAPI_PUBLIC(glfs_pwrite_async, 3.4.0);
+ssize_t
+glfs_pread(glfs_fd_t *fd, void *buf, size_t count, off_t offset, int flags,
+           struct glfs_stat *poststat) __THROW GFAPI_PUBLIC(glfs_pread, 6.0);
+
+ssize_t
+glfs_pwrite(glfs_fd_t *fd, const void *buf, size_t count, off_t offset,
+            int flags, struct glfs_stat *prestat,
+            struct glfs_stat *poststat) __THROW GFAPI_PUBLIC(glfs_pwrite, 6.0);
+
+int
+glfs_pread_async(glfs_fd_t *fd, void *buf, size_t count, off_t offset,
+                 int flags, glfs_io_cbk fn, void *data) __THROW
+    GFAPI_PUBLIC(glfs_pread_async, 6.0);
+
+int
+glfs_pwrite_async(glfs_fd_t *fd, const void *buf, int count, off_t offset,
+                  int flags, glfs_io_cbk fn, void *data) __THROW
+    GFAPI_PUBLIC(glfs_pwrite_async, 6.0);
 
 // glfs_p{read,write}v[_async]
 
-ssize_t glfs_preadv (glfs_fd_t *fd, const struct iovec *iov, int iovcnt,
-		     off_t offset, int flags) __THROW
-        GFAPI_PUBLIC(glfs_preadv, 3.4.0);
-ssize_t glfs_pwritev (glfs_fd_t *fd, const struct iovec *iov, int iovcnt,
-		      off_t offset, int flags) __THROW
-        GFAPI_PUBLIC(glfs_pwritev, 3.4.0);
-int glfs_preadv_async (glfs_fd_t *fd, const struct iovec *iov,
-                       int count, off_t offset, int flags,
-                       glfs_io_cbk fn, void *data) __THROW
-        GFAPI_PUBLIC(glfs_preadv_async, 3.4.0);
-int glfs_pwritev_async (glfs_fd_t *fd, const struct iovec *iov,
-                        int count, off_t offset, int flags,
-                        glfs_io_cbk fn, void *data) __THROW
-        GFAPI_PUBLIC(glfs_pwritev_async, 3.4.0);
+ssize_t
+glfs_preadv(glfs_fd_t *fd, const struct iovec *iov, int iovcnt, off_t offset,
+            int flags) __THROW GFAPI_PUBLIC(glfs_preadv, 3.4.0);
+
+ssize_t
+glfs_pwritev(glfs_fd_t *fd, const struct iovec *iov, int iovcnt, off_t offset,
+             int flags) __THROW GFAPI_PUBLIC(glfs_pwritev, 3.4.0);
+
+int
+glfs_preadv_async(glfs_fd_t *fd, const struct iovec *iov, int count,
+                  off_t offset, int flags, glfs_io_cbk fn, void *data) __THROW
+    GFAPI_PUBLIC(glfs_preadv_async, 6.0);
+
+int
+glfs_pwritev_async(glfs_fd_t *fd, const struct iovec *iov, int count,
+                   off_t offset, int flags, glfs_io_cbk fn, void *data) __THROW
+    GFAPI_PUBLIC(glfs_pwritev_async, 6.0);
+
+off_t
+glfs_lseek(glfs_fd_t *fd, off_t offset, int whence) __THROW
+    GFAPI_PUBLIC(glfs_lseek, 3.4.0);
+
+ssize_t
+glfs_copy_file_range(struct glfs_fd *glfd_in, off64_t *off_in,
+                     struct glfs_fd *glfd_out, off64_t *off_out, size_t len,
+                     unsigned int flags, struct glfs_stat *statbuf,
+                     struct glfs_stat *prestat,
+                     struct glfs_stat *poststat) __THROW
+    GFAPI_PUBLIC(glfs_copy_file_range, 6.0);
+
+int
+glfs_truncate(glfs_t *fs, const char *path, off_t length) __THROW
+    GFAPI_PUBLIC(glfs_truncate, 3.7.15);
+
+int
+glfs_ftruncate(glfs_fd_t *fd, off_t length, struct glfs_stat *prestat,
+               struct glfs_stat *poststat) __THROW
+    GFAPI_PUBLIC(glfs_ftruncate, 6.0);
+
+int
+glfs_ftruncate_async(glfs_fd_t *fd, off_t length, glfs_io_cbk fn,
+                     void *data) __THROW
+    GFAPI_PUBLIC(glfs_ftruncate_async, 6.0);
+
+int
+glfs_lstat(glfs_t *fs, const char *path, struct stat *buf) __THROW
+    GFAPI_PUBLIC(glfs_lstat, 3.4.0);
+
+int
+glfs_stat(glfs_t *fs, const char *path, struct stat *buf) __THROW
+    GFAPI_PUBLIC(glfs_stat, 3.4.0);
+
+int
+glfs_fstat(glfs_fd_t *fd, struct stat *buf) __THROW
+    GFAPI_PUBLIC(glfs_fstat, 3.4.0);
+
+int
+glfs_fsync(glfs_fd_t *fd, struct glfs_stat *prestat,
+           struct glfs_stat *poststat) __THROW GFAPI_PUBLIC(glfs_fsync, 6.0);
+
+int
+glfs_fsync_async(glfs_fd_t *fd, glfs_io_cbk fn, void *data) __THROW
+    GFAPI_PUBLIC(glfs_fsync_async, 6.0);
+
+int
+glfs_fdatasync(glfs_fd_t *fd, struct glfs_stat *prestat,
+               struct glfs_stat *poststat) __THROW
+    GFAPI_PUBLIC(glfs_fdatasync, 6.0);
+
+int
+glfs_fdatasync_async(glfs_fd_t *fd, glfs_io_cbk fn, void *data) __THROW
+    GFAPI_PUBLIC(glfs_fdatasync_async, 6.0);
+
+int
+glfs_access(glfs_t *fs, const char *path, int mode) __THROW
+    GFAPI_PUBLIC(glfs_access, 3.4.0);
+
+int
+glfs_symlink(glfs_t *fs, const char *oldpath, const char *newpath) __THROW
+    GFAPI_PUBLIC(glfs_symlink, 3.4.0);
+
+int
+glfs_readlink(glfs_t *fs, const char *path, char *buf, size_t bufsiz) __THROW
+    GFAPI_PUBLIC(glfs_readlink, 3.4.0);
+
+int
+glfs_mknod(glfs_t *fs, const char *path, mode_t mode, dev_t dev) __THROW
+    GFAPI_PUBLIC(glfs_mknod, 3.4.0);
+
+int
+glfs_mkdir(glfs_t *fs, const char *path, mode_t mode) __THROW
+    GFAPI_PUBLIC(glfs_mkdir, 3.4.0);
+
+int
+glfs_unlink(glfs_t *fs, const char *path) __THROW
+    GFAPI_PUBLIC(glfs_unlink, 3.4.0);
+
+int
+glfs_rmdir(glfs_t *fs, const char *path) __THROW
+    GFAPI_PUBLIC(glfs_rmdir, 3.4.0);
+
+int
+glfs_rename(glfs_t *fs, const char *oldpath, const char *newpath) __THROW
+    GFAPI_PUBLIC(glfs_rename, 3.4.0);
+
+int
+glfs_link(glfs_t *fs, const char *oldpath, const char *newpath) __THROW
+    GFAPI_PUBLIC(glfs_link, 3.4.0);
+
+glfs_fd_t *
+glfs_opendir(glfs_t *fs, const char *path) __THROW
+    GFAPI_PUBLIC(glfs_opendir, 3.4.0);
+
+/*
+ * @glfs_readdir_r and @glfs_readdirplus_r ARE thread safe AND re-entrant,
+ * but the interface has ambiguity about the size of @dirent to be allocated
+ * before calling the APIs. 512 byte buffer (for @dirent) is sufficient for
+ * all known systems which are tested againt glusterfs/gfapi, but may be
+ * insufficient in the future.
+ */
+
+int
+glfs_readdir_r(glfs_fd_t *fd, struct dirent *dirent,
+               struct dirent **result) __THROW
+    GFAPI_PUBLIC(glfs_readdir_r, 3.4.0);
+
+int
+glfs_readdirplus_r(glfs_fd_t *fd, struct stat *stat, struct dirent *dirent,
+                   struct dirent **result) __THROW
+    GFAPI_PUBLIC(glfs_readdirplus_r, 3.4.0);
+
+/*
+ * @glfs_readdir and @glfs_readdirplus are NEITHER thread safe NOR re-entrant
+ * when called on the same directory handle. However they ARE thread safe
+ * AND re-entrant when called on different directory handles (which may be
+ * referring to the same directory too.)
+ */
+
+struct dirent *
+glfs_readdir(glfs_fd_t *fd) __THROW GFAPI_PUBLIC(glfs_readdir, 3.5.0);
+
+struct dirent *
+glfs_readdirplus(glfs_fd_t *fd, struct stat *stat) __THROW
+    GFAPI_PUBLIC(glfs_readdirplus, 3.5.0);
+
+long
+glfs_telldir(glfs_fd_t *fd) __THROW GFAPI_PUBLIC(glfs_telldir, 3.4.0);
+
+void
+glfs_seekdir(glfs_fd_t *fd, long offset) __THROW
+    GFAPI_PUBLIC(glfs_seekdir, 3.4.0);
+
+int
+glfs_closedir(glfs_fd_t *fd) __THROW GFAPI_PUBLIC(glfs_closedir, 3.4.0);
+
+int
+glfs_statvfs(glfs_t *fs, const char *path, struct statvfs *buf) __THROW
+    GFAPI_PUBLIC(glfs_statvfs, 3.4.0);
+
+int
+glfs_chmod(glfs_t *fs, const char *path, mode_t mode) __THROW
+    GFAPI_PUBLIC(glfs_chmod, 3.4.0);
+
+int
+glfs_fchmod(glfs_fd_t *fd, mode_t mode) __THROW
+    GFAPI_PUBLIC(glfs_fchmod, 3.4.0);
+
+int
+glfs_chown(glfs_t *fs, const char *path, uid_t uid, gid_t gid) __THROW
+    GFAPI_PUBLIC(glfs_chown, 3.4.0);
+
+int
+glfs_lchown(glfs_t *fs, const char *path, uid_t uid, gid_t gid) __THROW
+    GFAPI_PUBLIC(glfs_lchown, 3.4.0);
 
+int
+glfs_fchown(glfs_fd_t *fd, uid_t uid, gid_t gid) __THROW
+    GFAPI_PUBLIC(glfs_fchown, 3.4.0);
 
-off_t glfs_lseek (glfs_fd_t *fd, off_t offset, int whence) __THROW
-        GFAPI_PUBLIC(glfs_lseek, 3.4.0);
+int
+glfs_utimens(glfs_t *fs, const char *path,
+             const struct timespec times[2]) __THROW
+    GFAPI_PUBLIC(glfs_utimens, 3.4.0);
 
-int glfs_truncate (glfs_t *fs, const char *path, off_t length) __THROW
-        GFAPI_PUBLIC(glfs_truncate, 3.4.0);
+int
+glfs_lutimens(glfs_t *fs, const char *path,
+              const struct timespec times[2]) __THROW
+    GFAPI_PUBLIC(glfs_lutimens, 3.4.0);
 
-int glfs_ftruncate (glfs_fd_t *fd, off_t length) __THROW
-        GFAPI_PUBLIC(glfs_ftruncate, 3.4.0);
-int glfs_ftruncate_async (glfs_fd_t *fd, off_t length, glfs_io_cbk fn,
-			  void *data) __THROW
-        GFAPI_PUBLIC(glfs_ftruncate_async, 3.4.0);
+int
+glfs_futimens(glfs_fd_t *fd, const struct timespec times[2]) __THROW
+    GFAPI_PUBLIC(glfs_futimens, 3.4.0);
 
-int glfs_lstat (glfs_t *fs, const char *path, struct stat *buf) __THROW
-        GFAPI_PUBLIC(glfs_lstat, 3.4.0);
-int glfs_stat (glfs_t *fs, const char *path, struct stat *buf) __THROW
-        GFAPI_PUBLIC(glfs_stat, 3.4.0);
-int glfs_fstat (glfs_fd_t *fd, struct stat *buf) __THROW
-        GFAPI_PUBLIC(glfs_fstat, 3.4.0);
+ssize_t
+glfs_getxattr(glfs_t *fs, const char *path, const char *name, void *value,
+              size_t size) __THROW GFAPI_PUBLIC(glfs_getxattr, 3.4.0);
 
-int glfs_fsync (glfs_fd_t *fd) __THROW
-        GFAPI_PUBLIC(glfs_fsync, 3.4.0);
-int glfs_fsync_async (glfs_fd_t *fd, glfs_io_cbk fn, void *data) __THROW
-        GFAPI_PUBLIC(glfs_fsync_async, 3.4.0);
+ssize_t
+glfs_lgetxattr(glfs_t *fs, const char *path, const char *name, void *value,
+               size_t size) __THROW GFAPI_PUBLIC(glfs_lgetxattr, 3.4.0);
 
-int glfs_fdatasync (glfs_fd_t *fd) __THROW
-        GFAPI_PUBLIC(glfs_fdatasync, 3.4.0);
-int glfs_fdatasync_async (glfs_fd_t *fd, glfs_io_cbk fn, void *data) __THROW
-        GFAPI_PUBLIC(glfs_fdatasync_async, 3.4.0);
+ssize_t
+glfs_fgetxattr(glfs_fd_t *fd, const char *name, void *value,
+               size_t size) __THROW GFAPI_PUBLIC(glfs_fgetxattr, 3.4.0);
 
-int glfs_access (glfs_t *fs, const char *path, int mode) __THROW
-        GFAPI_PUBLIC(glfs_access, 3.4.0);
+ssize_t
+glfs_listxattr(glfs_t *fs, const char *path, void *value, size_t size) __THROW
+    GFAPI_PUBLIC(glfs_listxattr, 3.4.0);
 
-int glfs_symlink (glfs_t *fs, const char *oldpath, const char *newpath) __THROW
-        GFAPI_PUBLIC(glfs_symlink, 3.4.0);
+ssize_t
+glfs_llistxattr(glfs_t *fs, const char *path, void *value, size_t size) __THROW
+    GFAPI_PUBLIC(glfs_llistxattr, 3.4.0);
 
-int glfs_readlink (glfs_t *fs, const char *path,
-                   char *buf, size_t bufsiz) __THROW
-        GFAPI_PUBLIC(glfs_readlink, 3.4.0);
+ssize_t
+glfs_flistxattr(glfs_fd_t *fd, void *value, size_t size) __THROW
+    GFAPI_PUBLIC(glfs_flistxattr, 3.4.0);
 
-int glfs_mknod (glfs_t *fs, const char *path, mode_t mode, dev_t dev) __THROW
-        GFAPI_PUBLIC(glfs_mknod, 3.4.0);
+int
+glfs_setxattr(glfs_t *fs, const char *path, const char *name, const void *value,
+              size_t size, int flags) __THROW
+    GFAPI_PUBLIC(glfs_setxattr, 3.4.0);
 
-int glfs_mkdir (glfs_t *fs, const char *path, mode_t mode) __THROW
-        GFAPI_PUBLIC(glfs_mkdir, 3.4.0);
+int
+glfs_lsetxattr(glfs_t *fs, const char *path, const char *name,
+               const void *value, size_t size, int flags) __THROW
+    GFAPI_PUBLIC(glfs_lsetxattr, 3.4.0);
 
-int glfs_unlink (glfs_t *fs, const char *path) __THROW
-        GFAPI_PUBLIC(glfs_unlink, 3.4.0);
+int
+glfs_fsetxattr(glfs_fd_t *fd, const char *name, const void *value, size_t size,
+               int flags) __THROW GFAPI_PUBLIC(glfs_fsetxattr, 3.4.0);
 
-int glfs_rmdir (glfs_t *fs, const char *path) __THROW
-        GFAPI_PUBLIC(glfs_rmdir, 3.4.0);
+int
+glfs_removexattr(glfs_t *fs, const char *path, const char *name) __THROW
+    GFAPI_PUBLIC(glfs_removexattr, 3.4.0);
 
-int glfs_rename (glfs_t *fs, const char *oldpath, const char *newpath) __THROW
-        GFAPI_PUBLIC(glfs_rename, 3.4.0);
+int
+glfs_lremovexattr(glfs_t *fs, const char *path, const char *name) __THROW
+    GFAPI_PUBLIC(glfs_lremovexattr, 3.4.0);
 
-int glfs_link (glfs_t *fs, const char *oldpath, const char *newpath) __THROW
-        GFAPI_PUBLIC(glfs_link, 3.4.0);
+int
+glfs_fremovexattr(glfs_fd_t *fd, const char *name) __THROW
+    GFAPI_PUBLIC(glfs_fremovexattr, 3.4.0);
 
-glfs_fd_t *glfs_opendir (glfs_t *fs, const char *path) __THROW
-        GFAPI_PUBLIC(glfs_opendir, 3.4.0);
+int
+glfs_fallocate(glfs_fd_t *fd, int keep_size, off_t offset, size_t len) __THROW
+    GFAPI_PUBLIC(glfs_fallocate, 3.5.0);
+
+int
+glfs_discard(glfs_fd_t *fd, off_t offset, size_t len) __THROW
+    GFAPI_PUBLIC(glfs_discard, 3.5.0);
+
+int
+glfs_discard_async(glfs_fd_t *fd, off_t length, size_t lent, glfs_io_cbk fn,
+                   void *data) __THROW GFAPI_PUBLIC(glfs_discard_async, 6.0);
+
+int
+glfs_zerofill(glfs_fd_t *fd, off_t offset, off_t len) __THROW
+    GFAPI_PUBLIC(glfs_zerofill, 3.5.0);
+
+int
+glfs_zerofill_async(glfs_fd_t *fd, off_t length, off_t len, glfs_io_cbk fn,
+                    void *data) __THROW GFAPI_PUBLIC(glfs_zerofill_async, 6.0);
+
+char *
+glfs_getcwd(glfs_t *fs, char *buf, size_t size) __THROW
+    GFAPI_PUBLIC(glfs_getcwd, 3.4.0);
+
+int
+glfs_chdir(glfs_t *fs, const char *path) __THROW
+    GFAPI_PUBLIC(glfs_chdir, 3.4.0);
+
+int
+glfs_fchdir(glfs_fd_t *fd) __THROW GFAPI_PUBLIC(glfs_fchdir, 3.4.0);
+
+char *
+glfs_realpath(glfs_t *fs, const char *path, char *resolved_path) __THROW
+    GFAPI_PUBLIC(glfs_realpath, 3.7.17);
 
 /*
- * @glfs_readdir_r and @glfs_readdirplus_r ARE thread safe AND re-entrant,
- * but the interface has ambiguity about the size of @dirent to be allocated
- * before calling the APIs. 512 byte buffer (for @dirent) is sufficient for
- * all known systems which are tested againt glusterfs/gfapi, but may be
- * insufficient in the future.
+ * @cmd and @flock are as specified in man fcntl(2).
+ */
+int
+glfs_posix_lock(glfs_fd_t *fd, int cmd, struct flock *flock) __THROW
+    GFAPI_PUBLIC(glfs_posix_lock, 3.4.0);
+
+/*
+  SYNOPSIS
+
+  glfs_file_lock: Request extended byte range lock on a file
+
+  DESCRIPTION
+
+  This function is capable of requesting either advisory or mandatory type
+  byte range locks on a file.
+
+  Note: To set a unique owner key for locks based on a particular file
+  descriptor, make use of glfs_fd_set_lkowner() api to do so before
+  requesting lock via this api. This owner key will be further consumed
+  by other incoming data modifying file operations via the same file
+  descriptor.
+
+  PARAMETERS
+
+  @fd: File descriptor
+
+  @cmd: As specified in man fcntl(2).
+
+  @flock: As specified in man fcntl(2).
+
+  @lk_mode: Required lock type from options available with the
+            enum glfs_lock_mode_t defined below.
+
+  RETURN VALUES
+
+  0   : Success. Lock has been granted.
+  -1  : Failure. @errno will be set indicating the type of failure.
+
  */
 
-int glfs_readdir_r (glfs_fd_t *fd, struct dirent *dirent,
-		    struct dirent **result) __THROW
-        GFAPI_PUBLIC(glfs_readdir_r, 3.4.0);
+/* Lock modes used by glfs_file_lock() */
+enum glfs_lock_mode { GLFS_LK_ADVISORY = 0, GLFS_LK_MANDATORY };
+typedef enum glfs_lock_mode glfs_lock_mode_t;
+
+int
+glfs_file_lock(glfs_fd_t *fd, int cmd, struct flock *flock,
+               glfs_lock_mode_t lk_mode) __THROW
+    GFAPI_PUBLIC(glfs_file_lock, 3.13.0);
 
-int glfs_readdirplus_r (glfs_fd_t *fd, struct stat *stat, struct dirent *dirent,
-			struct dirent **result) __THROW
-        GFAPI_PUBLIC(glfs_readdirplus_r, 3.4.0);
+glfs_fd_t *
+glfs_dup(glfs_fd_t *fd) __THROW GFAPI_PUBLIC(glfs_dup, 3.4.0);
+
+void
+glfs_free(void *ptr) __THROW GFAPI_PUBLIC(glfs_free, 3.7.16);
 
 /*
- * @glfs_readdir and @glfs_readdirplus are NEITHER thread safe NOR re-entrant
- * when called on the same directory handle. However they ARE thread safe
- * AND re-entrant when called on different directory handles (which may be
- * referring to the same directory too.)
+ * glfs_sysrq: send a system request to the @fs instance
+ *
+ * Different commands for @sysrq are possible, the defines for these are listed
+ * below the function definition.
+ *
+ * This function always returns success if the @sysrq is recognized. The return
+ * value does not way anythin about the result of the @sysrq execution. Not all
+ * @sysrq command will be able to return a success/failure status.
  */
+int
+glfs_sysrq(glfs_t *fs, char sysrq) __THROW GFAPI_PUBLIC(glfs_sysrq, 3.10.0);
 
-struct dirent *glfs_readdir (glfs_fd_t *fd) __THROW
-        GFAPI_PUBLIC(glfs_readdir, 3.5.0);
+#define GLFS_SYSRQ_HELP 'h' /* log a message with supported sysrq commands */
+#define GLFS_SYSRQ_STATEDUMP 's' /* create a statedump */
 
-struct dirent *glfs_readdirplus (glfs_fd_t *fd, struct stat *stat) __THROW
-        GFAPI_PUBLIC(glfs_readdirplus, 3.5.0);
+/*
+ * Structure returned as part of xreaddirplus
+ */
+struct glfs_xreaddirp_stat;
+typedef struct glfs_xreaddirp_stat glfs_xreaddirp_stat_t;
 
-long glfs_telldir (glfs_fd_t *fd) __THROW
-        GFAPI_PUBLIC(glfs_telldir, 3.4.0);
+/* Request flags to be used in XREADDIRP operation */
+#define GFAPI_XREADDIRP_NULL                                                   \
+    0x00000000 /* by default, no stat will be fetched */
+#define GFAPI_XREADDIRP_STAT 0x00000001   /* Get stat */
+#define GFAPI_XREADDIRP_HANDLE 0x00000002 /* Get object handle */
 
-void glfs_seekdir (glfs_fd_t *fd, long offset) __THROW
-        GFAPI_PUBLIC(glfs_seekdir, 3.4.0);
+/*
+ * This stat structure returned gets freed as part of glfs_free(xstat)
+ */
+struct stat *
+glfs_xreaddirplus_get_stat(glfs_xreaddirp_stat_t *xstat) __THROW
+    GFAPI_PUBLIC(glfs_xreaddirplus_get_stat, 3.11.0);
 
-int glfs_closedir (glfs_fd_t *fd) __THROW
-        GFAPI_PUBLIC(glfs_closedir, 3.4.0);
+/*
+ * SYNOPSIS
+ *
+ * glfs_xreaddirplus_r: Extended Readirplus operation
+ *
+ * DESCRIPTION
+ *
+ * This API does readdirplus operation, but along with stat it can fetch other
+ * extra information like object handles etc for each of the dirents returned
+ * based on requested flags. On success it returns the set of flags successfully
+ * processed.
+ *
+ * Note that there are chances that some of the requested information may not be
+ * available or returned (for example if reached EOD). Ensure to validate the
+ * returned value to determine what flags have been successfully processed
+ * & set.
+ *
+ * PARAMETERS
+ *
+ * INPUT:
+ * @glfd: GFAPI file descriptor of the directory
+ * @flags: Flags determining xreaddirp_stat requested
+ *         Current available values are:
+ *              GFAPI_XREADDIRP_NULL
+ *              GFAPI_XREADDIRP_STAT
+ *              GFAPI_XREADDIRP_HANDLE
+ * @ext: Dirent struture to copy the values to
+ *       (though optional recommended to be allocated by application
+ *        esp., in multi-threaded environment)
+ *
+ * OUTPUT:
+ * @res: to store the next dirent value. If NULL and return value is '0',
+ *       it means it reached end of the directory.
+ * @xstat_p: Pointer to contain all the requested data returned
+ *           for that dirent. Application should make use of glfs_free() API
+ *           to free this pointer and the variables returned by
+ *           glfs_xreaddirplus_get_*() APIs.
+ *
+ * RETURN VALUE:
+ * >=0: SUCCESS (value contains the flags successfully processed)
+ *  -1: FAILURE
+ */
+int
+glfs_xreaddirplus_r(glfs_fd_t *glfd, uint32_t flags,
+                    glfs_xreaddirp_stat_t **xstat_p, struct dirent *ext,
+                    struct dirent **res) __THROW
+    GFAPI_PUBLIC(glfs_xreaddirplus_r, 3.11.0);
 
-int glfs_statvfs (glfs_t *fs, const char *path, struct statvfs *buf) __THROW
-        GFAPI_PUBLIC(glfs_statvfs, 3.4.0);
+#define GFAPI_MAX_LOCK_OWNER_LEN 255
 
-int glfs_chmod (glfs_t *fs, const char *path, mode_t mode) __THROW
-        GFAPI_PUBLIC(glfs_chmod, 3.4.0);
+/*
+ *
+ * DESCRIPTION
+ *
+ * This API allows application to set lk_owner on a fd.
+ * A glfd can be associated with only single lk_owner. In case if there
+ * is need to set another lk_owner, applications can make use of
+ * 'glfs_dup' to get duplicate glfd and set new lk_owner on that second
+ * glfd.
+ *
+ * Also its not recommended to override or clear lk_owner value as the
+ * same shall be used to flush any outstanding locks while closing the fd.
+ *
+ * PARAMETERS
+ *
+ * INPUT:
+ * @glfd: GFAPI file descriptor
+ * @len: Size of lk_owner buffer. Max value can be GFAPI_MAX_LOCK_OWNER_LEN
+ * @data: lk_owner data buffer.
+ *
+ * OUTPUT:
+ * 0: SUCCESS
+ * -1: FAILURE
+ */
+int
+glfs_fd_set_lkowner(glfs_fd_t *glfd, void *data, int len) __THROW
+    GFAPI_PUBLIC(glfs_fd_set_lkowner, 3.10.7);
 
-int glfs_fchmod (glfs_fd_t *fd, mode_t mode) __THROW
-        GFAPI_PUBLIC(glfs_fchmod, 3.4.0);
+/*
+ * Applications (currently NFS-Ganesha) can make use of this
+ * structure to read upcall notifications sent by server either
+ * by polling or registering a callback function.
+ *
+ * On success, applications need to check for 'reason' to decide
+ * if any upcall event is received.
+ *
+ * Currently supported upcall_events -
+ *      GLFS_UPCALL_INODE_INVALIDATE -
+ *              'event_arg' - glfs_upcall_inode
+ *
+ * After processing the event, applications need to free 'event_arg' with
+ * glfs_free().
+ *
+ * Also similar to I/Os, the application should ideally stop polling
+ * or unregister upcall_cbk function before calling glfs_fini(..).
+ * Hence making an assumption that 'fs' & ctx structures cannot be
+ * freed while in this routine.
+ */
+struct glfs_upcall;
+typedef struct glfs_upcall glfs_upcall_t;
 
-int glfs_chown (glfs_t *fs, const char *path, uid_t uid, gid_t gid) __THROW
-        GFAPI_PUBLIC(glfs_chown, 3.4.0);
+glfs_t *
+glfs_upcall_get_fs(glfs_upcall_t *arg) __THROW
+    GFAPI_PUBLIC(glfs_upcall_get_fs, 3.7.16);
 
-int glfs_lchown (glfs_t *fs, const char *path, uid_t uid, gid_t gid) __THROW
-        GFAPI_PUBLIC(glfs_lchown, 3.4.0);
+enum glfs_upcall_reason {
+    GLFS_UPCALL_EVENT_NULL = 0,
+    GLFS_UPCALL_INODE_INVALIDATE, /* invalidate cache entry */
+    GLFS_UPCALL_RECALL_LEASE,     /* recall lease */
+};
+typedef enum glfs_upcall_reason glfs_upcall_reason_t;
 
-int glfs_fchown (glfs_fd_t *fd, uid_t uid, gid_t gid) __THROW
-        GFAPI_PUBLIC(glfs_fchown, 3.4.0);
+glfs_upcall_reason_t
+glfs_upcall_get_reason(glfs_upcall_t *arg) __THROW
+    GFAPI_PUBLIC(glfs_upcall_get_reason, 3.7.16);
 
-int glfs_utimens (glfs_t *fs, const char *path,
-                  struct timespec times[2]) __THROW
-        GFAPI_PUBLIC(glfs_utimens, 3.4.0);
+/*
+ * Applications first need to make use of above API i.e,
+ * "glfs_upcall_get_reason" to determine which upcall event it has
+ * received. Post that below API - "glfs_upcall_get_event" should
+ * be used to get corresponding upcall event object.
+ *
+ * Below are the upcall_reason and corresponding upcall_event objects:
+ *      ==========================================================
+ *      glfs_upcall_reason           -    event_object
+ *      ==========================================================
+ *      GLFS_UPCALL_EVENT_NULL       -    NULL
+ *      GLFS_UPCALL_INODE_INVALIDATE -    struct glfs_upcall_inode
+ *      GLFS_UPCALL_RECALL_LEASE     -    struct glfs_upcall_lease
+ *
+ * After processing upcall event, glfs_free() should be called on the
+ * glfs_upcall.
+ */
+void *
+glfs_upcall_get_event(glfs_upcall_t *arg) __THROW
+    GFAPI_PUBLIC(glfs_upcall_get_event, 3.7.16);
 
-int glfs_lutimens (glfs_t *fs, const char *path,
-                   struct timespec times[2]) __THROW
-        GFAPI_PUBLIC(glfs_lutimens, 3.4.0);
+/*
+ * SYNOPSIS
+ *
+ * glfs_upcall_cbk: Upcall callback definition
+ *
+ * This is function type definition of the callback function pointer
+ * which has to be provided by the caller while registering for any
+ * upcall events.
+ *
+ * This function is called whenever any upcall which the application
+ * has registered for is received from the server.
+ *
+ * @up_arg: Upcall structure whose contents need to be interpreted by
+ * making use of glfs_upcall_* helper routines.
+ *
+ * @data: The same context pointer provided by the caller at the time of
+ * registering of upcall events. This may be used by the caller for any
+ * of its internal use while processing upcalls.
+ */
+typedef void (*glfs_upcall_cbk)(glfs_upcall_t *up_arg, void *data);
 
-int glfs_futimens (glfs_fd_t *fd, struct timespec times[2]) __THROW
-        GFAPI_PUBLIC(glfs_futimens, 3.4.0);
+/*
+ * List of upcall events supported by gluster/gfapi
+ */
+#define GLFS_EVENT_INODE_INVALIDATE 0x00000001 /* invalidate cache entry */
+#define GLFS_EVENT_RECALL_LEASE 0x00000002     /* Recall lease */
+#define GLFS_EVENT_ANY 0xffffffff              /* for all the above events */
 
-ssize_t glfs_getxattr (glfs_t *fs, const char *path, const char *name,
-		       void *value, size_t size) __THROW
-        GFAPI_PUBLIC(glfs_getxattr, 3.4.0);
+/*
+ * SYNOPSIS
+ *
+ * glfs_upcall_register: Register for upcall events
+ *
+ * DESCRIPTION
+ *
+ * This function is used to register for various upcall events application
+ * is interested in and the callback function to be invoked when such
+ * events are triggered.
+ *
+ * Multiple calls of this routine shall override cbk function. That means
+ * only one cbk function can be used for all the upcall events registered
+ * and that shall be the one last updated.
+ *
+ * PARAMETERS:
+ *
+ * INPUT:
+ * @fs: The 'virtual mount' object
+ *
+ * @event_list: List of upcall events to be registered.
+ *              Current available values are:
+ *               - GLFS_EVENT_INODE_INVALIDATE
+ *               - GLFS_EVENT_RECALL_LEASE
+ *
+ * @cbk: The cbk routine to be invoked in case of any upcall received
+ * @data: Any opaque pointer provided by caller which shall be using while
+ * making cbk calls. This pointer may be used by caller for any of its
+ * internal use while processing upcalls. Can be NULL.
+ *
+ * RETURN VALUE:
+ * >0: SUCCESS (value contains the events successfully registered)
+ * -1: FAILURE
+ */
+int
+glfs_upcall_register(glfs_t *fs, uint32_t event_list, glfs_upcall_cbk cbk,
+                     void *data) __THROW
+    GFAPI_PUBLIC(glfs_upcall_register, 3.13.0);
 
-ssize_t glfs_lgetxattr (glfs_t *fs, const char *path, const char *name,
-			void *value, size_t size) __THROW
-        GFAPI_PUBLIC(glfs_lgetxattr, 3.4.0);
+/*
+ * SYNOPSIS
+ *
+ * glfs_upcall_unregister: Unregister for upcall events
+ *
+ * DESCRIPTION
+ *
+ * This function is used to unregister the upcall events application
+ * is not interested in. In case if the caller unregisters all the events
+ * it has registered for, it shall no more receive any upcall event.
+ *
+ * PARAMETERS:
+ *
+ * INPUT:
+ * @fs: The 'virtual mount' object
+ *
+ * @event_list: List of upcall events to be unregistered.
+ *              Current available values are:
+ *               - GLFS_EVENT_INODE_INVALIDATE
+ *               - GLFS_EVENT_RECALL_LEASE
+ * RETURN VALUE:
+ * >0: SUCCESS (value contains the events successfully unregistered)
+ * -1: FAILURE
+ */
+int
+glfs_upcall_unregister(glfs_t *fs, uint32_t event_list) __THROW
+    GFAPI_PUBLIC(glfs_upcall_unregister, 3.13.0);
+
+/* Lease Types */
+enum glfs_lease_types {
+    GLFS_LEASE_NONE = 0,
+    GLFS_RD_LEASE = 1,
+    GLFS_RW_LEASE = 2,
+};
+typedef enum glfs_lease_types glfs_lease_types_t;
+
+/* Lease cmds */
+enum glfs_lease_cmds {
+    GLFS_GET_LEASE = 1,
+    GLFS_SET_LEASE = 2,
+    GLFS_UNLK_LEASE = 3,
+};
+typedef enum glfs_lease_cmds glfs_lease_cmds_t;
+
+struct glfs_lease {
+    glfs_lease_cmds_t cmd;
+    glfs_lease_types_t lease_type;
+    glfs_leaseid_t lease_id;
+    unsigned int lease_flags;
+};
+typedef struct glfs_lease glfs_lease_t;
+
+typedef void (*glfs_recall_cbk)(glfs_lease_t lease, void *data);
 
-ssize_t glfs_fgetxattr (glfs_fd_t *fd, const char *name,
-			void *value, size_t size) __THROW
-        GFAPI_PUBLIC(glfs_fgetxattr, 3.4.0);
+/*
+  SYNOPSIS
 
-ssize_t glfs_listxattr (glfs_t *fs, const char *path,
-                        void *value, size_t size) __THROW
-        GFAPI_PUBLIC(glfs_listxattr, 3.4.0);
+  glfs_lease: Takes a lease on a file.
 
-ssize_t glfs_llistxattr (glfs_t *fs, const char *path, void *value,
-			 size_t size) __THROW
-        GFAPI_PUBLIC(glfs_llistxattr, 3.4.0);
+  DESCRIPTION
 
-ssize_t glfs_flistxattr (glfs_fd_t *fd, void *value, size_t size) __THROW
-        GFAPI_PUBLIC(glfs_flistxattr, 3.4.0);
+  This function takes lease on an open file.
 
-int glfs_setxattr (glfs_t *fs, const char *path, const char *name,
-		   const void *value, size_t size, int flags) __THROW
-        GFAPI_PUBLIC(glfs_setxattr, 3.4.0);
+  PARAMETERS
 
-int glfs_lsetxattr (glfs_t *fs, const char *path, const char *name,
-		    const void *value, size_t size, int flags) __THROW
-        GFAPI_PUBLIC(glfs_lsetxattr, 3.4.0);
+  @glfd: The fd of the file on which lease should be taken,
+   this fd is returned by glfs_open/glfs_create.
 
-int glfs_fsetxattr (glfs_fd_t *fd, const char *name,
-		    const void *value, size_t size, int flags) __THROW
-        GFAPI_PUBLIC(glfs_fsetxattr, 3.4.0);
+  @lease: Struct that defines the lease operation to be performed
+   on the file.
+      @lease.cmd - Can be one of the following values
+         GF_GET_LEASE:  Get the lease type currently present on the file,
+                        lease.lease_type will contain GF_RD_LEASE
+                        or GF_RW_LEASE or 0 if no leases.
+         GF_SET_LEASE:  Set the lease of given lease.lease_type on the file.
+         GF_UNLK_LEASE: Unlock the lease present on the given fd.
+                        Note that the every lease request should have
+                        a corresponding unlk_lease.
 
-int glfs_removexattr (glfs_t *fs, const char *path, const char *name) __THROW
-        GFAPI_PUBLIC(glfs_removexattr, 3.4.0);
+      @lease.lease_type - Can be one of the following values
+         GF_RD_LEASE:   Read lease on a file, shared lease.
+         GF_RW_LEASE:   Read-Write lease on a file, exclusive lease.
 
-int glfs_lremovexattr (glfs_t *fs, const char *path, const char *name) __THROW
-        GFAPI_PUBLIC(glfs_lremovexattr, 3.4.0);
+      @lease.lease_id - A unique identification of lease, 128bits.
 
-int glfs_fremovexattr (glfs_fd_t *fd, const char *name) __THROW
-        GFAPI_PUBLIC(glfs_fremovexattr, 3.4.0);
+  @fn: This is the function that is invoked when the lease has to be recalled
+  @data: It is a cookie, this pointer is returned as a part of recall
+
+  fn and data field are stored as a part of glfs_fd, hence if there are multiple
+  glfs_lease calls, each of them updates the fn and data fields. glfs_recall_cbk
+  will be invoked with the last updated fn and data
+
+  RETURN VALUES
+  0:  Successful completion
+  <0: Failure. @errno will be set with the type of failure
+*/
 
-int glfs_fallocate(glfs_fd_t *fd, int keep_size,
-                   off_t offset, size_t len) __THROW
-        GFAPI_PUBLIC(glfs_fallocate, 3.5.0);
+int
+glfs_lease(glfs_fd_t *glfd, glfs_lease_t *lease, glfs_recall_cbk fn,
+           void *data) __THROW GFAPI_PUBLIC(glfs_lease, 4.0.0);
 
-int glfs_discard(glfs_fd_t *fd, off_t offset, size_t len) __THROW
-        GFAPI_PUBLIC(glfs_discard, 3.5.0);
+/*
+  SYNOPSIS
 
+  glfs_fsetattr: Function to set attributes.
+  glfs_setattr: Function to set attributes
 
-int glfs_discard_async (glfs_fd_t *fd, off_t length, size_t lent,
-			glfs_io_cbk fn, void *data) __THROW
-        GFAPI_PUBLIC(glfs_discard_async, 3.5.0);
+  DESCRIPTION
 
-int glfs_zerofill(glfs_fd_t *fd, off_t offset, off_t len) __THROW
-        GFAPI_PUBLIC(glfs_zerofill, 3.5.0);
+  The functions are used to set attributes on the file.
 
-int glfs_zerofill_async (glfs_fd_t *fd, off_t length, off_t len,
-                        glfs_io_cbk fn, void *data) __THROW
-        GFAPI_PUBLIC(glfs_zerofill_async, 3.5.0);
+  PARAMETERS
 
-char *glfs_getcwd (glfs_t *fs, char *buf, size_t size) __THROW
-        GFAPI_PUBLIC(glfs_getcwd, 3.4.0);
+  @glfs_fsetattr
 
-int glfs_chdir (glfs_t *fs, const char *path) __THROW
-        GFAPI_PUBLIC(glfs_chdir, 3.4.0);
+     @glfd: The fd of the file for which the attributes are to be set,
+            this fd is returned by glfs_open/glfs_create.
 
-int glfs_fchdir (glfs_fd_t *fd) __THROW
-        GFAPI_PUBLIC(glfs_fchdir, 3.4.0);
+  @glfs_setattr
 
-char *glfs_realpath (glfs_t *fs, const char *path, char *resolved_path) __THROW
-        GFAPI_PUBLIC(glfs_realpath, 3.4.0);
+         @fs: File object.
+
+         @path: The path of the file that is being operated on.
+
+         @follow: Flag used to resolve symlink.
+
+
+  @stat: Struct that has information about the file.
+
+  @valid: This is the mask bit, that accepts GFAPI_SET_ATTR* masks.
+          Refer glfs.h to see the mask definitions.
+
+  Both functions are similar in functionality, just that the
+  func setattr() uses file path whereas the func fsetattr()
+  uses the fd.
+
+  RETURN VALUES
+  0:  Successful completion
+  <0: Failure. @errno will be set with the type of failure
 
-/*
- * @cmd and @flock are as specified in man fcntl(2).
  */
-int glfs_posix_lock (glfs_fd_t *fd, int cmd, struct flock *flock) __THROW
-        GFAPI_PUBLIC(glfs_posix_lock, 3.4.0);
 
-glfs_fd_t *glfs_dup (glfs_fd_t *fd) __THROW
-        GFAPI_PUBLIC(glfs_dup, 3.4.0);
+int
+glfs_fsetattr(struct glfs_fd *glfd, struct glfs_stat *stat) __THROW
+    GFAPI_PUBLIC(glfs_fsetattr, 6.0);
+
+int
+glfs_setattr(struct glfs *fs, const char *path, struct glfs_stat *stat,
+             int follow) __THROW GFAPI_PUBLIC(glfs_setattr, 6.0);
 
 /*
- * No xdata support for now.  Nobody needs this call at all yet except for the
- * test script, and that doesn't need xdata.  Adding dict_t support and a new
- * header-file requirement doesn't seem worth it until the need is greater.
+  SYNOPSIS
+
+  glfs_set_statedump_path: Function to set statedump path.
+
+  DESCRIPTION
+
+  This function is used to set statedump directory
+
+  PARAMETERS
+
+  @fs: The 'virtual mount' object to be configured with the volume
+       specification file.
+
+  @path: statedump path. Should be a directory. But the API won't fail if the
+  directory doesn't exist yet, as one may create it later.
+
+  RETURN VALUES
+
+   0 : Success.
+  -1 : Failure. @errno will be set with the type of failure.
+
  */
-int glfs_ipc (glfs_fd_t *fd, int cmd) __THROW
-        GFAPI_PUBLIC(glfs_ipc, 3.7.0);
 
-__END_DECLS
+int
+glfs_set_statedump_path(struct glfs *fs, const char *path) __THROW
+    GFAPI_PUBLIC(glfs_set_statedump_path, 7.0);
 
+__END_DECLS
 #endif /* !_GLFS_H */
diff --git a/autogen.sh b/autogen.sh
index fef2f037426..c8cdc3f89fa 100755
--- a/autogen.sh
+++ b/autogen.sh
@@ -85,11 +85,7 @@ $TOOL --automake --copy --force
 echo Running ${AUTOCONF}...
 $AUTOCONF
 echo Running ${AUTOMAKE}...
-$AUTOMAKE --add-missing --copy --foreign
-
-# Run autogen in the argp-standalone sub-directory
-echo "Running autogen.sh in argp-standalone ..."
-( cd contrib/argp-standalone;./autogen.sh )
+$AUTOMAKE --add-missing --force-missing --copy
 
 # Instruct user on next steps
 echo
diff --git a/extras/checkpatch.pl b/build-aux/checkpatch.pl
index 5a4e018def9..17ae4e4d579 100755
--- a/extras/checkpatch.pl
+++ b/build-aux/checkpatch.pl
@@ -612,7 +612,6 @@ sub top_of_glusterfs_tree {
         "geo-replication",
         "glusterfs-api.pc.in",
         "glusterfsd",
-        "glusterfs-hadoop",
         "glusterfs.spec.in",
         "heal",
         "INSTALL",
@@ -2084,6 +2083,8 @@ sub process {
                  "Use of $flag is deprecated, please use \`$replacement->{$flag} instead.\n" . $herecurr) if ($replacement->{$flag});
         }
 
+# check we are in .spec file, then ignore this hunk
+        next if ($realfile eq "glusterfs.spec.in");
 
 # check we are in a valid source file if not then ignore this hunk
         next if ($realfile !~ /\.(h|c|pl|py|l|y|sh|in)$/);
@@ -2633,17 +2634,6 @@ sub process {
             }
         }
 
-# check for new typedefs, only function parameters and sparse annotations
-# make sense.
-        if ($line =~ /\btypedef\s/ &&
-            $line !~ /\btypedef\s+$Type\s*\(\s*\*?$Ident\s*\)\s*\(/ &&
-            $line !~ /\btypedef\s+$Type\s+$Ident\s*\(/ &&
-            $line !~ /\b$typeTypedefs\b/ &&
-            $line !~ /\b__bitwise(?:__|)\b/) {
-            WARN("NEW_TYPEDEFS",
-                 "do not add new typedefs\n" . $herecurr);
-        }
-
 # * goes on variable not on type
         # (char*[ const])
         while ($line =~ m{(\($NonptrType(\s*(?:$Modifier\b\s*|\*\s*)+)\))}g) {
@@ -3109,7 +3099,7 @@ sub process {
             if (ERROR("SPACING",
                       "space required before the open brace '{'\n" . $herecurr) &&
                 $fix) {
-                $fixed[$linenr - 1] =~ s/^(\+.*(?:do|\))){/$1 {/;
+                $fixed[$linenr - 1] =~ s/^(\+.*(?:do|\)))\{/$1 {/;
             }
         }
 
diff --git a/build-aux/config.guess.dist b/build-aux/config.guess.dist
new file mode 100755
index 00000000000..881ba7a0438
--- /dev/null
+++ b/build-aux/config.guess.dist
@@ -0,0 +1,14 @@
+#!/bin/sh
+#
+# This script is intentionally left empty. Distributions that package GlusterFS
+# may want to to replace it with an updated copy from the automake project.
+#
+
+cat << EOM
+It is not expected to execute this script. When you are building from a
+released tarball (generated with 'make dist'), you are expected to pass
+--build=... and --host=... to ./configure or replace this config.guess script
+in the sources with an updated version.
+EOM
+
+exit 0
diff --git a/build-aux/config.sub.dist b/build-aux/config.sub.dist
new file mode 100755
index 00000000000..c5a0dbad282
--- /dev/null
+++ b/build-aux/config.sub.dist
@@ -0,0 +1,14 @@
+#!/bin/sh
+#
+# This script is intentionally left empty. Distributions that package GlusterFS
+# may want to to replace it with an updated copy from the automake project.
+#
+
+cat << EOM
+It is not expected to execute this script. When you are building from a
+released tarball (generated with 'make dist'), you are expected to pass
+--build=... and --host=... to ./configure or replace this config.sub script in
+the sources with an updated version.
+EOM
+
+exit 0
diff --git a/build-aux/pkg-version b/build-aux/pkg-version
index 83d4a5f9136..17ceab70c03 100755
--- a/build-aux/pkg-version
+++ b/build-aux/pkg-version
@@ -1,9 +1,13 @@
-#!/bin/sh
+#!/bin/bash
 
 # To override version/release from git,
 # create VERSION file containing text with version/release
 # eg. v3.4.0-1
-PKG_VERSION=`cat VERSION 2> /dev/null || git describe --tags --match "v[0-9]*"`
+
+# One thing to note, If one does 'git clone --depth N glusterfs.git',
+# the git describe command doesn't work. Hence you notice below that
+# we have added timestamp as version (YYYY.MM.DD) and release (HH.mmss)
+PKG_VERSION=`cat VERSION 2> /dev/null || git describe --tags --match "v[0-9]*" 2>/dev/null`
 
 get_version()
 {
@@ -18,7 +22,11 @@ get_version()
       sub(/^v/,"") ; print $1
     }'
 
-    echo $PKG_VERSION | awk "$AWK_VERSION" | tr -cd '[:alnum:].'
+    version=$(echo $PKG_VERSION | awk "$AWK_VERSION" | tr -cd '[:alnum:].')
+    if [ "x${version}" == "x" ] ; then
+       version=$(date +%Y.%m.%d | tr -d '\n')
+    fi
+    echo $version | tr -d '\n'
 }
 
 get_release()
@@ -37,7 +45,11 @@ get_release()
       else if (NF == 4) print $2, $3, "git" substr($4, 2)
     }'
 
-    echo $PKG_VERSION | awk "$AWK_RELEASE" | tr -cd '[:alnum:].'
+    release=$(echo $PKG_VERSION | awk "$AWK_RELEASE" | tr -cd '[:alnum:].')
+    if [ "x${release}" == "x" ] ; then
+       release=$(date +%H.%M%S | tr -d '\n')
+    fi
+    echo $release | tr -d '\n'
 }
 
 if test "x$1" = "x--full"; then
diff --git a/build-aux/xdrgen b/build-aux/xdrgen
deleted file mode 100755
index 0cefc9b4890..00000000000
--- a/build-aux/xdrgen
+++ /dev/null
@@ -1,104 +0,0 @@
-#!/bin/sh
-
-append_licence_header ()
-{
-    local src_file=$1;
-    local dst_file=$2;
-
-    cat >$dst_file <<EOF
-/*
-  Copyright (c) 2007-2014 Red Hat, Inc. <http://www.redhat.com>
-  This file is part of GlusterFS.
-
-  This file is licensed to you under your choice of the GNU Lesser
-  General Public License, version 3 or any later version (LGPLv3 or
-  later), or the GNU General Public License, version 2 (GPLv2), in all
-  cases as published by the Free Software Foundation.
-*/
-
-#include "compat.h"
-#include "xdr-common.h"
-#include "xdr-nfs3.h"
-
-#if defined(__GNUC__)
-#if __GNUC__ >= 4
-#if !defined(__clang__)
-#if !defined(__NetBSD__)
-#pragma GCC diagnostic ignored "-Wunused-but-set-variable"
-#pragma GCC diagnostic ignored "-Wunused-variable"
-#endif
-#else
-#pragma clang diagnostic ignored "-Wunused-variable"
-#pragma clang diagnostic ignored "-Wunused-value"
-#endif
-#endif
-#endif
-
-EOF
-
-    cat $src_file >> $dst_file;
-
-}
-
-gen_sources ()
-{
-    local xfile="$1";
-
-    local cfile="${1%.x}.c";
-    local hfile="${1%.x}.h";
-
-    local tmp_cfile="$1.c.tmp";
-    local tmp_hfile="$1.h.tmp";
-
-    rm -f $cfile;
-    rpcgen -c -o $cfile $xfile;
-    append_licence_header $cfile $tmp_cfile;
-    mv $tmp_cfile $cfile;
-    # remove unwanted temporary files (if any)
-    rm -f $tmp_cfile;
-    echo "Generated $cfile"
-    return
-}
-
-gen_headers ()
-{
-    local xfile="$1";
-
-    local cfile="${1%.x}.c";
-    local hfile="${1%.x}.h";
-
-    local tmp_cfile="$1.c.tmp";
-    local tmp_hfile="$1.h.tmp";
-
-    # no need for a temporary file here as there are no changes from glusterfs
-    rm -f $hfile;
-    rpcgen -h -o $hfile $xfile;
-    # the '#ifdef' part of file should be fixed
-    sed -e 's/-/_/g' $hfile > ${hfile}.new && mv ${hfile}.new $hfile;
-    # Gen header to temp file and append generated file
-    append_licence_header $hfile $tmp_hfile;
-    # now move the destination file to actual original file
-    mv $tmp_hfile $hfile;
-    rm -f $tmp_hfile;
-    echo "Generated $hfile";
-    return
-}
-
-main ()
-{
-    if [ $# -ne 2 ]; then
-        echo "wrong number of arguments given"
-        echo " $0 [header|source] <XDR-definition-file>.x"
-        exit 1;
-    fi
-
-    if [ $1 = "header" ]; then
-        gen_headers $2
-    fi
-
-    if [ $1 = "source" ]; then
-        gen_sources $2
-    fi
-}
-
-main "$@";
diff --git a/cli/src/Makefile.am b/cli/src/Makefile.am
index 91819d111e3..16063f27c7f 100644
--- a/cli/src/Makefile.am
+++ b/cli/src/Makefile.am
@@ -5,6 +5,7 @@ gluster_SOURCES = cli.c registry.c input.c cli-cmd.c cli-rl.c cli-cmd-global.c \
 	 cli-cmd-system.c cli-cmd-misc.c cli-xml-output.c cli-quotad-client.c cli-cmd-snapshot.c
 
 gluster_LDADD = $(top_builddir)/libglusterfs/src/libglusterfs.la $(GF_LDADD) \
+		$(top_builddir)/libglusterd/src/libglusterd.la \
 		$(RLLIBS) $(top_builddir)/rpc/xdr/src/libgfxdr.la \
 		$(top_builddir)/rpc/rpc-lib/src/libgfrpc.la \
 		$(XML_LIBS)
@@ -13,20 +14,24 @@ gluster_LDFLAGS = $(GF_LDFLAGS)
 noinst_HEADERS = cli.h cli-mem-types.h cli-cmd.h cli-quotad-client.h
 
 AM_CPPFLAGS = $(GF_CPPFLAGS) \
-	-I$(top_srcdir)/libglusterfs/src -I$(top_srcdir)/rpc/rpc-lib/src\
-	-I$(top_srcdir)/rpc/xdr/src\
+	-I$(top_srcdir)/libglusterfs/src -I$(top_srcdir)/rpc/rpc-lib/src \
+	-I$(top_srcdir)/rpc/xdr/src -I$(top_srcdir)/libglusterd/src \
+	-I$(top_builddir)/rpc/xdr/src \
 	-DDATADIR=\"$(localstatedir)\" \
 	-DCONFDIR=\"$(sysconfdir)/glusterfs\" \
-	-DGSYNCD_PREFIX=\"$(libexecdir)/glusterfs\"\
-	-DSYNCDAEMON_COMPILE=$(SYNCDAEMON_COMPILE) -DSBIN_DIR=\"$(sbindir)\"\
-	$(XML_CPPFLAGS)
+	-DGSYNCD_PREFIX=\"$(GLUSTERFS_LIBEXECDIR)\"\
+	-DGLFSHEAL_PREFIX=\"$(GLUSTERFS_LIBEXECDIR)\"\
+	-DSYNCDAEMON_COMPILE=$(SYNCDAEMON_COMPILE)
 
-AM_CFLAGS = -Wall $(GF_CFLAGS)
+AM_CFLAGS = -Wall $(GF_CFLAGS) $(XML_CFLAGS)
 
 CLEANFILES =
 
 $(top_builddir)/libglusterfs/src/libglusterfs.la:
 	$(MAKE) -C $(top_builddir)/libglusterfs/src/ all
 
+$(top_builddir)/libglusterd/src/libglusterd.la:
+	$(MAKE) -C $(top_builddir)/libglusterd/src/ all
+
 install-data-hook:
 	$(mkdir_p) $(DESTDIR)$(localstatedir)/run/gluster
diff --git a/cli/src/cli-cmd-global.c b/cli/src/cli-cmd-global.c
index 53ee0ab2517..2c9a5f01bb1 100644
--- a/cli/src/cli-cmd-global.c
+++ b/cli/src/cli-cmd-global.c
@@ -23,113 +23,173 @@
 #include "cli-cmd.h"
 #include "cli-mem-types.h"
 #include "cli1-xdr.h"
-#include "run.h"
-#include "syscall.h"
-#include "common-utils.h"
-
-extern rpc_clnt_prog_t *cli_rpc_prog;
+#include <glusterfs/run.h>
+#include <glusterfs/syscall.h>
+#include <glusterfs/common-utils.h>
 
 int
-cli_cmd_global_help_cbk (struct cli_state *state, struct cli_cmd_word *in_word,
-                         const char **words, int wordcount);
-int cli_cmd_ganesha_cbk (struct cli_state *state, struct cli_cmd_word *word,
-                                         const char **words, int wordcount);
+cli_cmd_global_help_cbk(struct cli_state *state, struct cli_cmd_word *in_word,
+                        const char **words, int wordcount);
+int
+cli_cmd_get_state_cbk(struct cli_state *state, struct cli_cmd_word *word,
+                      const char **words, int wordcount);
 
+int
+cli_cmd_ganesha_cbk(struct cli_state *state, struct cli_cmd_word *word,
+                    const char **words, int wordcount);
 
 struct cli_cmd global_cmds[] = {
-        { "global help",
-           cli_cmd_global_help_cbk,
-           "list global commands",
-        },
-        { "nfs-ganesha {enable| disable} ",
-           cli_cmd_ganesha_cbk,
-          "Enable/disable NFS-Ganesha support",
-        },
-        {NULL,  NULL,  NULL}
-};
+    {
+        "global help",
+        cli_cmd_global_help_cbk,
+        "list global commands",
+    },
+    {
+        "get-state [<daemon>] [[odir </path/to/output/dir/>] "
+        "[file <filename>]] [detail|volumeoptions]",
+        cli_cmd_get_state_cbk,
+        "Get local state representation of mentioned daemon",
+    },
+    {
+        "nfs-ganesha {enable| disable} ",
+        cli_cmd_ganesha_cbk,
+        "Enable/disable NFS-Ganesha support",
+    },
+    {NULL, NULL, NULL}};
 
 int
-cli_cmd_global_help_cbk (struct cli_state *state, struct cli_cmd_word *in_word,
-                      const char **words, int wordcount)
+cli_cmd_global_help_cbk(struct cli_state *state, struct cli_cmd_word *in_word,
+                        const char **words, int wordcount)
 {
-        struct cli_cmd        *cmd = NULL;
-        struct cli_cmd        *global_cmd = NULL;
-        int                   count     = 0;
-
-        cmd = GF_CALLOC (1, sizeof (global_cmds), cli_mt_cli_cmd);
-        memcpy (cmd, global_cmds, sizeof (global_cmds));
-        count = (sizeof (global_cmds) / sizeof (struct cli_cmd));
-        cli_cmd_sort (cmd, count);
-
-        for (global_cmd = cmd; global_cmd->pattern; global_cmd++)
-                if (_gf_false == global_cmd->disable)
-                        cli_out ("%s - %s", global_cmd->pattern,
-                                 global_cmd->desc);
-
-        GF_FREE (cmd);
-        return 0;
+    struct cli_cmd *cmd = NULL;
+    struct cli_cmd *global_cmd = NULL;
+    int count = 0;
+
+    cmd = GF_MALLOC(sizeof(global_cmds), cli_mt_cli_cmd);
+    memcpy(cmd, global_cmds, sizeof(global_cmds));
+    count = (sizeof(global_cmds) / sizeof(struct cli_cmd));
+    cli_cmd_sort(cmd, count);
+
+    cli_out("\ngluster global commands");
+    cli_out("========================\n");
+    for (global_cmd = cmd; global_cmd->pattern; global_cmd++)
+        if (_gf_false == global_cmd->disable)
+            cli_out("%s - %s", global_cmd->pattern, global_cmd->desc);
+
+    cli_out("\n");
+    GF_FREE(cmd);
+    return 0;
 }
 
 int
-cli_cmd_global_register (struct cli_state *state)
+cli_cmd_global_register(struct cli_state *state)
 {
-        int ret = 0;
-        struct cli_cmd *cmd =  NULL;
-        for (cmd = global_cmds; cmd->pattern; cmd++) {
-                ret = cli_cmd_register (&state->tree, cmd);
-                        if (ret)
-                                goto out;
-        }
+    int ret = 0;
+    struct cli_cmd *cmd = NULL;
+    for (cmd = global_cmds; cmd->pattern; cmd++) {
+        ret = cli_cmd_register(&state->tree, cmd);
+        if (ret)
+            goto out;
+    }
 out:
-        return ret;
-
+    return ret;
 }
 
-int cli_cmd_ganesha_cbk (struct cli_state *state, struct cli_cmd_word *word,
-                         const char **words, int wordcount)
+int
+cli_cmd_ganesha_cbk(struct cli_state *state, struct cli_cmd_word *word,
+                    const char **words, int wordcount)
 
 {
-         int                     sent        =   0;
-         int                     parse_error =   0;
-         int                     ret         =  -1;
-         rpc_clnt_procedure_t    *proc       =  NULL;
-         call_frame_t            *frame      =  NULL;
-         dict_t                  *options    =  NULL;
-         cli_local_t             *local      =  NULL;
-         char                    *op_errstr  =  NULL;
-
-        proc = &cli_rpc_prog->proctable[GLUSTER_CLI_GANESHA];
-
-        frame = create_frame (THIS, THIS->ctx->pool);
-        if (!frame)
-                goto out;
-
-        ret = cli_cmd_ganesha_parse (state, words, wordcount,
-                                     &options, &op_errstr);
-        if (ret) {
-                if (op_errstr) {
-                    cli_err ("%s", op_errstr);
-                    GF_FREE (op_errstr);
-                } else
-                    cli_usage_out (word->pattern);
-                parse_error = 1;
-                goto out;
-        }
-
-        CLI_LOCAL_INIT (local, words, frame, options);
-
-        if (proc->fn) {
-                ret = proc->fn (frame, THIS, options);
-        }
+    int sent = 0;
+    int parse_error = 0;
+    int ret = -1;
+    rpc_clnt_procedure_t *proc = NULL;
+    call_frame_t *frame = NULL;
+    dict_t *options = NULL;
+    cli_local_t *local = NULL;
+    char *op_errstr = NULL;
+
+    proc = &cli_rpc_prog->proctable[GLUSTER_CLI_GANESHA];
+
+    frame = create_frame(THIS, THIS->ctx->pool);
+    if (!frame)
+        goto out;
+
+    ret = cli_cmd_ganesha_parse(state, words, wordcount, &options, &op_errstr);
+    if (ret) {
+        if (op_errstr) {
+            cli_err("%s", op_errstr);
+            GF_FREE(op_errstr);
+        } else
+            cli_usage_out(word->pattern);
+        parse_error = 1;
+        goto out;
+    }
+
+    CLI_LOCAL_INIT(local, words, frame, options);
+
+    if (proc->fn) {
+        ret = proc->fn(frame, THIS, options);
+    }
 
 out:
-        if (ret) {
-                cli_cmd_sent_status_get (&sent);
-                if ((sent == 0) && (parse_error == 0))
-                        cli_out ("Setting global option failed");
-        }
-
-        CLI_STACK_DESTROY (frame);
-        return ret;
+    if (ret) {
+        cli_cmd_sent_status_get(&sent);
+        if ((sent == 0) && (parse_error == 0))
+            cli_out("Setting global option failed");
+    }
+
+    CLI_STACK_DESTROY(frame);
+    return ret;
 }
 
+int
+cli_cmd_get_state_cbk(struct cli_state *state, struct cli_cmd_word *word,
+                      const char **words, int wordcount)
+{
+    int sent = 0;
+    int parse_error = 0;
+    int ret = -1;
+    rpc_clnt_procedure_t *proc = NULL;
+    call_frame_t *frame = NULL;
+    dict_t *options = NULL;
+    cli_local_t *local = NULL;
+    char *op_errstr = NULL;
+
+    ret = cli_cmd_get_state_parse(state, words, wordcount, &options,
+                                  &op_errstr);
+
+    if (ret) {
+        if (op_errstr) {
+            cli_err("%s", op_errstr);
+            cli_usage_out(word->pattern);
+            GF_FREE(op_errstr);
+        } else
+            cli_usage_out(word->pattern);
+
+        parse_error = 1;
+        goto out;
+    }
+
+    frame = create_frame(THIS, THIS->ctx->pool);
+    if (!frame) {
+        ret = -1;
+        goto out;
+    }
+
+    CLI_LOCAL_INIT(local, words, frame, options);
+
+    proc = &cli_rpc_prog->proctable[GLUSTER_CLI_GET_STATE];
+    if (proc->fn)
+        ret = proc->fn(frame, THIS, options);
+out:
+    if (ret) {
+        cli_cmd_sent_status_get(&sent);
+        if ((sent == 0) && (parse_error == 0))
+            cli_out("Getting daemon state failed");
+    }
+
+    CLI_STACK_DESTROY(frame);
+
+    return ret;
+}
diff --git a/cli/src/cli-cmd-misc.c b/cli/src/cli-cmd-misc.c
index 9f8c159f073..e961d88da86 100644
--- a/cli/src/cli-cmd-misc.c
+++ b/cli/src/cli-cmd-misc.c
@@ -18,11 +18,9 @@
 #include "cli-mem-types.h"
 #include "protocol-common.h"
 
-extern struct rpc_clnt *global_rpc;
-
-extern rpc_clnt_prog_t *cli_rpc_prog;
-
 extern struct cli_cmd volume_cmds[];
+extern struct cli_cmd bitrot_cmds[];
+extern struct cli_cmd quota_cmds[];
 extern struct cli_cmd cli_probe_cmds[];
 extern struct cli_cmd cli_log_cmds[];
 extern struct cli_cmd cli_system_cmds[];
@@ -32,63 +30,88 @@ extern struct cli_cmd global_cmds[];
 struct cli_cmd cli_misc_cmds[];
 
 int
-cli_cmd_quit_cbk (struct cli_state *state, struct cli_cmd_word *word,
-                   const char **words, int wordcount)
+cli_cmd_quit_cbk(struct cli_state *state, struct cli_cmd_word *word,
+                 const char **words, int wordcount)
 {
-        exit (0);
+    exit(0);
 }
 
-int
-cli_cmd_display_help (struct cli_state *state, struct cli_cmd_word *in_word,
-                      const char **words, int wordcount)
+static gf_boolean_t
+cli_is_help_command(const char *pattern)
 {
-        struct cli_cmd        *cmd[] = {volume_cmds, cli_probe_cmds,
-                                       cli_misc_cmds, snapshot_cmds,
-                                       global_cmds, NULL};
-        struct cli_cmd        *cmd_ind = NULL;
-        int                   i = 0;
-
-         /* cli_system_cmds commands for internal usage
-           they are not exposed
-         */
-        for (i=0; cmd[i]!=NULL; i++)
-                for (cmd_ind = cmd[i]; cmd_ind->pattern; cmd_ind++)
-                        if (_gf_false == cmd_ind->disable)
-                                cli_out ("%s - %s", cmd_ind->pattern,
-                                         cmd_ind->desc);
-
-        return 0;
+    /* FixFixFix
+     * This is not the best way to determine whether
+     * this is a help command
+     */
+    if (strstr(pattern, "help"))
+        return _gf_true;
+
+    return _gf_false;
 }
 
-struct cli_cmd cli_misc_cmds[] = {
-        { "quit",
-          cli_cmd_quit_cbk,
-          "quit"},
+int
+cli_cmd_display_help(struct cli_state *state, struct cli_cmd_word *in_word,
+                     const char **words, int wordcount)
+{
+    static struct cli_cmd *cmd[] = {
+        cli_misc_cmds, cli_probe_cmds, volume_cmds, bitrot_cmds,
+        quota_cmds,    snapshot_cmds,  global_cmds, NULL};
+    struct cli_cmd *cmd_ind = NULL;
+    int i = 0;
+    gf_boolean_t list_all = _gf_false;
+
+    /* cli_system_cmds commands for internal usage
+      they are not exposed
+    */
+
+    /* If "help all" */
+    if (wordcount == 2)
+        list_all = _gf_true;
+
+    for (i = 0; cmd[i] != NULL; i++) {
+        for (cmd_ind = cmd[i]; cmd_ind->pattern; cmd_ind++) {
+            if ((_gf_false == cmd_ind->disable) &&
+                cli_is_help_command(cmd_ind->pattern)) {
+                if (list_all && (cmd_ind->cbk)) {
+                    cmd_ind->cbk(state, in_word, words, wordcount);
+                } else {
+                    cli_out(" %-25s- %s", cmd_ind->pattern, cmd_ind->desc);
+                }
+            }
+        }
+    }
 
-        { "help",
-           cli_cmd_display_help,
-           "display command options"},
+    cli_out("\n");
+    return 0;
+}
+
+struct cli_cmd cli_help_cmds[] = {
+    {"help [all]", cli_cmd_display_help, "display help for command classes"},
 
-        { "exit",
-          cli_cmd_quit_cbk,
-          "exit"},
+    {NULL, NULL, NULL}};
 
-        { NULL, NULL, NULL }
-};
+struct cli_cmd cli_misc_cmds[] = {{"quit", cli_cmd_quit_cbk, "quit"},
+                                  {"exit", cli_cmd_quit_cbk, "exit"},
 
+                                  {NULL, NULL, NULL}};
 
 int
-cli_cmd_misc_register (struct cli_state *state)
+cli_cmd_misc_register(struct cli_state *state)
 {
-        int  ret = 0;
-        struct cli_cmd *cmd = NULL;
-
-        for (cmd = cli_misc_cmds; cmd->pattern; cmd++) {
-
-                ret = cli_cmd_register (&state->tree, cmd);
-                if (ret)
-                        goto out;
-        }
+    int ret = 0;
+    struct cli_cmd *cmd = NULL;
+
+    for (cmd = cli_misc_cmds; cmd->pattern; cmd++) {
+        ret = cli_cmd_register(&state->tree, cmd);
+        if (ret)
+            goto out;
+    }
+
+    for (cmd = cli_help_cmds; cmd->pattern; cmd++) {
+        ret = cli_cmd_register(&state->tree, cmd);
+        if (ret)
+            goto out;
+    }
 out:
-        return ret;
+    return ret;
 }
diff --git a/cli/src/cli-cmd-parser.c b/cli/src/cli-cmd-parser.c
index 9254a3e34bd..34620b4a31b 100644
--- a/cli/src/cli-cmd-parser.c
+++ b/cli/src/cli-cmd-parser.c
@@ -12,2412 +12,2757 @@
 #include <stdlib.h>
 #include <stdint.h>
 #include <pthread.h>
+#include <fnmatch.h>
+#include <time.h>
 
 #include "cli.h"
 #include "cli-cmd.h"
 #include "cli-mem-types.h"
-#include "dict.h"
+#include <glusterfs/dict.h>
+#include <glusterfs/list.h>
 
 #include "protocol-common.h"
 #include "cli1-xdr.h"
 
 #define MAX_SNAP_DESCRIPTION_LEN 1024
 
-struct snap_config_opt_vals_ snap_confopt_vals[] = {
-        {.op_name        = "snap-max-hard-limit",
-         .question       = "Changing snapshot-max-hard-limit "
-                           "will limit the creation of new snapshots "
-                           "if they exceed the new limit.\n"
-                           "Do you want to continue?"
-        },
-        {.op_name        = "snap-max-soft-limit",
-         .question       = "If Auto-delete is enabled, snap-max-soft-limit will"
-                           " trigger deletion of oldest snapshot, on the "
-                           "creation of new snapshot, when the "
-                           "snap-max-soft-limit is reached.\n"
-                           "Do you want to change the snap-max-soft-limit?"
-        },
-        {.op_name        = "both",
-        .question        = "Changing snapshot-max-hard-limit "
-                           "will limit the creation of new snapshots "
-                           "if they exceed the new snapshot-max-hard-limit.\n"
-                           "If Auto-delete is enabled, snap-max-soft-limit will"
-                           " trigger deletion of oldest snapshot, on the "
-                           "creation of new snapshot, when the "
-                           "snap-max-soft-limit is reached.\n"
-                           "Do you want to continue?"
-        },
-        {.op_name        = NULL,
-        }
-};
+static struct snap_config_opt_vals_ snap_confopt_vals[] = {
+    {.op_name = "snap-max-hard-limit",
+     .question = "Changing snapshot-max-hard-limit "
+                 "will limit the creation of new snapshots "
+                 "if they exceed the new limit.\n"
+                 "Do you want to continue?"},
+    {.op_name = "snap-max-soft-limit",
+     .question = "If Auto-delete is enabled, snap-max-soft-limit will"
+                 " trigger deletion of oldest snapshot, on the "
+                 "creation of new snapshot, when the "
+                 "snap-max-soft-limit is reached.\n"
+                 "Do you want to change the snap-max-soft-limit?"},
+    {.op_name = "both",
+     .question = "Changing snapshot-max-hard-limit "
+                 "will limit the creation of new snapshots "
+                 "if they exceed the new snapshot-max-hard-limit.\n"
+                 "If Auto-delete is enabled, snap-max-soft-limit will"
+                 " trigger deletion of oldest snapshot, on the "
+                 "creation of new snapshot, when the "
+                 "snap-max-soft-limit is reached.\n"
+                 "Do you want to continue?"},
+    {
+        .op_name = NULL,
+    }};
 
 enum cli_snap_config_set_types {
-        GF_SNAP_CONFIG_SET_HARD = 0,
-        GF_SNAP_CONFIG_SET_SOFT = 1,
-        GF_SNAP_CONFIG_SET_BOTH = 2,
+    GF_SNAP_CONFIG_SET_HARD = 0,
+    GF_SNAP_CONFIG_SET_SOFT = 1,
+    GF_SNAP_CONFIG_SET_BOTH = 2,
 };
 typedef enum cli_snap_config_set_types cli_snap_config_set_types;
 
+typedef struct _cli_brick {
+    struct list_head list;
+    const char *name;
+    int32_t len;
+} cli_brick_t;
+
 int
-cli_cmd_validate_volume (char *volname);
+cli_cmd_validate_volume(char *volname);
 
 static const char *
-id_sel (void *wcon)
+id_sel(void *wcon)
 {
-        return (const char *)wcon;
+    return (const char *)wcon;
 }
 
 static char *
-str_getunamb (const char *tok, char **opwords)
+str_getunamb(const char *tok, char **opwords)
 {
-        return (char *)cli_getunamb (tok, (void **)opwords, id_sel);
+    return (char *)cli_getunamb(tok, (void **)opwords, id_sel);
 }
 
 int32_t
-cli_cmd_bricks_parse (const char **words, int wordcount, int brick_index,
-                      char **bricks, int *brick_count)
+cli_cmd_ta_brick_parse(const char **words, int wordcount, char **ta_brick)
 {
-        int                    ret = 0;
-        char             *tmp_list = NULL;
-        char    brick_list[120000] = {0,};
-        char                *space = " ";
-        char            *delimiter = NULL;
-        char            *host_name = NULL;
-        char        *free_list_ptr = NULL;
-        char               *tmpptr = NULL;
-        int                      j = 0;
-        int         brick_list_len = 0;
-        char             *tmp_host = NULL;
-
-        GF_ASSERT (words);
-        GF_ASSERT (wordcount);
-        GF_ASSERT (bricks);
-        GF_ASSERT (brick_index > 0);
-        GF_ASSERT (brick_index < wordcount);
-
-        strncpy (brick_list, space, strlen (space));
-        brick_list_len++;
-        while (brick_index < wordcount) {
-                if (validate_brick_name ((char *)words[brick_index])) {
-                        cli_err ("Wrong brick type: %s, use <HOSTNAME>:"
-                                 "<export-dir-abs-path>", words[brick_index]);
-                        ret = -1;
-                        goto out;
-                } else {
-                        delimiter = strrchr (words[brick_index], ':');
-                        ret = gf_canonicalize_path (delimiter + 1);
-                        if (ret)
-                                goto out;
-                }
-
-                if ((brick_list_len + strlen (words[brick_index]) + 1) > sizeof (brick_list)) {
-                        cli_err ("Total brick list is larger than a request. "
-                                 "Can take (brick_count %d)", *brick_count);
-                        ret = -1;
-                        goto out;
-                }
+    char *host_name = NULL;
+    char *tmp_host = NULL;
+    char *delimiter = NULL;
+    cli_brick_t *brick = NULL;
+    int ret = 0;
+
+    GF_ASSERT(words);
+    GF_ASSERT(wordcount);
+
+    if (validate_brick_name((char *)words[wordcount - 1])) {
+        cli_err(
+            "Wrong brick type: %s, use <HOSTNAME>:"
+            "<export-dir-abs-path>",
+            words[wordcount - 1]);
+        ret = -1;
+        goto out;
+    } else {
+        delimiter = strrchr(words[wordcount - 1], ':');
+        ret = gf_canonicalize_path(delimiter + 1);
+        if (ret)
+            goto out;
+    }
 
-                tmp_host = gf_strdup ((char *)words[brick_index]);
-                if (!tmp_host) {
-                        gf_log ("cli", GF_LOG_ERROR, "Out of memory");
-                        ret = -1;
-                        goto out;
-                }
-                get_host_name (tmp_host, &host_name);
-                if (!host_name) {
-                        ret = -1;
-                        gf_log("cli",GF_LOG_ERROR, "Unable to allocate "
-                               "memory");
-                        goto out;
-                }
+    tmp_host = gf_strdup((char *)words[wordcount - 1]);
+    if (!tmp_host) {
+        gf_log("cli", GF_LOG_ERROR, "Out of memory");
+        ret = -1;
+        goto out;
+    }
+    get_host_name(tmp_host, &host_name);
+    if (!host_name) {
+        ret = -1;
+        gf_log("cli", GF_LOG_ERROR,
+               "Unable to retrieve "
+               "hostname");
+        goto out;
+    }
+
+    if (!(strcmp(host_name, "localhost") && strcmp(host_name, "127.0.0.1") &&
+          strncmp(host_name, "0.", 2))) {
+        cli_err(
+            "Please provide a valid hostname/ip other "
+            "than localhost, 127.0.0.1 or loopback "
+            "address (0.0.0.0 to 0.255.255.255).");
+        ret = -1;
+        goto out;
+    }
+    if (!valid_internet_address(host_name, _gf_false, _gf_false)) {
+        cli_err(
+            "internet address '%s' does not conform to "
+            "standards",
+            host_name);
+    }
+
+    brick = GF_MALLOC(sizeof(cli_brick_t), gf_common_list_node);
+    if (brick == NULL) {
+        ret = -1;
+        gf_log("cli", GF_LOG_ERROR, "Out of memory");
+        goto out;
+    }
 
-                if (!(strcmp (host_name, "localhost") &&
-                      strcmp (host_name, "127.0.0.1") &&
-                      strncmp (host_name, "0.", 2))) {
-                        cli_err ("Please provide a valid hostname/ip other "
-                                 "than localhost, 127.0.0.1 or loopback "
-                                 "address (0.0.0.0 to 0.255.255.255).");
-                        ret = -1;
-                        GF_FREE (tmp_host);
-                        goto out;
-                }
-                if (!valid_internet_address (host_name, _gf_false)) {
-                        cli_err ("internet address '%s' does not conform to "
-                                 "standards", host_name);
-                }
-                GF_FREE (tmp_host);
-                tmp_list = gf_strdup (brick_list + 1);
-                if (free_list_ptr) {
-                        GF_FREE (free_list_ptr);
-                        free_list_ptr = NULL;
-                }
-                free_list_ptr = tmp_list;
-                j = 0;
-                while(j < *brick_count) {
-                        strtok_r (tmp_list, " ", &tmpptr);
-                        if (!(strcmp (tmp_list, words[brick_index]))) {
-                                ret = -1;
-                                cli_err ("Found duplicate"
-                                         " exports %s",words[brick_index]);
-                                goto out;
-                       }
-                       tmp_list = tmpptr;
-                       j++;
-                }
-                strcat (brick_list, words[brick_index]);
-                strcat (brick_list, " ");
-                brick_list_len += (strlen (words[brick_index]) + 1);
-                ++(*brick_count);
-                ++brick_index;
-        }
+    brick->name = words[wordcount - 1];
+    brick->len = strlen(words[wordcount - 1]);
+    *ta_brick = GF_MALLOC(brick->len + 3, gf_common_mt_char);
+    if (*ta_brick == NULL) {
+        ret = -1;
+        gf_log("cli", GF_LOG_ERROR, "Out of memory");
+        goto out;
+    }
 
-        *bricks = gf_strdup (brick_list);
-        if (!*bricks)
-                ret = -1;
+    strcat(*ta_brick, " ");
+    strcat(*ta_brick, brick->name);
+    strcat(*ta_brick, " ");
 out:
-        GF_FREE (free_list_ptr);
-        return ret;
+    if (tmp_host) {
+        GF_FREE(tmp_host);
+        tmp_host = NULL;
+    }
+    if (brick) {
+        GF_FREE(brick);
+        brick = NULL;
+    }
+
+    return ret;
 }
 
 int32_t
-cli_cmd_create_disperse_check (struct cli_state *state, int *disperse,
-                               int *redundancy, int *data, int count)
+cli_cmd_bricks_parse(const char **words, int wordcount, int brick_index,
+                     char **bricks, int *brick_count)
 {
-        int i = 0;
-        int tmp = 0;
-        gf_answer_t answer = GF_ANSWER_NO;
-        char question[128];
-
-        const char *question1 = "There isn't an optimal redundancy value "
-                                "for this configuration. Do you want to "
-                                "create the volume with redundancy 1 ?";
-
-        const char *question2 = "The optimal redundancy for this "
-                                "configuration is %d. Do you want to create "
-                                "the volume with this value ?";
-
-        const char *question3 = "This configuration is not optimal on most "
-                                "workloads. Do you want to use it ?";
-
-        const char *question4 = "Redundancy for this configuration is %d. "
-                                "Do you want to create "
-                                "the volume with this value ?";
-
-        if (*data > 0) {
-                if (*disperse > 0 && *redundancy > 0) {
-                        if (*disperse != (*data + *redundancy)) {
-                                cli_err ("Disperse count(%d) should be equal "
-                                         "to sum of disperse-data count(%d) and "
-                                         "redundancy count(%d)", *disperse,
-                                         *data, *redundancy);
-                                return -1;
-                        }
-                } else if (*redundancy > 0) {
-                        *disperse = *data + *redundancy;
-                } else if (*disperse > 0) {
-                        *redundancy = *disperse - *data;
-                } else {
-                        if ((count - *data) >= *data) {
-                                cli_err ("Please provide redundancy count "
-                                         "along with disperse-data count");
-                                return -1;
-                        } else {
-                                sprintf (question, question4, count - *data);
-                                answer = cli_cmd_get_confirmation (state,
-                                                                   question);
-                                if (answer == GF_ANSWER_NO)
-                                        return -1;
-                                *redundancy = count - *data;
-                                *disperse = count;
-                        }
-                }
+    int ret = 0;
+    char *delimiter = NULL;
+    char *host_name = NULL;
+    char *tmp_host = NULL;
+    char *bricks_str = NULL;
+    int len = 0;
+    int brick_list_len = 1; /* For initial space */
+    struct list_head brick_list = {
+        0,
+    };
+    cli_brick_t *brick = NULL;
+
+    GF_ASSERT(words);
+    GF_ASSERT(wordcount);
+    GF_ASSERT(bricks);
+    GF_ASSERT(brick_index > 0);
+    GF_ASSERT(brick_index < wordcount);
+
+    INIT_LIST_HEAD(&brick_list);
+
+    while (brick_index < wordcount) {
+        if (validate_brick_name((char *)words[brick_index])) {
+            cli_err(
+                "Wrong brick type: %s, use <HOSTNAME>:"
+                "<export-dir-abs-path>",
+                words[brick_index]);
+            ret = -1;
+            goto out;
+        } else {
+            delimiter = strrchr(words[brick_index], ':');
+            ret = gf_canonicalize_path(delimiter + 1);
+            if (ret)
+                goto out;
         }
 
-        if (*disperse <= 0) {
-                if (count < 3) {
-                        cli_err ("number of bricks must be greater "
-                                 "than 2");
+        tmp_host = gf_strdup((char *)words[brick_index]);
+        if (!tmp_host) {
+            gf_log("cli", GF_LOG_ERROR, "Out of memory");
+            ret = -1;
+            goto out;
+        }
+        get_host_name(tmp_host, &host_name);
+        if (!host_name) {
+            ret = -1;
+            gf_log("cli", GF_LOG_ERROR,
+                   "Unable to allocate "
+                   "memory");
+            GF_FREE(tmp_host);
+            goto out;
+        }
+
+        if (!(strcmp(host_name, "localhost") &&
+              strcmp(host_name, "127.0.0.1") && strncmp(host_name, "0.", 2))) {
+            cli_err(
+                "Please provide a valid hostname/ip other "
+                "than localhost, 127.0.0.1 or loopback "
+                "address (0.0.0.0 to 0.255.255.255).");
+            ret = -1;
+            GF_FREE(tmp_host);
+            goto out;
+        }
+        if (!valid_internet_address(host_name, _gf_false, _gf_false)) {
+            cli_err(
+                "internet address '%s' does not conform to "
+                "standards",
+                host_name);
+        }
+        GF_FREE(tmp_host);
+        list_for_each_entry(brick, &brick_list, list)
+        {
+            if (strcmp(brick->name, words[brick_index]) == 0) {
+                ret = -1;
+                cli_err("Found duplicate exports %s", words[brick_index]);
+                goto out;
+            }
+        }
 
-                        return -1;
-                }
-                *disperse = count;
+        brick = GF_MALLOC(sizeof(cli_brick_t), gf_common_list_node);
+        if (brick == NULL) {
+            ret = -1;
+            gf_log("cli", GF_LOG_ERROR, "Out of memory");
+            goto out;
         }
+        len = strlen(words[brick_index]);
+        brick->name = words[brick_index];
+        brick->len = len;
+        list_add_tail(&brick->list, &brick_list);
 
-        if (*redundancy == -1) {
-                tmp = *disperse - 1;
-                for (i = tmp / 2;
-                     (i > 0) && ((tmp & -tmp) != tmp);
-                     i--, tmp--);
+        brick_list_len += len + 1; /* Brick name + space */
+        ++(*brick_count);
+        ++brick_index;
+    }
 
-                if (i == 0) {
-                        answer = cli_cmd_get_confirmation(state, question1);
-                        if (answer == GF_ANSWER_NO)
-                                return -1;
+    /* If brick count is not valid exit here */
+    if (!*brick_count) {
+        cli_err("No bricks specified");
+        ret = -1;
+        goto out;
+    }
 
-                        *redundancy = 1;
-                }
-                else
-                {
-                        *redundancy = *disperse - tmp;
-                        if (*redundancy > 1) {
-                                sprintf(question, question2, *redundancy);
-                                answer = cli_cmd_get_confirmation(state,
-                                                                  question);
-                                if (answer == GF_ANSWER_NO)
-                                        return -1;
-                        }
-                }
+    brick_list_len++; /* For terminating null char */
+
+    bricks_str = GF_MALLOC(brick_list_len, gf_common_mt_char);
+    if (bricks_str == NULL) {
+        ret = -1;
+        gf_log("cli", GF_LOG_ERROR, "Out of memory");
+        goto out;
+    }
+    *bricks = bricks_str;
+    *bricks_str = ' ';
+    bricks_str++;
+    while (!list_empty(&brick_list)) {
+        brick = list_first_entry(&brick_list, cli_brick_t, list);
+        list_del_init(&brick->list);
+        memcpy(bricks_str, brick->name, brick->len);
+        bricks_str[brick->len] = ' ';
+        bricks_str += brick->len + 1;
+        GF_FREE(brick);
+    }
+    *bricks_str = 0;
+
+out:
+    while (!list_empty(&brick_list)) {
+        brick = list_first_entry(&brick_list, cli_brick_t, list);
+        list_del_init(&brick->list);
+        GF_FREE(brick);
+    }
 
-                tmp = 0;
+    return ret;
+}
+
+int32_t
+cli_cmd_create_disperse_check(struct cli_state *state, int *disperse,
+                              int *redundancy, int *data, int count)
+{
+    int i = 0;
+    int tmp = 0;
+    gf_answer_t answer = GF_ANSWER_NO;
+    char question[128];
+
+    const char *question1 =
+        "There isn't an optimal redundancy value "
+        "for this configuration. Do you want to "
+        "create the volume with redundancy 1 ?";
+
+    const char *question2 =
+        "The optimal redundancy for this "
+        "configuration is %d. Do you want to create "
+        "the volume with this value ?";
+
+    const char *question3 =
+        "This configuration is not optimal on most "
+        "workloads. Do you want to use it ?";
+
+    const char *question4 =
+        "Redundancy for this configuration is %d. "
+        "Do you want to create "
+        "the volume with this value ?";
+
+    if (*data > 0) {
+        if (*disperse > 0 && *redundancy > 0) {
+            if (*disperse != (*data + *redundancy)) {
+                cli_err(
+                    "Disperse count(%d) should be equal "
+                    "to sum of disperse-data count(%d) and "
+                    "redundancy count(%d)",
+                    *disperse, *data, *redundancy);
+                return -1;
+            }
+        } else if (*redundancy > 0) {
+            *disperse = *data + *redundancy;
+        } else if (*disperse > 0) {
+            *redundancy = *disperse - *data;
         } else {
-                tmp = *disperse - *redundancy;
+            if ((count - *data) >= *data) {
+                cli_err(
+                    "Please provide redundancy count "
+                    "along with disperse-data count");
+                return -1;
+            } else {
+                sprintf(question, question4, count - *data);
+                answer = cli_cmd_get_confirmation(state, question);
+                if (answer == GF_ANSWER_NO)
+                    return -1;
+                *redundancy = count - *data;
+                *disperse = count;
+            }
         }
+    }
 
-        if (*redundancy > (*disperse - 1) / 2) {
-                cli_err ("redundancy must be less than %d for a "
-                         "disperse %d volume",
-                         (*disperse + 1) / 2, *disperse);
+    if (*disperse <= 0) {
+        if (count < 3) {
+            cli_err(
+                "number of bricks must be greater "
+                "than 2");
 
-                return -1;
+            return -1;
         }
+        *disperse = count;
+    }
+
+    if (*redundancy == -1) {
+        tmp = *disperse - 1;
+        for (i = tmp / 2; (i > 0) && ((tmp & -tmp) != tmp); i--, tmp--)
+            ;
+
+        if (i == 0) {
+            answer = cli_cmd_get_confirmation(state, question1);
+            if (answer == GF_ANSWER_NO)
+                return -1;
 
-        if ((tmp & -tmp) != tmp) {
-                answer = cli_cmd_get_confirmation(state, question3);
+            *redundancy = 1;
+        } else {
+            *redundancy = *disperse - tmp;
+            if (*redundancy > 1) {
+                sprintf(question, question2, *redundancy);
+                answer = cli_cmd_get_confirmation(state, question);
                 if (answer == GF_ANSWER_NO)
-                        return -1;
+                    return -1;
+            }
         }
 
-        return 0;
+        tmp = 0;
+    } else {
+        tmp = *disperse - *redundancy;
+    }
+
+    if ((*redundancy < 1) || (*redundancy > (*disperse - 1) / 2)) {
+        cli_err(
+            "redundancy must be greater than or equal to 1 and "
+            "less than %d for a disperse %d volume",
+            (*disperse + 1) / 2, *disperse);
+
+        return -1;
+    }
+
+    if ((tmp & -tmp) != tmp) {
+        answer = cli_cmd_get_confirmation(state, question3);
+        if (answer == GF_ANSWER_NO)
+            return -1;
+    }
+
+    return 0;
 }
 
 static int32_t
-cli_validate_disperse_volume (char *word, gf1_cluster_type type,
-                              const char **words, int32_t wordcount,
-                              int32_t index, int32_t *disperse_count,
-                              int32_t *redundancy_count,
-                              int32_t *data_count)
+cli_validate_disperse_volume(char *word, gf1_cluster_type type,
+                             const char **words, int32_t wordcount,
+                             int32_t index, int32_t *disperse_count,
+                             int32_t *redundancy_count, int32_t *data_count)
 {
-        int     ret = -1;
+    int ret = -1;
 
-        switch (type) {
+    switch (type) {
         case GF_CLUSTER_TYPE_NONE:
         case GF_CLUSTER_TYPE_DISPERSE:
-                if (strcmp (word, "disperse") == 0) {
-                        if (*disperse_count >= 0) {
-                                cli_err ("disperse option given twice");
-                                goto out;
-                        }
-                        if (wordcount < (index+2)) {
-                                goto out;
-                        }
-                        ret = gf_string2int (words[index + 1], disperse_count);
-                        if (ret == -1 && errno == EINVAL) {
-                                *disperse_count = 0;
-                                ret = 1;
-                        } else if (ret == -1) {
-                                goto out;
-                        } else {
-                                if (*disperse_count < 3) {
-                                        cli_err ("disperse count must "
-                                                 "be greater than 2");
-                                        goto out;
-                                }
-                                ret = 2;
-                        }
-                } else if (strcmp (word, "disperse-data") == 0) {
-                        if (*data_count >= 0) {
-                                cli_err ("disperse-data option given twice");
-                                goto out;
-                        }
-                        if (wordcount < (index+2)) {
-                                goto out;
-                        }
-                        ret = gf_string2int (words[index+1], data_count);
-                        if (ret == -1 || *data_count < 2) {
-                                cli_err ("disperse-data must be greater than 1");
-                                goto out;
-                        }
-                        ret = 2;
-                } else if (strcmp (word, "redundancy") == 0) {
-                        if (*redundancy_count >= 0) {
-                                cli_err ("redundancy option given twice");
-                                goto out;
-                        }
-                        if (wordcount < (index+2)) {
-                                goto out;
-                        }
-                        ret = gf_string2int (words[index+1], redundancy_count);
-                        if (ret == -1 || *redundancy_count < 1) {
-                                cli_err ("redundancy must be greater than 0");
-                                goto out;
-                        }
-                        ret = 2;
-                }
-                break;
-        case GF_CLUSTER_TYPE_STRIPE_REPLICATE:
-                cli_err ("striped-replicated-dispersed volume "
-                         "is not supported");
-                goto out;
-        case GF_CLUSTER_TYPE_TIER:
-                cli_err ("tier-dispersed volume is not "
-                         "supported");
-                goto out;
-        case GF_CLUSTER_TYPE_STRIPE:
-                cli_err ("striped-dispersed volume is not "
-                         "supported");
-                goto out;
+            if (strcmp(word, "disperse") == 0) {
+                if (*disperse_count >= 0) {
+                    cli_err("disperse option given twice");
+                    goto out;
+                }
+                if (wordcount < (index + 2)) {
+                    goto out;
+                }
+                ret = gf_string2int(words[index + 1], disperse_count);
+                if (ret == -1 && errno == EINVAL) {
+                    *disperse_count = 0;
+                    ret = 1;
+                } else if (ret == -1) {
+                    goto out;
+                } else {
+                    if (*disperse_count < 3) {
+                        cli_err(
+                            "disperse count must "
+                            "be greater than 2");
+                        goto out;
+                    }
+                    ret = 2;
+                }
+            } else if (strcmp(word, "disperse-data") == 0) {
+                if (*data_count >= 0) {
+                    cli_err("disperse-data option given twice");
+                    goto out;
+                }
+                if (wordcount < (index + 2)) {
+                    goto out;
+                }
+                ret = gf_string2int(words[index + 1], data_count);
+                if (ret == -1 || *data_count < 2) {
+                    cli_err("disperse-data must be greater than 1");
+                    goto out;
+                }
+                ret = 2;
+            } else if (strcmp(word, "redundancy") == 0) {
+                if (*redundancy_count >= 0) {
+                    cli_err("redundancy option given twice");
+                    goto out;
+                }
+                if (wordcount < (index + 2)) {
+                    goto out;
+                }
+                ret = gf_string2int(words[index + 1], redundancy_count);
+                if (ret == -1 || *redundancy_count < 1) {
+                    cli_err("redundancy must be greater than 0");
+                    goto out;
+                }
+                ret = 2;
+            }
+            break;
         case GF_CLUSTER_TYPE_REPLICATE:
-                cli_err ("replicated-dispersed volume is not "
-                         "supported");
-                goto out;
+            cli_err(
+                "replicated-dispersed volume is not "
+                "supported");
+            goto out;
         default:
-                cli_err ("Invalid type given");
-                break;
-        }
+            cli_err("Invalid type given");
+            break;
+    }
 out:
-        return ret;
+    return ret;
 }
 
 int32_t
-cli_validate_volname (const char *volname)
+cli_validate_volname(const char *volname)
 {
-        int32_t            ret                       = -1;
-        int32_t            i                         = -1;
-        static const char * const invalid_volnames[] = {
-                                      "volume", "type", "subvolumes", "option",
-                                      "end-volume", "all", "volume_not_in_ring",
-                                      "description", "force",
-                                      "snap-max-hard-limit",
-                                      "snap-max-soft-limit", "auto-delete",
-                                      "activate-on-create", NULL};
-
-        if (volname[0] == '-')
-                goto out;
-
-        for (i = 0; invalid_volnames[i]; i++) {
-                if (!strcmp (volname, invalid_volnames[i])) {
-                        cli_err ("\"%s\" cannot be the name of a volume.",
-                                 volname);
-                        goto out;
-                }
-        }
+    int32_t ret = -1;
+    int32_t i = -1;
+    int volname_len;
+    static const char *const invalid_volnames[] = {"volume",
+                                                   "type",
+                                                   "subvolumes",
+                                                   "option",
+                                                   "end-volume",
+                                                   "all",
+                                                   "volume_not_in_ring",
+                                                   "description",
+                                                   "force",
+                                                   "snap-max-hard-limit",
+                                                   "snap-max-soft-limit",
+                                                   "auto-delete",
+                                                   "activate-on-create",
+                                                   NULL};
 
-        if (strchr (volname, '/'))
-                goto out;
+    if (volname[0] == '-')
+        goto out;
 
-        if (strlen (volname) > GD_VOLUME_NAME_MAX) {
-                cli_err("Volume name exceeds %d characters.",
-                        GD_VOLUME_NAME_MAX);
-                goto out;
+    for (i = 0; invalid_volnames[i]; i++) {
+        if (!strcmp(volname, invalid_volnames[i])) {
+            cli_err("\"%s\" cannot be the name of a volume.", volname);
+            goto out;
         }
+    }
 
-        for (i = 0; i < strlen (volname); i++) {
-                if (!isalnum (volname[i]) && (volname[i] != '_') &&
-                   (volname[i] != '-')) {
-                        cli_err ("Volume name should not contain \"%c\""
-                                 " character.\nVolume names can only"
-                                 "contain alphanumeric, '-' and '_' "
-                                 "characters.", volname[i]);
-                        goto out;
-                }
-        }
+    if (strchr(volname, '/'))
+        goto out;
 
-        ret = 0;
+    volname_len = strlen(volname);
+    if (volname_len > GD_VOLUME_NAME_MAX) {
+        cli_err("Volume name exceeds %d characters.", GD_VOLUME_NAME_MAX);
+        goto out;
+    }
+
+    for (i = 0; i < volname_len; i++) {
+        if (!isalnum(volname[i]) && (volname[i] != '_') &&
+            (volname[i] != '-')) {
+            cli_err(
+                "Volume name should not contain \"%c\""
+                " character.\nVolume names can only"
+                "contain alphanumeric, '-' and '_' "
+                "characters.",
+                volname[i]);
+            goto out;
+        }
+    }
+
+    ret = 0;
 out:
-        return ret;
+    return ret;
 }
 
 int32_t
-cli_cmd_volume_create_parse (struct cli_state *state, const char **words,
-                             int wordcount, dict_t **options)
+cli_cmd_volume_create_parse(struct cli_state *state, const char **words,
+                            int wordcount, dict_t **options, char **brick_list)
 {
-        dict_t  *dict = NULL;
-        char    *volname = NULL;
-        int     ret = -1;
-        gf1_cluster_type type = GF_CLUSTER_TYPE_NONE;
-        int     count = 1;
-        int     sub_count = 1;
-        int     brick_index = 0;
-        char    *trans_type = NULL;
-        int32_t index = 0;
-        char    *bricks = NULL;
-        int32_t brick_count = 0;
-        char    *opwords[] = { "replica", "stripe", "transport", "disperse",
-                               "redundancy", "disperse-data", "arbiter", NULL };
-
-        char    *w = NULL;
-        char    *ptr = NULL;
-        int      op_count = 0;
-        int32_t  replica_count = 1;
-        int32_t  arbiter_count = 0;
-        int32_t  stripe_count = 1;
-        int32_t  disperse_count = -1;
-        int32_t  redundancy_count = -1;
-        int32_t  disperse_data_count = -1;
-        gf_boolean_t is_force = _gf_false;
-        int wc = wordcount;
-
-        GF_ASSERT (words);
-        GF_ASSERT (options);
-
-        dict = dict_new ();
-
-        if (!dict)
-                goto out;
+    dict_t *dict = NULL;
+    char *volname = NULL;
+    int ret = -1;
+    gf1_cluster_type type = GF_CLUSTER_TYPE_NONE;
+    int sub_count = 1;
+    int brick_index = 0;
+    char *trans_type = NULL;
+    int32_t index = 0;
+    char *bricks = NULL;
+    char *ta_brick = NULL;
+    int32_t brick_count = 0;
+    static char *opwords[] = {"replica",  "stripe",       "transport",
+                              "disperse", "redundancy",   "disperse-data",
+                              "arbiter",  "thin-arbiter", NULL};
+
+    char *w = NULL;
+    int op_count = 0;
+    int32_t replica_count = 1;
+    int32_t arbiter_count = 0;
+    int32_t thin_arbiter_count = 0;
+    int32_t stripe_count = 1;
+    int32_t disperse_count = -1;
+    int32_t redundancy_count = -1;
+    int32_t disperse_data_count = -1;
+    gf_boolean_t is_force = _gf_false;
+    int wc = wordcount;
+    gf_answer_t answer = GF_ANSWER_NO;
+    const char *question = NULL;
+
+    GF_ASSERT(words);
+    GF_ASSERT(options);
+
+    dict = dict_new();
+
+    if (!dict)
+        goto out;
 
-        if (wordcount < 3)
-                goto out;
+    if (wordcount < 3)
+        goto out;
 
-        volname = (char *)words[2];
+    volname = (char *)words[2];
 
-        GF_ASSERT (volname);
+    GF_ASSERT(volname);
 
-        /* Validate the volume name here itself */
-        if (cli_validate_volname (volname) < 0)
-                goto out;
+    /* Validate the volume name here itself */
+    if (cli_validate_volname(volname) < 0)
+        goto out;
 
-        if (wordcount < 4) {
-                ret = -1;
-                goto out;
-        }
+    if (wordcount < 4) {
+        ret = -1;
+        goto out;
+    }
 
-        type = GF_CLUSTER_TYPE_NONE;
-        index = 3;
+    type = GF_CLUSTER_TYPE_NONE;
+    index = 3;
 
-        while (op_count < 3) {
+    while (op_count < 3) {
+        ret = -1;
+        w = str_getunamb(words[index], opwords);
+        if (!w) {
+            break;
+        } else if ((strcmp(w, "replica")) == 0) {
+            switch (type) {
+                case GF_CLUSTER_TYPE_STRIPE_REPLICATE:
+                case GF_CLUSTER_TYPE_REPLICATE:
+                    cli_err("replica option given twice");
+                    goto out;
+                case GF_CLUSTER_TYPE_NONE:
+                    type = GF_CLUSTER_TYPE_REPLICATE;
+                    break;
+                case GF_CLUSTER_TYPE_STRIPE:
+                    cli_err("stripe option not supported");
+                    goto out;
+                case GF_CLUSTER_TYPE_DISPERSE:
+                    cli_err(
+                        "replicated-dispersed volume is not "
+                        "supported");
+                    goto out;
+                default:
+                    cli_err("Invalid type given");
+                    goto out;
+            }
+
+            if (wordcount < (index + 2)) {
                 ret = -1;
-                w = str_getunamb (words[index], opwords);
-                if (!w) {
-                        break;
-                } else if ((strcmp (w, "replica")) == 0) {
-                        switch (type) {
-                        case GF_CLUSTER_TYPE_STRIPE_REPLICATE:
-                        case GF_CLUSTER_TYPE_REPLICATE:
-                                cli_err ("replica option given twice");
-                                goto out;
-                        case GF_CLUSTER_TYPE_NONE:
-                                type = GF_CLUSTER_TYPE_REPLICATE;
-                                break;
-                        case GF_CLUSTER_TYPE_STRIPE:
-                                type = GF_CLUSTER_TYPE_STRIPE_REPLICATE;
-                                break;
-                        case GF_CLUSTER_TYPE_TIER:
-                                cli_err ("replicated-tiered volume is not "
-                                         "supported");
-                                goto out;
-                                break;
-                        case GF_CLUSTER_TYPE_DISPERSE:
-                                cli_err ("replicated-dispersed volume is not "
-                                         "supported");
-                                goto out;
-                        default:
-                                cli_err ("Invalid type given");
-                                goto out;
-                        }
-
-                        if (wordcount < (index+2)) {
-                                ret = -1;
-                                goto out;
-                        }
-                        replica_count = strtol (words[index+1], NULL, 0);
-                        if (replica_count < 2) {
-                                cli_err ("replica count should be greater"
-                                         " than 1");
-                                ret = -1;
-                                goto out;
-                        }
-                        ret = dict_set_int32 (dict, "replica-count", replica_count);
-                        if (ret)
-                                goto out;
-
-                        index += 2;
-                        if (words[index]) {
-                                if (!strcmp (words[index], "arbiter")) {
-                                        ret = gf_string2int (words[index+1],
-                                                             &arbiter_count);
-                                        if (ret == -1 || arbiter_count != 1 ||
-                                            replica_count != 3) {
-                                                cli_err ("For arbiter "
-                                                         "configuration, "
-                                                         "replica count must be"
-                                                         " 3 and arbiter count "
-                                                         "must be 1. The 3rd "
-                                                         "brick of the replica "
-                                                         "will be the arbiter");
-                                                ret = -1;
-                                                goto out;
-                                        }
-                                        ret = dict_set_int32 (dict, "arbiter-count",
-                                                              arbiter_count);
-                                        if (ret)
-                                                goto out;
-                                        index += 2;
-                                }
-                        }
-
-                } else if ((strcmp (w, "stripe")) == 0) {
-                        switch (type) {
-                        case GF_CLUSTER_TYPE_STRIPE_REPLICATE:
-                        case GF_CLUSTER_TYPE_STRIPE:
-                                cli_err ("stripe option given twice");
-                                goto out;
-                        case GF_CLUSTER_TYPE_NONE:
-                                type = GF_CLUSTER_TYPE_STRIPE;
-                                break;
-                        case GF_CLUSTER_TYPE_REPLICATE:
-                                type = GF_CLUSTER_TYPE_STRIPE_REPLICATE;
-                                break;
-                        case GF_CLUSTER_TYPE_DISPERSE:
-                                cli_err ("striped-dispersed volume is not "
-                                         "supported");
-                                goto out;
-                        case GF_CLUSTER_TYPE_TIER:
-                                cli_err ("striped-tier volume is not "
-                                         "supported");
-                                goto out;
-                        default:
-                                cli_err ("Invalid type given");
-                                goto out;
-                        }
-                        if (wordcount < (index + 2)) {
-                                ret = -1;
-                                goto out;
-                        }
-                        stripe_count = strtol (words[index+1], NULL, 0);
-                        if (stripe_count < 2) {
-                                cli_err ("stripe count should be greater"
-                                         " than 1");
-                                ret = -1;
-                                goto out;
-                        }
-                        ret = dict_set_int32 (dict, "stripe-count", stripe_count);
-                        if (ret)
-                                goto out;
-
-                        index += 2;
+                goto out;
+            }
 
-                } else if ((strcmp (w, "transport")) == 0) {
-                        if (trans_type) {
-                                cli_err ("'transport' option given more"
-                                         " than one time");
-                                goto out;
-                        }
-                        if ((strcasecmp (words[index+1], "tcp") == 0)) {
-                                trans_type = gf_strdup ("tcp");
-                        } else if ((strcasecmp (words[index+1], "rdma") == 0)) {
-                                trans_type = gf_strdup ("rdma");
-                        } else if ((strcasecmp (words[index+1], "tcp,rdma") == 0) ||
-                                   (strcasecmp (words[index+1], "rdma,tcp") == 0)) {
-                                trans_type = gf_strdup ("tcp,rdma");
-                        } else {
-                                gf_log ("", GF_LOG_ERROR, "incorrect transport"
-                                        " protocol specified");
-                                ret = -1;
-                                goto out;
-                        }
-                        index += 2;
-
-                } else if ((strcmp (w, "disperse") == 0) ||
-                           (strcmp (w, "redundancy") == 0) ||
-                           (strcmp (w, "disperse-data") == 0)) {
-                        ret = cli_validate_disperse_volume (w, type, words,
-                                      wordcount, index, &disperse_count,
-                                      &redundancy_count, &disperse_data_count);
-                        if (ret < 0)
-                                goto out;
-                        index += ret;
-                        type = GF_CLUSTER_TYPE_DISPERSE;
-                } else {
-                        GF_ASSERT (!"opword mismatch");
+            replica_count = strtol(words[index + 1], NULL, 0);
+            if (replica_count < 2) {
+                cli_err(
+                    "replica count should be greater"
+                    " than 1");
+                ret = -1;
+                goto out;
+            }
+
+            index += 2;
+            if (words[index]) {
+                if (!strcmp(words[index], "arbiter")) {
+                    ret = gf_string2int(words[index + 1], &arbiter_count);
+                    if ((ret == -1) || (arbiter_count != 1)) {
+                        cli_err(
+                            "For arbiter "
+                            "configuration, "
+                            "replica count must be"
+                            " 2 and arbiter count "
+                            "must be 1. The 3rd "
+                            "brick of the replica "
+                            "will be the arbiter");
                         ret = -1;
                         goto out;
-                }
-                op_count++;
-        }
-
-        if (!trans_type)
-                trans_type = gf_strdup ("tcp");
-
-        /* reset the count value now */
-        count = 1;
-
-        if (index >= wordcount) {
+                    }
+                    ret = dict_set_int32(dict, "arbiter-count", arbiter_count);
+                    if (ret)
+                        goto out;
+                    index += 2;
+                } else if (!strcmp(words[index], "thin-arbiter")) {
+                    ret = gf_string2int(words[index + 1], &thin_arbiter_count);
+                    if ((ret == -1) || (thin_arbiter_count != 1) ||
+                        (replica_count != 2)) {
+                        cli_err(
+                            "For thin-arbiter "
+                            "configuration, "
+                            "replica count must be"
+                            " 2 and thin-arbiter count "
+                            "must be 1. The 3rd "
+                            "brick of the replica "
+                            "will be the thin-arbiter brick");
+                        ret = -1;
+                        goto out;
+                    }
+                    ret = dict_set_int32(dict, "thin-arbiter-count",
+                                         thin_arbiter_count);
+                    if (ret)
+                        goto out;
+                    index += 2;
+                }
+            }
+
+            /* Do this to keep glusterd happy with sending
+               "replica 3 arbiter 1" options to server */
+            if ((arbiter_count == 1) && (replica_count == 2))
+                replica_count += arbiter_count;
+
+            if (replica_count == 2 && thin_arbiter_count == 0) {
+                if (strcmp(words[wordcount - 1], "force")) {
+                    question =
+                        "Replica 2 volumes are prone"
+                        " to split-brain. Use "
+                        "Arbiter or Replica 3 to "
+                        "avoid this. See: "
+                        "http://docs.gluster.org/en/latest/"
+                        "Administrator%20Guide/"
+                        "Split%20brain%20and%20ways%20to%20deal%20with%20it/."
+                        "\nDo you still want to "
+                        "continue?\n";
+                    answer = cli_cmd_get_confirmation(state, question);
+                    if (GF_ANSWER_NO == answer) {
+                        gf_log("cli", GF_LOG_ERROR,
+                               "Volume create "
+                               "cancelled, exiting");
+                        ret = -1;
+                        goto out;
+                    }
+                }
+            }
+            ret = dict_set_int32(dict, "replica-count", replica_count);
+            if (ret)
+                goto out;
+
+        } else if ((strcmp(w, "stripe")) == 0) {
+            cli_err("stripe option not supported");
+            goto out;
+        } else if ((strcmp(w, "transport")) == 0) {
+            if (trans_type) {
+                cli_err(
+                    "'transport' option given more"
+                    " than one time");
+                goto out;
+            }
+            if ((strcasecmp(words[index + 1], "tcp") == 0)) {
+                trans_type = gf_strdup("tcp");
+            } else if ((strcasecmp(words[index + 1], "rdma") == 0)) {
+                trans_type = gf_strdup("rdma");
+            } else if ((strcasecmp(words[index + 1], "tcp,rdma") == 0) ||
+                       (strcasecmp(words[index + 1], "rdma,tcp") == 0)) {
+                trans_type = gf_strdup("tcp,rdma");
+            } else {
+                gf_log("", GF_LOG_ERROR,
+                       "incorrect transport"
+                       " protocol specified");
                 ret = -1;
                 goto out;
+            }
+            index += 2;
+
+        } else if ((strcmp(w, "disperse") == 0) ||
+                   (strcmp(w, "redundancy") == 0) ||
+                   (strcmp(w, "disperse-data") == 0)) {
+            ret = cli_validate_disperse_volume(
+                w, type, words, wordcount, index, &disperse_count,
+                &redundancy_count, &disperse_data_count);
+            if (ret < 0)
+                goto out;
+            index += ret;
+            type = GF_CLUSTER_TYPE_DISPERSE;
+        } else if ((strcmp(w, "arbiter") == 0)) {
+            cli_err(
+                "arbiter option must be preceded by replica "
+                "option.");
+            ret = -1;
+            goto out;
+        } else if ((strcmp(w, "thin-arbiter") == 0)) {
+            cli_err(
+                "thin-arbiter option must be preceded by replica "
+                "option.");
+            ret = -1;
+            goto out;
+        } else {
+            GF_ASSERT(!"opword mismatch");
+            ret = -1;
+            goto out;
         }
+        op_count++;
+    }
 
-        brick_index = index;
+    if (!trans_type)
+        trans_type = gf_strdup("tcp");
 
-        if (strcmp (words[wordcount - 1], "force") == 0) {
-                is_force = _gf_true;
-                wc = wordcount - 1;
-        }
+    if (index >= wordcount) {
+        ret = -1;
+        goto out;
+    }
 
-        ret = cli_cmd_bricks_parse (words, wc, brick_index, &bricks,
-                                    &brick_count);
-        if (ret)
-                goto out;
+    brick_index = index;
 
-        /* If brick-count is not valid when replica or stripe is
-           given, exit here */
-        if (!brick_count) {
-                cli_err ("No bricks specified");
-                ret = -1;
-                goto out;
-        }
+    if (strcmp(words[wordcount - 1], "force") == 0) {
+        is_force = _gf_true;
+        wc = wordcount - 1;
+    }
 
-        if (type == GF_CLUSTER_TYPE_DISPERSE) {
-                ret = cli_cmd_create_disperse_check (state, &disperse_count,
-                                                     &redundancy_count,
-                                                     &disperse_data_count,
-                                                     brick_count);
-                if (!ret)
-                        ret = dict_set_int32 (dict, "disperse-count",
-                                              disperse_count);
-                if (!ret)
-                        ret = dict_set_int32 (dict, "redundancy-count",
-                                              redundancy_count);
-                if (ret)
-                        goto out;
+    // Exclude the thin-arbiter-brick i.e. last brick in the bricks list
+    if (thin_arbiter_count == 1) {
+        ret = cli_cmd_bricks_parse(words, wc - 1, brick_index, &bricks,
+                                   &brick_count);
+        if (ret)
+            goto out;
 
-                sub_count = disperse_count;
-        } else
-                sub_count = stripe_count * replica_count;
-
-        if (brick_count % sub_count) {
-                if (type == GF_CLUSTER_TYPE_STRIPE)
-                        cli_err ("number of bricks is not a multiple of "
-                                 "stripe count");
-                else if (type == GF_CLUSTER_TYPE_REPLICATE)
-                        cli_err ("number of bricks is not a multiple of "
-                                 "replica count");
-                else if (type == GF_CLUSTER_TYPE_DISPERSE)
-                        cli_err ("number of bricks is not a multiple of "
-                                 "disperse count");
-                else
-                        cli_err ("number of bricks given doesn't match "
-                                 "required count");
+        ret = cli_cmd_ta_brick_parse(words, wc, &ta_brick);
 
-                ret = -1;
-                goto out;
-        }
+    } else {
+        ret = cli_cmd_bricks_parse(words, wc, brick_index, &bricks,
+                                   &brick_count);
+    }
 
-        /* Everything is parsed fine. start setting info in dict */
-        ret = dict_set_str (dict, "volname", volname);
-        if (ret)
-                goto out;
+    if (ret)
+        goto out;
 
-        ret = dict_set_int32 (dict, "type", type);
+    if (type == GF_CLUSTER_TYPE_DISPERSE) {
+        ret = cli_cmd_create_disperse_check(state, &disperse_count,
+                                            &redundancy_count,
+                                            &disperse_data_count, brick_count);
+        if (!ret)
+            ret = dict_set_int32(dict, "disperse-count", disperse_count);
+        if (!ret)
+            ret = dict_set_int32(dict, "redundancy-count", redundancy_count);
         if (ret)
-                goto out;
+            goto out;
+
+        sub_count = disperse_count;
+    } else
+        sub_count = stripe_count * replica_count;
+
+    if (brick_count % sub_count) {
+        if (type == GF_CLUSTER_TYPE_STRIPE)
+            cli_err(
+                "number of bricks is not a multiple of "
+                "stripe count");
+        else if (type == GF_CLUSTER_TYPE_REPLICATE)
+            cli_err(
+                "number of bricks is not a multiple of "
+                "replica count");
+        else if (type == GF_CLUSTER_TYPE_DISPERSE)
+            cli_err(
+                "number of bricks is not a multiple of "
+                "disperse count");
+        else
+            cli_err(
+                "number of bricks given doesn't match "
+                "required count");
 
-        ret = dict_set_dynstr (dict, "transport", trans_type);
-        if (ret)
-                goto out;
-        trans_type = NULL;
+        ret = -1;
+        goto out;
+    }
 
-        ret = dict_set_dynstr (dict, "bricks", bricks);
-        if (ret)
-                goto out;
+    /* Everything is parsed fine. start setting info in dict */
+    ret = dict_set_str(dict, "volname", volname);
+    if (ret)
+        goto out;
 
-        ret = dict_set_int32 (dict, "count", brick_count);
-        if (ret)
-                goto out;
+    ret = dict_set_int32(dict, "type", type);
+    if (ret)
+        goto out;
 
-        ret = dict_set_int32 (dict, "force", is_force);
+    ret = dict_set_dynstr(dict, "transport", trans_type);
+    if (ret)
+        goto out;
+    trans_type = NULL;
+
+    ret = dict_set_dynstr(dict, "bricks", bricks);
+    if (ret)
+        goto out;
+
+    if (thin_arbiter_count == 1) {
+        ret = dict_set_dynstr(dict, "ta-brick", ta_brick);
         if (ret)
-                goto out;
+            goto out;
+    }
 
-        *options = dict;
+    ret = dict_set_int32(dict, "count", brick_count);
+    if (ret)
+        goto out;
+
+    ret = dict_set_int32(dict, "force", is_force);
+    if (ret)
+        goto out;
 
+    *options = dict;
+    *brick_list = bricks;
 out:
-        if (ret) {
-                gf_log ("cli", GF_LOG_ERROR, "Unable to parse create volume CLI");
-                if (dict)
-                        dict_destroy (dict);
-        }
+    if (ret) {
+        GF_FREE(bricks);
+        GF_FREE(ta_brick);
+        gf_log("cli", GF_LOG_ERROR, "Unable to parse create volume CLI");
+        if (dict)
+            dict_unref(dict);
+    }
 
-        GF_FREE (trans_type);
+    GF_FREE(trans_type);
 
-        return ret;
+    return ret;
 }
 
 int32_t
-cli_cmd_volume_reset_parse (const char **words, int wordcount, dict_t **options)
+cli_cmd_volume_reset_parse(const char **words, int wordcount, dict_t **options)
 {
-        dict_t  *dict = NULL;
-        char    *volname = NULL;
-        int     ret = -1;
+    dict_t *dict = NULL;
+    char *volname = NULL;
+    int ret = -1;
 
-        GF_ASSERT (words);
-        GF_ASSERT (options);
+    GF_ASSERT(words);
+    GF_ASSERT(options);
 
-        dict = dict_new ();
+    dict = dict_new();
 
-        if (!dict)
-                goto out;
+    if (!dict)
+        goto out;
 
-        if (wordcount < 3)
-                goto out;
+    if (wordcount < 3)
+        goto out;
 
-        if (wordcount > 5)
-                goto out;
+    if (wordcount > 5)
+        goto out;
 
-        volname = (char *)words[2];
+    volname = (char *)words[2];
 
-        if (!volname) {
-                ret = -1;
-                goto out;
-        }
+    if (!volname) {
+        ret = -1;
+        goto out;
+    }
 
-        ret = dict_set_str (dict, "volname", volname);
-        if (ret)
-                goto out;
+    ret = dict_set_str(dict, "volname", volname);
+    if (ret)
+        goto out;
 
-        if (wordcount == 3) {
-                ret = dict_set_str (dict, "key", "all");
-                if (ret)
-                        goto out;
-        }
+    if (wordcount == 3) {
+        ret = dict_set_str(dict, "key", "all");
+        if (ret)
+            goto out;
+    }
 
-        if (wordcount >= 4) {
-                if (!strcmp ("force", (char*)words[3])) {
-                        ret = dict_set_int32 (dict, "force", 1);
-                        if (ret)
-                                goto out;
-                        ret = dict_set_str (dict, "key", "all");
-                        if (ret)
-                                goto out;
-                } else {
-                        ret = dict_set_str (dict, "key", (char *)words[3]);
-                        if (ret)
-                                goto out;
-                }
+    if (wordcount >= 4) {
+        if (!strcmp("force", (char *)words[3])) {
+            ret = dict_set_int32(dict, "force", 1);
+            if (ret)
+                goto out;
+            ret = dict_set_str(dict, "key", "all");
+            if (ret)
+                goto out;
+        } else {
+            ret = dict_set_str(dict, "key", (char *)words[3]);
+            if (ret)
+                goto out;
         }
+    }
 
-        if (wordcount == 5) {
-                if (strcmp ("force", (char*)words[4])) {
-                        ret = -1;
-                        goto out;
-                } else {
-                        ret = dict_set_int32 (dict, "force", 1);
-                        if (ret)
-                                goto out;
-                }
+    if (wordcount == 5) {
+        if (strcmp("force", (char *)words[4])) {
+            ret = -1;
+            goto out;
+        } else {
+            ret = dict_set_int32(dict, "force", 1);
+            if (ret)
+                goto out;
         }
+    }
 
-        *options = dict;
+    *options = dict;
 
 out:
-        if (ret && dict) {
-                dict_destroy (dict);
-        }
+    if (ret && dict) {
+        dict_unref(dict);
+    }
 
-                return ret;
+    return ret;
 }
 
-/* Parsing global option for NFS-Ganesha config
- *  gluster nfs-ganesha enable/disable */
-
 int32_t
-cli_cmd_ganesha_parse (struct cli_state *state,
-                       const char **words, int wordcount,
-                       dict_t **options, char **op_errstr)
+cli_cmd_get_state_parse(struct cli_state *state, const char **words,
+                        int wordcount, dict_t **options, char **op_errstr)
 {
-        dict_t  *dict     =       NULL;
-        int     ret       =       -1;
-        int     flags     =       0;
-        char    *key      =       NULL;
-        char    *value    =       NULL;
-        int     i         =       0;
-        char    *w        =       NULL;
-        char   *opwords[] =      { "enable", "disable", NULL };
-        const char      *question       =       NULL;
-        gf_answer_t     answer          =       GF_ANSWER_NO;
+    dict_t *dict = NULL;
+    int ret = -1;
+    char *odir = NULL;
+    char *filename = NULL;
+    char *daemon_name = NULL;
+    int count = 0;
+    uint32_t cmd = 0;
+
+    GF_VALIDATE_OR_GOTO("cli", options, out);
+    GF_VALIDATE_OR_GOTO("cli", words, out);
+
+    dict = dict_new();
+    if (!dict)
+        goto out;
 
+    if (wordcount < 1 || wordcount > 7) {
+        *op_errstr = gf_strdup(
+            "Problem parsing arguments."
+            " Check usage.");
+        goto out;
+    }
+
+    if (wordcount >= 1) {
+        gf_asprintf(&daemon_name, "%s", "glusterd");
+
+        for (count = 1; count < wordcount; count++) {
+            if (strcmp(words[count], "odir") == 0 ||
+                strcmp(words[count], "file") == 0) {
+                if (strcmp(words[count], "odir") == 0) {
+                    if (++count < wordcount) {
+                        odir = (char *)words[count];
+                        continue;
+                    } else {
+                        ret = -1;
+                        goto out;
+                    }
+                } else if (strcmp(words[count], "file") == 0) {
+                    if (++count < wordcount) {
+                        filename = (char *)words[count];
+                        continue;
+                    } else {
+                        ret = -1;
+                        goto out;
+                    }
+                }
+            } else {
+                if (count > 1) {
+                    if (count == wordcount - 1) {
+                        if (strcmp(words[count], "detail") == 0) {
+                            cmd = GF_CLI_GET_STATE_DETAIL;
+                            continue;
+                        } else if (strcmp(words[count], "volumeoptions") == 0) {
+                            cmd = GF_CLI_GET_STATE_VOLOPTS;
+                            continue;
+                        }
+                    } else {
+                        *op_errstr = gf_strdup(
+                            "Problem"
+                            " parsing arguments. "
+                            "Check usage.");
+                        ret = -1;
+                        goto out;
+                    }
+                }
+                if (strcmp(words[count], "glusterd") == 0) {
+                    continue;
+                } else {
+                    if (count == wordcount - 1) {
+                        if (strcmp(words[count], "detail") == 0) {
+                            cmd = GF_CLI_GET_STATE_DETAIL;
+                            continue;
+                        } else if (strcmp(words[count], "volumeoptions") == 0) {
+                            cmd = GF_CLI_GET_STATE_VOLOPTS;
+                            continue;
+                        }
+                    }
 
-        GF_ASSERT (words);
-        GF_ASSERT (options);
+                    *op_errstr = gf_strdup(
+                        "glusterd is "
+                        "the only supported daemon.");
+                    ret = -1;
+                    goto out;
+                }
+            }
+        }
 
-        dict = dict_new ();
+        ret = dict_set_dynstr(dict, "daemon", daemon_name);
+        if (ret) {
+            *op_errstr = gf_strdup(
+                "Command failed. Please check "
+                " log file for more details.");
+            gf_log(THIS->name, GF_LOG_ERROR,
+                   "Setting daemon name to dictionary failed");
+            goto out;
+        }
+        daemon_name = NULL;
+
+        if (odir) {
+            ret = dict_set_str(dict, "odir", odir);
+            if (ret) {
+                *op_errstr = gf_strdup(
+                    "Command failed. Please"
+                    " check log file for"
+                    " more details.");
+                gf_log(THIS->name, GF_LOG_ERROR,
+                       "Setting output directory to"
+                       "dictionary failed");
+                goto out;
+            }
+        }
+
+        if (filename) {
+            ret = dict_set_str(dict, "filename", filename);
+            if (ret) {
+                *op_errstr = gf_strdup(
+                    "Command failed. Please"
+                    " check log file for"
+                    " more  details.");
+                gf_log(THIS->name, GF_LOG_ERROR,
+                       "Setting filename to dictionary failed");
+                goto out;
+            }
+        }
+
+        if (cmd) {
+            ret = dict_set_uint32(dict, "getstate-cmd", cmd);
+            if (ret) {
+                *op_errstr = gf_strdup(
+                    "Command failed. Please"
+                    " check log file for"
+                    " more details.");
+                gf_log(THIS->name, GF_LOG_ERROR,
+                       "Setting "
+                       "get-state command type to dictionary "
+                       "failed");
+                goto out;
+            }
+        }
+    }
 
-        if (!dict)
-                goto out;
+out:
+    if (dict)
+        *options = dict;
 
-        if (wordcount != 2)
-                goto out;
+    if (ret && dict)
+        dict_unref(dict);
 
-        key   = (char *) words[0];
-        value = (char *) words[1];
+    GF_FREE(daemon_name);
 
-        if (!key || !value) {
-                cli_out ("Usage : nfs-ganesha <enable/disable>");
-                ret = -1;
-                goto out;
-        }
+    return ret;
+}
 
-        ret = gf_strip_whitespace (value, strlen (value));
-        if (ret == -1)
-                goto out;
+int32_t
+cli_cmd_inode_quota_parse(const char **words, int wordcount, dict_t **options)
+{
+    dict_t *dict = NULL;
+    char *volname = NULL;
+    int ret = -1;
 
-        if (strcmp (key, "nfs-ganesha")) {
-                gf_asprintf (op_errstr, "Global option: error: ' %s '"
-                          "is not a valid global option.", key);
-                ret = -1;
-                goto out;
-        }
+    GF_ASSERT(words);
+    GF_ASSERT(options);
 
-        w = str_getunamb (value, opwords);
-        if (!w) {
-                cli_out ("Invalid global option \n"
-                         "Usage : nfs-ganesha <enable/disable>");
-                ret = -1;
-                goto out;
-        }
+    dict = dict_new();
+    if (!dict) {
+        gf_log("cli", GF_LOG_ERROR, "dict_new failed");
+        goto out;
+    }
 
-        question = "Enabling NFS-Ganesha requires Gluster-NFS to be"
-                   " disabled across the trusted pool. Do you "
-                   "still want to continue?\n";
+    if (wordcount != 4)
+        goto out;
 
-        if (strcmp (value, "enable") == 0) {
-                answer = cli_cmd_get_confirmation (state, question);
-                if (GF_ANSWER_NO == answer) {
-                        gf_log ("cli", GF_LOG_ERROR, "Global operation "
-                                "cancelled, exiting");
-                        ret = -1;
-                        goto out;
-                }
-        }
-        cli_out ("This will take a few minutes to complete. Please wait ..");
+    volname = (char *)words[2];
+    if (!volname) {
+        ret = -1;
+        goto out;
+    }
 
-        ret = dict_set_str (dict, "key", key);
-        if (ret) {
-               gf_log (THIS->name, GF_LOG_ERROR, "dict set on key failed");
-                goto out;
-        }
+    /* Validate the volume name here itself */
+    if (cli_validate_volname(volname) < 0)
+        goto out;
 
-        ret = dict_set_str (dict, "value", value);
-        if (ret) {
-               gf_log (THIS->name, GF_LOG_ERROR, "dict set on value failed");
-                goto out;
-        }
+    ret = dict_set_str(dict, "volname", volname);
+    if (ret < 0)
+        goto out;
 
-        ret = dict_set_str (dict, "globalname", "All");
-        if (ret) {
-                gf_log (THIS->name, GF_LOG_ERROR, "dict set on global"
-                        " key failed.");
-                goto out;
-        }
+    if (strcmp(words[3], "enable") != 0) {
+        cli_out("Invalid quota option : %s", words[3]);
+        ret = -1;
+        goto out;
+    }
 
-        ret = dict_set_int32 (dict, "hold_global_locks", _gf_true);
-        if (ret) {
-                gf_log (THIS->name, GF_LOG_ERROR, "dict set on global key "
-                        "failed.");
-                goto out;
-        }
+    ret = dict_set_int32(dict, "type", GF_QUOTA_OPTION_TYPE_ENABLE_OBJECTS);
+    if (ret < 0)
+        goto out;
 
-        *options = dict;
+    *options = dict;
 out:
-        if (ret)
-                dict_unref (dict);
+    if (ret < 0) {
+        if (dict)
+            dict_unref(dict);
+    }
 
-        return ret;
+    return ret;
 }
 
 int32_t
-cli_cmd_inode_quota_parse (const char **words, int wordcount, dict_t **options)
+cli_cmd_quota_parse(const char **words, int wordcount, dict_t **options)
 {
-        dict_t          *dict    = NULL;
-        char            *volname = NULL;
-        int              ret     = -1;
-
-        GF_ASSERT (words);
-        GF_ASSERT (options);
+    dict_t *dict = NULL;
+    char *volname = NULL;
+    int ret = -1;
+    int i = -1;
+    char key[20] = {
+        0,
+    };
+    int64_t value = 0;
+    gf_quota_type type = GF_QUOTA_OPTION_TYPE_NONE;
+    static char *opwords[] = {"enable",
+                              "disable",
+                              "limit-usage",
+                              "remove",
+                              "list",
+                              "alert-time",
+                              "soft-timeout",
+                              "hard-timeout",
+                              "default-soft-limit",
+                              "limit-objects",
+                              "list-objects",
+                              "remove-objects",
+                              NULL};
+    char *w = NULL;
+    uint32_t time = 0;
+    double percent = 0;
+    char *end_ptr = NULL;
+    int64_t limit = 0;
+
+    GF_ASSERT(words);
+    GF_ASSERT(options);
+
+    dict = dict_new();
+    if (!dict) {
+        gf_log("cli", GF_LOG_ERROR, "dict_new failed");
+        goto out;
+    }
 
-        dict = dict_new ();
-        if (!dict) {
-                gf_log ("cli", GF_LOG_ERROR, "dict_new failed");
-                goto out;
+    if (wordcount < 4) {
+        if ((wordcount == 3) && !(strcmp(words[2], "help"))) {
+            ret = 1;
         }
+        goto out;
+    }
 
-        if (wordcount != 4)
-                goto out;
+    volname = (char *)words[2];
+    if (!volname) {
+        ret = -1;
+        goto out;
+    }
 
-        volname = (char *)words[2];
-        if (!volname) {
-                ret = -1;
-                goto out;
-        }
+    /* Validate the volume name here itself */
+    if (cli_validate_volname(volname) < 0)
+        goto out;
 
-        /* Validate the volume name here itself */
-        if (cli_validate_volname (volname) < 0)
-                goto out;
+    ret = dict_set_str(dict, "volname", volname);
+    if (ret < 0)
+        goto out;
 
-        ret = dict_set_str (dict, "volname", volname);
-        if (ret < 0)
-                goto out;
+    w = str_getunamb(words[3], opwords);
+    if (!w) {
+        cli_out("Invalid quota option : %s", words[3]);
+        ret = -1;
+        goto out;
+    }
 
-        if (strcmp (words[3], "enable") != 0) {
-                cli_out ("Invalid quota option : %s", words[3]);
-                ret = -1;
-                goto out;
+    if (strcmp(w, "enable") == 0) {
+        if (wordcount == 4) {
+            type = GF_QUOTA_OPTION_TYPE_ENABLE;
+            ret = 0;
+            goto set_type;
+        } else {
+            ret = -1;
+            goto out;
         }
+    }
 
-        ret = dict_set_int32 (dict, "type",
-                              GF_QUOTA_OPTION_TYPE_ENABLE_OBJECTS);
-        if (ret < 0)
-                goto out;
-
-        *options = dict;
-out:
-        if (ret < 0) {
-                if (dict)
-                        dict_destroy (dict);
+    if (strcmp(w, "disable") == 0) {
+        if (wordcount == 4) {
+            type = GF_QUOTA_OPTION_TYPE_DISABLE;
+            ret = 0;
+            goto set_type;
+        } else {
+            ret = -1;
+            goto out;
         }
+    }
 
-        return ret;
-}
+    if (strcmp(w, "limit-usage") == 0) {
+        type = GF_QUOTA_OPTION_TYPE_LIMIT_USAGE;
+    } else if (strcmp(w, "limit-objects") == 0) {
+        type = GF_QUOTA_OPTION_TYPE_LIMIT_OBJECTS;
+    }
 
-int32_t
-cli_cmd_quota_parse (const char **words, int wordcount, dict_t **options)
-{
-        dict_t          *dict    = NULL;
-        char            *volname = NULL;
-        int              ret     = -1;
-        int              i       = -1;
-        char             key[20] = {0, };
-        int64_t          value   = 0;
-        gf_quota_type    type    = GF_QUOTA_OPTION_TYPE_NONE;
-        char           *opwords[] = { "enable", "disable", "limit-usage",
-                                      "remove", "list", "alert-time",
-                                      "soft-timeout", "hard-timeout",
-                                      "default-soft-limit", "limit-objects",
-                                      "list-objects", "remove-objects", NULL};
-        char            *w       = NULL;
-        uint32_t         time    = 0;
-        double           percent = 0;
-        char            *end_ptr = NULL;
-        int64_t          limit   = 0;
-
-        GF_ASSERT (words);
-        GF_ASSERT (options);
-
-        dict = dict_new ();
-        if (!dict) {
-                gf_log ("cli", GF_LOG_ERROR, "dict_new failed");
-                goto out;
+    if (type == GF_QUOTA_OPTION_TYPE_LIMIT_USAGE ||
+        type == GF_QUOTA_OPTION_TYPE_LIMIT_OBJECTS) {
+        if (wordcount < 6 || wordcount > 7) {
+            ret = -1;
+            goto out;
         }
 
-        if (wordcount < 4)
+        if (words[4][0] != '/') {
+            cli_err("Please enter absolute path");
+            ret = -1;
+            goto out;
+        }
+        ret = dict_set_str(dict, "path", (char *)words[4]);
+        if (ret)
+            goto out;
+
+        if (!words[5]) {
+            cli_err("Please enter the limit value to be set");
+            ret = -1;
+            goto out;
+        }
+
+        if (type == GF_QUOTA_OPTION_TYPE_LIMIT_USAGE) {
+            ret = gf_string2bytesize_int64(words[5], &value);
+            if (ret != 0 || value <= 0) {
+                if (errno == ERANGE || value <= 0) {
+                    ret = -1;
+                    cli_err(
+                        "Please enter an integer "
+                        "value in the range of "
+                        "(1 - %" PRId64 ")",
+                        INT64_MAX);
+                } else
+                    cli_err(
+                        "Please enter a correct "
+                        "value");
                 goto out;
-
-        volname = (char *)words[2];
-        if (!volname) {
+            }
+        } else {
+            errno = 0;
+            limit = strtol(words[5], &end_ptr, 10);
+            if (errno == ERANGE || errno == EINVAL || limit <= 0 ||
+                strcmp(end_ptr, "") != 0) {
                 ret = -1;
+                cli_err(
+                    "Please enter an integer value in "
+                    "the range 1 - %" PRId64,
+                    INT64_MAX);
                 goto out;
+            }
         }
 
-        /* Validate the volume name here itself */
-        if (cli_validate_volname (volname) < 0)
-                goto out;
-
-        ret = dict_set_str (dict, "volname", volname);
+        ret = dict_set_str(dict, "hard-limit", (char *)words[5]);
         if (ret < 0)
-                goto out;
+            goto out;
 
-        w = str_getunamb (words[3], opwords);
-        if (!w) {
-                cli_out ("Invalid quota option : %s", words[3]);
-                ret = - 1;
+        if (wordcount == 7) {
+            ret = gf_string2percent(words[6], &percent);
+            if (ret != 0 || percent > 100) {
+                ret = -1;
+                cli_err(
+                    "Please enter a correct value "
+                    "in the range of 0 to 100");
                 goto out;
-        }
+            }
 
-        if (strcmp (w, "enable") == 0) {
-                if (wordcount == 4) {
-                        type = GF_QUOTA_OPTION_TYPE_ENABLE;
-                        ret = 0;
-                        goto set_type;
-                } else {
-                        ret = -1;
-                        goto out;
-                }
+            ret = dict_set_str(dict, "soft-limit", (char *)words[6]);
+            if (ret < 0)
+                goto out;
         }
 
-        if (strcmp (w, "disable") == 0) {
-                if (wordcount == 4) {
-                        type = GF_QUOTA_OPTION_TYPE_DISABLE;
-                        ret = 0;
-                        goto set_type;
-                } else {
-                        ret = -1;
-                        goto out;
-                }
-        }
+        goto set_type;
+    }
 
-        if (strcmp (w, "limit-usage") == 0) {
-                type = GF_QUOTA_OPTION_TYPE_LIMIT_USAGE;
-        } else if (strcmp (w, "limit-objects") == 0) {
-                type = GF_QUOTA_OPTION_TYPE_LIMIT_OBJECTS;
+    if (strcmp(w, "remove") == 0) {
+        if (wordcount != 5) {
+            ret = -1;
+            goto out;
         }
 
-        if (type == GF_QUOTA_OPTION_TYPE_LIMIT_USAGE ||
-            type == GF_QUOTA_OPTION_TYPE_LIMIT_OBJECTS) {
-
-                if (wordcount < 6 || wordcount > 7) {
-                        ret = -1;
-                        goto out;
-                }
-
-                if (words[4][0] != '/') {
-                        cli_err ("Please enter absolute path");
-                        ret = -1;
-                        goto out;
-                }
-                ret = dict_set_str (dict, "path", (char *) words[4]);
-                if (ret)
-                        goto out;
-
-                if (!words[5]) {
-                        cli_err ("Please enter the limit value to be set");
-                        ret = -1;
-                        goto out;
-                }
-
-                if (type == GF_QUOTA_OPTION_TYPE_LIMIT_USAGE) {
-                        ret = gf_string2bytesize_int64 (words[5], &value);
-                        if (ret != 0 || value < 0) {
-                                if (errno == ERANGE || value < 0)
-                                        cli_err ("Value out of range "
-                                                 "(0 - %"PRId64 "): %s",
-                                                 INT64_MAX, words[5]);
-                                else
-                                        cli_err ("Please enter a correct "
-                                                 "value");
-                                goto out;
-                        }
-                } else {
-                        errno = 0;
-                        limit = strtol (words[5], &end_ptr, 10);
-                        if (errno == ERANGE || errno == EINVAL || limit <= 0
-                                            || strcmp (end_ptr, "") != 0) {
-                                ret = -1;
-                                cli_err ("Please enter an interger value in "
-                                         "the range 1 - %"PRId64, INT64_MAX);
-                                goto out;
-                        }
-                }
-
-                ret  = dict_set_str (dict, "hard-limit", (char *) words[5]);
-                if (ret < 0)
-                        goto out;
-
-                if (wordcount == 7) {
-
-                        ret = gf_string2percent (words[6], &percent);
-                        if (ret != 0 || percent > 100) {
-                                ret = -1;
-                                cli_err ("Please enter a correct value "
-                                         "in the range of 0 to 100");
-                                goto out;
-                        }
-
-                        ret = dict_set_str (dict, "soft-limit",
-                                            (char *) words[6]);
-                        if (ret < 0)
-                                goto out;
-                }
+        type = GF_QUOTA_OPTION_TYPE_REMOVE;
 
-                goto set_type;
+        if (words[4][0] != '/') {
+            cli_err("Please enter absolute path");
+            ret = -1;
+            goto out;
         }
 
-        if (strcmp (w, "remove") == 0) {
-                if (wordcount != 5) {
-                        ret = -1;
-                        goto out;
-                }
-
-                type = GF_QUOTA_OPTION_TYPE_REMOVE;
-
-                if (words[4][0] != '/') {
-                        cli_err ("Please enter absolute path");
-                        ret = -1;
-                        goto out;
-                }
+        ret = dict_set_str(dict, "path", (char *)words[4]);
+        if (ret < 0)
+            goto out;
+        goto set_type;
+    }
 
-                ret = dict_set_str (dict, "path", (char *) words[4]);
-                if (ret < 0)
-                        goto out;
-                goto set_type;
+    if (strcmp(w, "remove-objects") == 0) {
+        if (wordcount != 5) {
+            ret = -1;
+            goto out;
         }
 
-        if (strcmp (w, "remove-objects") == 0) {
-                if (wordcount != 5) {
-                        ret = -1;
-                        goto out;
-                }
-
-                type = GF_QUOTA_OPTION_TYPE_REMOVE_OBJECTS;
-
-                if (words[4][0] != '/') {
-                        cli_err ("Please enter absolute path");
-                        ret = -1;
-                        goto out;
-                }
+        type = GF_QUOTA_OPTION_TYPE_REMOVE_OBJECTS;
 
-                ret = dict_set_str (dict, "path", (char *) words[4]);
-                if (ret < 0)
-                        goto out;
-                goto set_type;
+        if (words[4][0] != '/') {
+            cli_err("Please enter absolute path");
+            ret = -1;
+            goto out;
         }
 
-        if (strcmp (w, "list") == 0) {
-
-                type = GF_QUOTA_OPTION_TYPE_LIST;
-
-                if (words[4] && words[4][0] != '/') {
-                        cli_err ("Please enter absolute path");
-                        ret = -1;
-                        goto out;
-                }
+        ret = dict_set_str(dict, "path", (char *)words[4]);
+        if (ret < 0)
+            goto out;
+        goto set_type;
+    }
 
-                i = 4;
-                while (i < wordcount) {
-                        snprintf (key, 20, "path%d", i-4);
+    if (strcmp(w, "list") == 0) {
+        type = GF_QUOTA_OPTION_TYPE_LIST;
 
-                        ret = dict_set_str (dict, key, (char *) words [i++]);
-                        if (ret < 0)
-                                goto out;
-                }
+        if (words[4] && words[4][0] != '/') {
+            cli_err("Please enter absolute path");
+            ret = -1;
+            goto out;
+        }
 
-                ret = dict_set_int32 (dict, "count", i - 4);
-                if (ret < 0)
-                        goto out;
+        i = 4;
+        while (i < wordcount) {
+            snprintf(key, 20, "path%d", i - 4);
 
-                goto set_type;
+            ret = dict_set_str(dict, key, (char *)words[i++]);
+            if (ret < 0)
+                goto out;
         }
 
-        if (strcmp (w, "list-objects") == 0) {
-                if (wordcount < 4) {
-                        ret = -1;
-                        goto out;
-                }
+        ret = dict_set_int32(dict, "count", i - 4);
+        if (ret < 0)
+            goto out;
 
-                type = GF_QUOTA_OPTION_TYPE_LIST_OBJECTS;
+        goto set_type;
+    }
 
-                i = 4;
-                while (i < wordcount) {
-                        snprintf (key, 20, "path%d", i-4);
+    if (strcmp(w, "list-objects") == 0) {
+        type = GF_QUOTA_OPTION_TYPE_LIST_OBJECTS;
 
-                        ret = dict_set_str (dict, key, (char *) words[i++]);
-                        if (ret < 0) {
-                                gf_log ("cli", GF_LOG_ERROR, "Failed to set "
-                                        "quota patch in request dictionary");
-                                goto out;
-                        }
-                }
+        i = 4;
+        while (i < wordcount) {
+            snprintf(key, 20, "path%d", i - 4);
 
-                ret = dict_set_int32 (dict, "count", i - 4);
-                if (ret < 0) {
-                        gf_log ("cli", GF_LOG_ERROR, "Failed to set quota "
-                                "limit count in request dictionary");
-                        goto out;
-                }
+            ret = dict_set_str(dict, key, (char *)words[i++]);
+            if (ret < 0) {
+                gf_log("cli", GF_LOG_ERROR,
+                       "Failed to set "
+                       "quota patch in request dictionary");
+                goto out;
+            }
+        }
 
-                goto set_type;
+        ret = dict_set_int32(dict, "count", i - 4);
+        if (ret < 0) {
+            gf_log("cli", GF_LOG_ERROR,
+                   "Failed to set quota "
+                   "limit count in request dictionary");
+            goto out;
         }
 
-        if (strcmp (w, "alert-time") == 0) {
-                if (wordcount != 5) {
-                        ret = -1;
-                        goto out;
-                }
-                type = GF_QUOTA_OPTION_TYPE_ALERT_TIME;
+        goto set_type;
+    }
 
-                ret = gf_string2time (words[4], &time);
-                if (ret) {
-                        cli_err ("Invalid argument %s. Please enter a valid "
-                                 "string", words[4]);
-                        goto out;
-                }
+    if (strcmp(w, "alert-time") == 0) {
+        if (wordcount != 5) {
+            ret = -1;
+            goto out;
+        }
+        type = GF_QUOTA_OPTION_TYPE_ALERT_TIME;
 
-                ret = dict_set_str (dict, "value", (char *)words[4]);
-                if (ret < 0)
-                        goto out;
-                goto set_type;
+        ret = gf_string2time(words[4], &time);
+        if (ret) {
+            cli_err(
+                "Invalid argument %s. Please enter a valid "
+                "string",
+                words[4]);
+            goto out;
         }
 
-        if (strcmp (w, "soft-timeout") == 0) {
-                if (wordcount != 5) {
-                        ret = -1;
-                        goto out;
-                }
-                type = GF_QUOTA_OPTION_TYPE_SOFT_TIMEOUT;
+        ret = dict_set_str(dict, "value", (char *)words[4]);
+        if (ret < 0)
+            goto out;
+        goto set_type;
+    }
 
-                ret = gf_string2time (words[4], &time);
-                if (ret) {
-                        cli_err ("Invalid argument %s. Please enter a valid "
-                                 "string", words[4]);
-                        goto out;
-                }
+    if (strcmp(w, "soft-timeout") == 0) {
+        if (wordcount != 5) {
+            ret = -1;
+            goto out;
+        }
+        type = GF_QUOTA_OPTION_TYPE_SOFT_TIMEOUT;
 
-                ret = dict_set_str (dict, "value", (char *)words[4]);
-                if (ret < 0)
-                        goto out;
-                goto set_type;
+        ret = gf_string2time(words[4], &time);
+        if (ret) {
+            cli_err(
+                "Invalid argument %s. Please enter a valid "
+                "string",
+                words[4]);
+            goto out;
         }
 
-        if (strcmp (w, "hard-timeout") == 0) {
-                if(wordcount != 5) {
-                        ret = -1;
-                        goto out;
-                }
-                type = GF_QUOTA_OPTION_TYPE_HARD_TIMEOUT;
+        ret = dict_set_str(dict, "value", (char *)words[4]);
+        if (ret < 0)
+            goto out;
+        goto set_type;
+    }
 
-                ret = gf_string2time (words[4], &time);
-                if (ret) {
-                        cli_err ("Invalid argument %s. Please enter a valid "
-                                 "string", words[4]);
-                        goto out;
-                }
+    if (strcmp(w, "hard-timeout") == 0) {
+        if (wordcount != 5) {
+            ret = -1;
+            goto out;
+        }
+        type = GF_QUOTA_OPTION_TYPE_HARD_TIMEOUT;
 
-                ret = dict_set_str (dict, "value", (char *)words[4]);
-                if (ret < 0)
-                        goto out;
-                goto set_type;
+        ret = gf_string2time(words[4], &time);
+        if (ret) {
+            cli_err(
+                "Invalid argument %s. Please enter a valid "
+                "string",
+                words[4]);
+            goto out;
         }
-        if (strcmp (w, "default-soft-limit") == 0) {
-                if(wordcount != 5) {
-                        ret = -1;
-                        goto out;
-                }
-                type = GF_QUOTA_OPTION_TYPE_DEFAULT_SOFT_LIMIT;
 
-                ret = dict_set_str (dict, "value", (char *)words[4]);
-                if (ret < 0)
-                        goto out;
-                goto set_type;
-        } else {
-                GF_ASSERT (!"opword mismatch");
+        ret = dict_set_str(dict, "value", (char *)words[4]);
+        if (ret < 0)
+            goto out;
+        goto set_type;
+    }
+    if (strcmp(w, "default-soft-limit") == 0) {
+        if (wordcount != 5) {
+            ret = -1;
+            goto out;
         }
+        type = GF_QUOTA_OPTION_TYPE_DEFAULT_SOFT_LIMIT;
 
-set_type:
-        ret = dict_set_int32 (dict, "type", type);
+        ret = dict_set_str(dict, "value", (char *)words[4]);
         if (ret < 0)
-                goto out;
+            goto out;
+        goto set_type;
+    } else {
+        GF_ASSERT(!"opword mismatch");
+    }
 
-        *options = dict;
+set_type:
+    ret = dict_set_int32(dict, "type", type);
+    if (ret < 0)
+        goto out;
+
+    *options = dict;
 out:
-        if (ret < 0) {
-                if (dict)
-                        dict_destroy (dict);
-        }
+    if (ret < 0) {
+        if (dict)
+            dict_unref(dict);
+    }
 
-        return ret;
+    return ret;
 }
 
 static gf_boolean_t
-cli_is_key_spl (char *key)
+cli_is_key_spl(char *key)
 {
-        return (strcmp (key, "group") == 0);
+    return (strcmp(key, "group") == 0);
 }
 
-static int
-cli_add_key_group (dict_t *dict, char *key, char *value, char **op_errstr)
+static int32_t
+cli_add_key_group_value(dict_t *dict, const char *name, const char *value,
+                        int32_t id, char **op_errstr)
 {
-        int             ret = -1;
-        int             opt_count = 0;
-        char            iter_key[1024] = {0,};
-        char            iter_val[1024] = {0,};
-        char            *saveptr = NULL;
-        char            *tok_key = NULL;
-        char            *tok_val = NULL;
-        char            *dkey = NULL;
-        char            *dval = NULL;
-        char            *tagpath = NULL;
-        char            *buf = NULL;
-        char            line[PATH_MAX + 256] = {0,};
-        char            errstr[2048] = "";
-        FILE            *fp = NULL;
-
-        ret = gf_asprintf (&tagpath, "%s/groups/%s",
-                           GLUSTERD_DEFAULT_WORKDIR, value);
-        if (ret == -1) {
-                tagpath = NULL;
-                goto out;
-        }
+    char *key = NULL;
+    char *data = NULL;
+    int32_t ret = -1;
 
-        fp = fopen (tagpath, "r");
-        if (!fp) {
-                ret = -1;
-                snprintf(errstr, sizeof(errstr), "Unable to open file '%s'."
-                         " Error: %s", tagpath, strerror (errno));
-                if (op_errstr)
-                        *op_errstr = gf_strdup(errstr);
-                goto out;
-        }
+    ret = gf_asprintf(&key, "%s%d", name, id);
+    if (ret < 0) {
+        goto out;
+    }
+    data = gf_strdup(value);
+    if (data == NULL) {
+        gf_log(THIS->name, GF_LOG_ERROR, "Failed to allocate memory for data");
+        ret = -1;
+        goto out;
+    }
 
-        opt_count = 0;
-        buf = line;
-        while (fscanf (fp, "%s", buf) != EOF) {
+    ret = dict_set_dynstr(dict, key, data);
+    if (ret == 0) {
+        data = NULL;
+    }
 
-                opt_count++;
-                tok_key = strtok_r (line, "=", &saveptr);
-                tok_val = strtok_r (NULL, "=", &saveptr);
-                if (!tok_key || !tok_val) {
-                        ret = -1;
-                        snprintf(errstr, sizeof(errstr), "'%s' file format "
-                                 "not valid.", tagpath);
-                        if (op_errstr)
-                                *op_errstr = gf_strdup(errstr);
-                        goto out;
-                }
+out:
+    GF_FREE(key);
+    GF_FREE(data);
 
-                snprintf (iter_key, sizeof (iter_key), "key%d", opt_count);
-                dkey = gf_strdup (tok_key);
-                ret = dict_set_dynstr (dict, iter_key, dkey);
-                if (ret)
-                        goto out;
-                dkey = NULL;
+    if ((ret != 0) && (op_errstr != NULL)) {
+        *op_errstr = gf_strdup("Failed to allocate memory");
+    }
 
-                snprintf (iter_val, sizeof (iter_val), "value%d", opt_count);
-                dval = gf_strdup (tok_val);
-                ret = dict_set_dynstr (dict, iter_val, dval);
-                if (ret)
-                        goto out;
-                dval = NULL;
+    return ret;
+}
 
-        }
+static int
+cli_add_key_group(dict_t *dict, char *key, char *value, char **op_errstr)
+{
+    int ret = -1;
+    int opt_count = 0;
+    char *saveptr = NULL;
+    char *tok_key = NULL;
+    char *tok_val = NULL;
+    char *tagpath = NULL;
+    char line[PATH_MAX + 256] = {
+        0,
+    };
+    FILE *fp = NULL;
+
+    ret = gf_asprintf(&tagpath, "%s/groups/%s", GLUSTERD_DEFAULT_WORKDIR,
+                      value);
+    if (ret == -1) {
+        tagpath = NULL;
+        goto out;
+    }
 
-        if (!opt_count) {
-                ret = -1;
-                snprintf(errstr, sizeof(errstr), "'%s' file format "
-                         "not valid.", tagpath);
-                if (op_errstr)
-                        *op_errstr = gf_strdup(errstr);
-                goto out;
+    fp = fopen(tagpath, "r");
+    if (!fp) {
+        ret = -1;
+        if (op_errstr) {
+            gf_asprintf(op_errstr,
+                        "Unable to open file '%s'. "
+                        "Error: %s",
+                        tagpath, strerror(errno));
+        }
+        goto out;
+    }
+
+    opt_count = 0;
+    while (fgets(line, sizeof(line), fp) != NULL) {
+        if (strlen(line) >= sizeof(line) - 1) {
+            ret = -1;
+            if (op_errstr != NULL) {
+                *op_errstr = gf_strdup("Line too long");
+            }
+            goto out;
+        }
+
+        /* Treat line that start with "#" as comments */
+        if ('#' == line[0])
+            continue;
+
+        opt_count++;
+        tok_key = strtok_r(line, "=", &saveptr);
+        tok_val = strtok_r(NULL, "\r\n", &saveptr);
+        if (!tok_key || !tok_val) {
+            ret = -1;
+            if (op_errstr) {
+                gf_asprintf(op_errstr,
+                            "'%s' file format "
+                            "not valid.",
+                            tagpath);
+            }
+            goto out;
+        }
+
+        ret = cli_add_key_group_value(dict, "key", tok_key, opt_count,
+                                      op_errstr);
+        if (ret != 0) {
+            goto out;
+        }
+        ret = cli_add_key_group_value(dict, "value", tok_val, opt_count,
+                                      op_errstr);
+        if (ret != 0) {
+            goto out;
+        }
+    }
+
+    if (!opt_count) {
+        ret = -1;
+        if (op_errstr) {
+            gf_asprintf(op_errstr, "'%s' file format not valid.", tagpath);
         }
-        ret = dict_set_int32 (dict, "count", opt_count);
+        goto out;
+    }
+    ret = dict_set_int32(dict, "count", opt_count);
 out:
 
-        GF_FREE (tagpath);
-
-        if (ret) {
-                GF_FREE (dkey);
-                GF_FREE (dval);
-        }
+    GF_FREE(tagpath);
 
-        if (fp)
-                fclose (fp);
+    if (fp)
+        fclose(fp);
 
-        return ret;
+    return ret;
 }
 
 int32_t
-cli_cmd_volume_set_parse (struct cli_state *state, const char **words,
-                          int wordcount, dict_t **options, char **op_errstr)
+cli_cmd_volume_set_parse(struct cli_state *state, const char **words,
+                         int wordcount, dict_t **options, char **op_errstr)
 {
-        dict_t                 *dict      = NULL;
-        char                   *volname   = NULL;
-        int                     ret       = -1;
-        int                     count     = 0;
-        char                   *key       = NULL;
-        char                   *value     = NULL;
-        int                     i         = 0;
-        char                    str[50]   = {0,};
-        const char             *question  = NULL;
-        gf_answer_t             answer    = GF_ANSWER_NO;
-
-        GF_ASSERT (words);
-        GF_ASSERT (options);
-
-        dict = dict_new ();
-
-        if (!dict)
-                goto out;
-
-        if (wordcount < 3)
-                goto out;
+    dict_t *dict = NULL;
+    char *volname = NULL;
+    int ret = -1;
+    int count = 0;
+    char *key = NULL;
+    char *value = NULL;
+    int i = 0;
+    char str[50] = {
+        0,
+    };
+    const char *question = NULL;
+    gf_answer_t answer = GF_ANSWER_NO;
+
+    GF_ASSERT(words);
+    GF_ASSERT(options);
+
+    dict = dict_new();
+
+    if (!dict)
+        goto out;
 
-        volname = (char *)words[2];
+    if (wordcount < 3)
+        goto out;
 
-        GF_ASSERT (volname);
+    volname = (char *)words[2];
 
-        ret = dict_set_str (dict, "volname", volname);
+    GF_ASSERT(volname);
 
-        if (ret)
-                goto out;
+    ret = dict_set_str(dict, "volname", volname);
 
-        if (!strcmp (volname, "all")) {
-                ret = dict_set_str (dict, "globalname", "All");
-                if (ret) {
-                        gf_log (THIS->name, GF_LOG_ERROR,
-                                "dict set on global key failed.");
-                        goto out;
-                }
+    if (ret)
+        goto out;
 
-                ret = dict_set_int32 (dict, "hold_global_locks", _gf_true);
-                if (ret) {
-                        gf_log (THIS->name, GF_LOG_ERROR,
-                                "dict set on global key failed.");
-                        goto out;
-                }
+    if (!strcmp(volname, "all")) {
+        ret = dict_set_str(dict, "globalname", "All");
+        if (ret) {
+            gf_log(THIS->name, GF_LOG_ERROR, "dict set on global key failed.");
+            goto out;
         }
 
-        if ((!strcmp (volname, "help") || !strcmp (volname, "help-xml"))
-            && wordcount == 3 ) {
-                ret = dict_set_str (dict, volname, volname);
-                if (ret)
-                        goto out;
+        ret = dict_set_int32(dict, "hold_global_locks", _gf_true);
+        if (ret) {
+            gf_log(THIS->name, GF_LOG_ERROR, "dict set on global key failed.");
+            goto out;
+        }
+    }
 
-        } else if (wordcount < 5) {
-                ret = -1;
-                goto out;
+    if ((!strcmp(volname, "help") || !strcmp(volname, "help-xml")) &&
+        wordcount == 3) {
+        ret = dict_set_str(dict, volname, volname);
+        if (ret)
+            goto out;
 
-        } else if (wordcount == 5  && cli_is_key_spl ((char *)words[3])) {
-                key = (char *) words[3];
-                value = (char *) words[4];
-                if ( !key || !value) {
-                        ret = -1;
-                        goto out;
-                }
+    } else if (wordcount < 5) {
+        ret = -1;
+        goto out;
 
-                ret = gf_strip_whitespace (value, strlen (value));
-                if (ret == -1)
-                        goto out;
+    } else if (wordcount == 5 && cli_is_key_spl((char *)words[3])) {
+        key = (char *)words[3];
+        value = (char *)words[4];
+        if (!key || !value) {
+            ret = -1;
+            goto out;
+        }
 
-                if (strlen (value) == 0) {
-                        ret = -1;
-                        goto out;
-                }
+        ret = gf_strip_whitespace(value, strlen(value));
+        if (ret == -1)
+            goto out;
 
-                ret = cli_add_key_group (dict, key, value, op_errstr);
-                if (ret == 0)
-                        *options = dict;
-                goto out;
+        if (strlen(value) == 0) {
+            ret = -1;
+            goto out;
         }
 
-        for (i = 3; i < wordcount; i+=2) {
-
-                key = (char *) words[i];
-                value = (char *) words[i+1];
+        ret = cli_add_key_group(dict, key, value, op_errstr);
+        if (ret == 0)
+            *options = dict;
+        goto out;
+    }
 
-                if ( !key || !value) {
-                        ret = -1;
-                        goto out;
-                }
+    for (i = 3; i < wordcount; i += 2) {
+        key = (char *)words[i];
+        value = (char *)words[i + 1];
 
-                count++;
+        if (!key || !value) {
+            ret = -1;
+            goto out;
+        }
 
-                ret = gf_strip_whitespace (value, strlen (value));
-                if (ret == -1)
-                        goto out;
+        count++;
 
-                if (strlen (value) == 0) {
-                        ret = -1;
-                        goto out;
-                }
+        if (fnmatch("user.*", key, FNM_NOESCAPE) != 0) {
+            ret = gf_strip_whitespace(value, strlen(value));
+            if (ret == -1)
+                goto out;
+        }
 
-                if (cli_is_key_spl (key)) {
-                        ret = -1;
-                        goto out;
-                }
+        if (strlen(value) == 0) {
+            ret = -1;
+            goto out;
+        }
 
-                sprintf (str, "key%d", count);
-                ret = dict_set_str (dict, str, key);
-                if (ret)
-                        goto out;
+        if (cli_is_key_spl(key)) {
+            ret = -1;
+            goto out;
+        }
 
-                sprintf (str, "value%d", count);
-                ret = dict_set_str (dict, str, value);
+        sprintf(str, "key%d", count);
+        ret = dict_set_str(dict, str, key);
+        if (ret)
+            goto out;
 
-                if (ret)
-                        goto out;
+        sprintf(str, "value%d", count);
+        ret = dict_set_str(dict, str, value);
 
-                if ((!strcmp (key, "cluster.enable-shared-storage")) &&
-                    (!strcmp (value, "disable"))) {
-                        question = "Disabling cluster.enable-shared-storage "
-                                   "will delete the shared storage volume"
-                                   "(gluster_shared_storage), which is used "
-                                   "by snapshot scheduler, geo-replication "
-                                   "and NFS-Ganesha. Do you still want to "
-                                   "continue?";
-                        answer = cli_cmd_get_confirmation (state, question);
-                        if (GF_ANSWER_NO == answer) {
-                                gf_log ("cli", GF_LOG_ERROR, "Operation "
-                                        "cancelled, exiting");
-                                *op_errstr = gf_strdup ("Aborted by user.");
-                                ret = -1;
-                                goto out;
-                        }
-                }
+        if (ret)
+            goto out;
+
+        if ((!strcmp(key, "cluster.enable-shared-storage")) &&
+            (!strcmp(value, "disable"))) {
+            question =
+                "Disabling cluster.enable-shared-storage "
+                "will delete the shared storage volume"
+                "(gluster_shared_storage), which is used "
+                "by snapshot scheduler, geo-replication "
+                "and NFS-Ganesha. Do you still want to "
+                "continue?";
+            answer = cli_cmd_get_confirmation(state, question);
+            if (GF_ANSWER_NO == answer) {
+                gf_log("cli", GF_LOG_ERROR,
+                       "Operation "
+                       "cancelled, exiting");
+                *op_errstr = gf_strdup("Aborted by user.");
+                ret = -1;
+                goto out;
+            }
+        }
+        if ((!strcmp(key, "nfs.disable")) && (!strcmp(value, "off"))) {
+            question =
+                "Gluster NFS is being deprecated in favor "
+                "of NFS-Ganesha Enter \"yes\" to continue "
+                "using Gluster NFS";
+            answer = cli_cmd_get_confirmation(state, question);
+            if (GF_ANSWER_NO == answer) {
+                gf_log("cli", GF_LOG_ERROR,
+                       "Operation "
+                       "cancelled, exiting");
+                *op_errstr = gf_strdup("Aborted by user.");
+                ret = -1;
+                goto out;
+            }
         }
+    }
 
-        ret = dict_set_int32 (dict, "count", wordcount-3);
+    ret = dict_set_int32(dict, "count", wordcount - 3);
 
-        if (ret)
-                goto out;
+    if (ret)
+        goto out;
 
-        *options = dict;
+    *options = dict;
 
 out:
-        if (ret)
-                dict_destroy (dict);
+    if (ret && dict)
+        dict_unref(dict);
 
-        return ret;
+    return ret;
 }
 
 int32_t
-cli_cmd_volume_add_brick_parse (const char **words, int wordcount,
-                                dict_t **options, int *ret_type)
+cli_cmd_volume_add_brick_parse(struct cli_state *state, const char **words,
+                               int wordcount, dict_t **options, int *ret_type)
 {
-        dict_t  *dict = NULL;
-        char    *volname = NULL;
-        int     ret = -1;
-        int     brick_count = 0, brick_index = 0;
-        char    *bricks = NULL;
-        char    *opwords_cl[] = { "replica", "stripe", NULL };
-        gf1_cluster_type type = GF_CLUSTER_TYPE_NONE;
-        int     count = 1;
-        char    *w = NULL;
-        int     index;
-        gf_boolean_t is_force = _gf_false;
-        int wc = wordcount;
-
-        GF_ASSERT (words);
-        GF_ASSERT (options);
-
-        dict = dict_new ();
-
-        if (!dict)
-                goto out;
+    dict_t *dict = NULL;
+    char *volname = NULL;
+    int ret = -1;
+    int brick_count = 0, brick_index = 0;
+    char *bricks = NULL;
+    static char *opwords_cl[] = {"replica", "stripe", NULL};
+    gf1_cluster_type type = GF_CLUSTER_TYPE_NONE;
+    int count = 1;
+    int arbiter_count = 0;
+    char *w = NULL;
+    int index;
+    gf_boolean_t is_force = _gf_false;
+    int wc = wordcount;
+    gf_answer_t answer = GF_ANSWER_NO;
+    const char *question = NULL;
+
+    GF_ASSERT(words);
+    GF_ASSERT(options);
+
+    dict = dict_new();
+
+    if (!dict)
+        goto out;
 
-        if (wordcount < 3)
-                goto out;
+    if (wordcount < 3)
+        goto out;
 
-        volname = (char *)words[2];
+    volname = (char *)words[2];
 
-        GF_ASSERT (volname);
+    GF_ASSERT(volname);
 
-        ret = dict_set_str (dict, "volname", volname);
+    ret = dict_set_str(dict, "volname", volname);
 
-        if (ret)
-                goto out;
+    if (ret)
+        goto out;
 
-        if (wordcount < 4) {
+    if (wordcount < 4) {
+        ret = -1;
+        goto out;
+    }
+    if (wordcount < 6) {
+        /* seems no options are given, go directly to the parse_brick */
+        brick_index = 3;
+        type = GF_CLUSTER_TYPE_NONE;
+        goto parse_bricks;
+    }
+
+    w = str_getunamb(words[3], opwords_cl);
+    if (!w) {
+        type = GF_CLUSTER_TYPE_NONE;
+        index = 3;
+    } else if ((strcmp(w, "replica")) == 0) {
+        type = GF_CLUSTER_TYPE_REPLICATE;
+        count = strtol(words[4], NULL, 0);
+        if (!count || (count < 2)) {
+            cli_err("replica count should be greater than 1");
+            ret = -1;
+            goto out;
+        }
+        ret = dict_set_int32(dict, "replica-count", count);
+        if (ret)
+            goto out;
+        index = 5;
+        if (words[index] && !strcmp(words[index], "arbiter")) {
+            arbiter_count = strtol(words[6], NULL, 0);
+            if (arbiter_count != 1 || count != 3) {
+                cli_err(
+                    "For arbiter configuration, replica "
+                    "count must be 3 and arbiter count "
+                    "must be 1. The 3rd brick of the "
+                    "replica will be the arbiter");
                 ret = -1;
                 goto out;
-        }
-        if (wordcount < 6) {
-                /* seems no options are given, go directly to the parse_brick */
-                brick_index = 3;
-                type = GF_CLUSTER_TYPE_NONE;
-                goto parse_bricks;
+            }
+            ret = dict_set_int32(dict, "arbiter-count", arbiter_count);
+            if (ret)
+                goto out;
+            index = 7;
         }
 
-        w = str_getunamb (words[3], opwords_cl);
-        if (!w) {
-                type = GF_CLUSTER_TYPE_NONE;
-                index = 3;
-        } else if ((strcmp (w, "replica")) == 0) {
-                type = GF_CLUSTER_TYPE_REPLICATE;
-                count = strtol (words[4], NULL, 0);
-                if (!count || (count < 2)) {
-                        cli_err ("replica count should be greater than 1");
-                        ret = -1;
-                        goto out;
-                }
-                ret = dict_set_int32 (dict, "replica-count", count);
-                if (ret)
-                        goto out;
-                index = 5;
-        } else if ((strcmp (w, "stripe")) == 0) {
-                type = GF_CLUSTER_TYPE_STRIPE;
-                count = strtol (words[4], NULL, 0);
-                if (!count || (count < 2)) {
-                        cli_err ("stripe count should be greater than 1");
-                        ret = -1;
-                        goto out;
+        if (count == 2) {
+            if (strcmp(words[wordcount - 1], "force")) {
+                question =
+                    "Replica 2 volumes are prone to "
+                    "split-brain. Use Arbiter or "
+                    "Replica 3 to avoid this. See: "
+                    "http://docs.gluster.org/en/latest/Administrator%20Guide/"
+                    "Split%20brain%20and%20ways%20to%20deal%20with%20it/."
+                    "\nDo you still want to continue?\n";
+                answer = cli_cmd_get_confirmation(state, question);
+                if (GF_ANSWER_NO == answer) {
+                    gf_log("cli", GF_LOG_ERROR,
+                           "Add brick"
+                           " cancelled, exiting");
+                    ret = -1;
+                    goto out;
                 }
-                ret = dict_set_int32 (dict, "stripe-count", count);
-                if (ret)
-                        goto out;
-                index = 5;
-        } else {
-                GF_ASSERT (!"opword mismatch");
-                ret = -1;
-                goto out;
+            }
         }
+    } else if ((strcmp(w, "stripe")) == 0) {
+        cli_err("stripe option not supported");
+        goto out;
+    } else {
+        GF_ASSERT(!"opword mismatch");
+        ret = -1;
+        goto out;
+    }
 
-        brick_index = index;
+    brick_index = index;
 
 parse_bricks:
 
-        if (strcmp (words[wordcount - 1], "force") == 0) {
-                is_force = _gf_true;
-                wc = wordcount - 1;
-        }
+    if (strcmp(words[wordcount - 1], "force") == 0) {
+        is_force = _gf_true;
+        wc = wordcount - 1;
+    }
 
-        ret = cli_cmd_bricks_parse (words, wc, brick_index, &bricks,
-                                    &brick_count);
-        if (ret)
-                goto out;
+    ret = cli_cmd_bricks_parse(words, wc, brick_index, &bricks, &brick_count);
+    if (ret)
+        goto out;
 
-        ret = dict_set_dynstr (dict, "bricks", bricks);
-        if (ret)
-                goto out;
+    ret = dict_set_dynstr(dict, "bricks", bricks);
+    if (ret)
+        goto out;
 
-        ret = dict_set_int32 (dict, "count", brick_count);
+    ret = dict_set_int32(dict, "count", brick_count);
 
-        if (ret)
-                goto out;
+    if (ret)
+        goto out;
 
-        ret = dict_set_int32 (dict, "force", is_force);
-        if (ret)
-                goto out;
+    ret = dict_set_int32(dict, "force", is_force);
+    if (ret)
+        goto out;
 
-        *options = dict;
+    *options = dict;
 
 out:
-        if (ret_type)
-                *ret_type = type;
+    if (ret_type)
+        *ret_type = type;
 
-        if (ret) {
-                gf_log ("cli", GF_LOG_ERROR, "Unable to parse add-brick CLI");
-                if (dict)
-                        dict_destroy (dict);
-        }
+    if (ret) {
+        gf_log("cli", GF_LOG_ERROR, "Unable to parse add-brick CLI");
+        if (dict)
+            dict_unref(dict);
+    }
 
-        return ret;
+    return ret;
 }
 
 int32_t
-cli_cmd_volume_tier_parse (const char **words, int wordcount,
-                           dict_t **options)
+cli_cmd_volume_remove_brick_parse(struct cli_state *state, const char **words,
+                                  int wordcount, dict_t **options,
+                                  int *question, int *brick_count,
+                                  int32_t *comm)
 {
-        dict_t  *dict    = NULL;
-        char    *volname = NULL;
-        int      ret     = -1;
-        int32_t  command = GF_OP_CMD_NONE;
-        int32_t  is_force    = 0;
+    dict_t *dict = NULL;
+    char *volname = NULL;
+    char *delimiter = NULL;
+    int ret = -1;
+    char key[50];
+    int brick_index = 0;
+    int32_t tmp_index = 0;
+    int32_t j = 0;
+    char *tmp_brick = NULL;
+    char *tmp_brick1 = NULL;
+    static char *type_opword[] = {"replica", NULL};
+    static char *opwords[] = {"start",  "commit", "stop",
+                              "status", "force",  NULL};
+    char *w = NULL;
+    int32_t command = GF_OP_CMD_NONE;
+    long count = 0;
+    gf_answer_t answer = GF_ANSWER_NO;
+    const char *ques = NULL;
+
+    GF_ASSERT(words);
+    GF_ASSERT(options);
+
+    if (wordcount < 5)
+        goto out;
+
+    dict = dict_new();
+    if (!dict)
+        goto out;
 
-        GF_ASSERT (words);
-        GF_ASSERT (options);
+    volname = (char *)words[2];
 
-        dict = dict_new ();
+    GF_ASSERT(volname);
 
-        if (!dict)
-                goto out;
+    ret = dict_set_str(dict, "volname", volname);
+    if (ret)
+        goto out;
 
-        if (!(wordcount == 4 || wordcount == 5)) {
-                gf_log ("cli", GF_LOG_ERROR, "Invalid Syntax");
-                ret = -1;
-                goto out;
+    brick_index = 3;
+    w = str_getunamb(words[3], type_opword);
+    if (w && !strcmp("replica", w)) {
+        if (wordcount < 6) {
+            ret = -1;
+            goto out;
+        }
+        count = strtol(words[4], NULL, 0);
+        if (count < 1) {
+            cli_err(
+                "replica count should be greater than 0 in "
+                "case of remove-brick");
+            ret = -1;
+            goto out;
+        }
+
+        if (count == 2) {
+            if (strcmp(words[wordcount - 1], "force")) {
+                ques =
+                    "Replica 2 volumes are prone to "
+                    "split-brain. Use Arbiter or Replica 3 "
+                    "to avoid this. See: "
+                    "http://docs.gluster.org/en/latest/Administrator%20Guide/"
+                    "Split%20brain%20and%20ways%20to%20deal%20with%20it/."
+                    "\nDo you still want to continue?\n";
+                answer = cli_cmd_get_confirmation(state, ques);
+                if (GF_ANSWER_NO == answer) {
+                    gf_log("cli", GF_LOG_ERROR,
+                           "Remove "
+                           "brick cancelled, exiting");
+                    ret = -1;
+                    goto out;
+                }
+            }
         }
 
-        volname = (char *)words[2];
+        ret = dict_set_int32(dict, "replica-count", count);
+        if (ret)
+            goto out;
+        brick_index = 5;
+    } else if (w) {
+        GF_ASSERT(!"opword mismatch");
+    }
+
+    w = str_getunamb(words[wordcount - 1], opwords);
+    if (!w) {
+        ret = -1;
+        goto out;
+    } else {
+        /* handled this option */
+        wordcount--;
+        if (!strcmp("start", w)) {
+            command = GF_OP_CMD_START;
+            if (question)
+                *question = 1;
+        } else if (!strcmp("commit", w)) {
+            command = GF_OP_CMD_COMMIT;
+        } else if (!strcmp("stop", w)) {
+            command = GF_OP_CMD_STOP;
+        } else if (!strcmp("status", w)) {
+            command = GF_OP_CMD_STATUS;
+        } else if (!strcmp("force", w)) {
+            command = GF_OP_CMD_COMMIT_FORCE;
+            if (question)
+                *question = 1;
+        } else {
+            GF_ASSERT(!"opword mismatch");
+            ret = -1;
+            goto out;
+        }
+    }
 
-        GF_ASSERT (volname);
+    ret = dict_set_int32(dict, "command", command);
+    if (ret)
+        gf_log("cli", GF_LOG_INFO, "failed to set 'command' %d", command);
 
-        ret = cli_cmd_validate_volume (volname);
-        if (ret) {
-                gf_log ("cli", GF_LOG_ERROR, "Failed to validate volume name");
-                goto out;
-        }
+    tmp_index = brick_index;
+    tmp_brick = GF_MALLOC(2048 * sizeof(*tmp_brick), gf_common_mt_char);
 
-        ret = dict_set_str (dict, "volname", volname);
+    if (!tmp_brick) {
+        gf_log("", GF_LOG_ERROR,
+               "cli_cmd_volume_remove_brick_parse: "
+               "Unable to get memory");
+        ret = -1;
+        goto out;
+    }
 
-        if (ret)
-                goto out;
+    tmp_brick1 = GF_MALLOC(2048 * sizeof(*tmp_brick1), gf_common_mt_char);
 
-        volname = (char *)words[2];
-        if (wordcount == 4) {
-                if (!strcmp(words[3], "status"))
-                        command = GF_DEFRAG_CMD_STATUS_TIER;
-                else if (!strcmp(words[3], "start"))
-                        command = GF_DEFRAG_CMD_START_TIER;
-                else {
-                        ret = -1;
-                        goto out;
-                }
-        } else if (wordcount == 5) {
-                if ((!strcmp (words[3], "start")) &&
-                    (!strcmp (words[4], "force"))) {
-                        command = GF_DEFRAG_CMD_START_TIER;
-                        is_force = 1;
-                        ret = dict_set_int32 (dict, "force", is_force);
-                        if (ret)
-                                goto out;
-                } else {
-                        ret = -1;
-                        goto out;
-                }
+    if (!tmp_brick1) {
+        gf_log("", GF_LOG_ERROR,
+               "cli_cmd_volume_remove_brick_parse: "
+               "Unable to get memory");
+        ret = -1;
+        goto out;
+    }
+
+    while (brick_index < wordcount) {
+        if (validate_brick_name((char *)words[brick_index])) {
+            cli_err(
+                "wrong brick type: %s, use <HOSTNAME>:"
+                "<export-dir-abs-path>",
+                words[brick_index]);
+            ret = -1;
+            goto out;
         } else {
+            delimiter = strrchr(words[brick_index], ':');
+            ret = gf_canonicalize_path(delimiter + 1);
+            if (ret)
+                goto out;
+        }
+
+        j = tmp_index;
+        strcpy(tmp_brick, words[brick_index]);
+        while (j < brick_index) {
+            strcpy(tmp_brick1, words[j]);
+            if (!(strcmp(tmp_brick, tmp_brick1))) {
+                gf_log("", GF_LOG_ERROR,
+                       "Duplicate bricks"
+                       " found %s",
+                       words[brick_index]);
+                cli_err("Duplicate bricks found %s", words[brick_index]);
                 ret = -1;
                 goto out;
+            }
+            j++;
         }
+        snprintf(key, 50, "brick%d", ++(*brick_count));
+        ret = dict_set_str(dict, key, (char *)words[brick_index++]);
 
-        ret = dict_set_int32 (dict, "rebalance-command", command);
         if (ret)
-                goto out;
+            goto out;
+    }
+
+    if (command != GF_OP_CMD_STATUS && command != GF_OP_CMD_STOP) {
+        ret = dict_set_int32(dict, "count", *brick_count);
+        if (ret)
+            goto out;
+    }
+
+    *options = dict;
 
-        *options = dict;
 out:
+    if (ret) {
+        gf_log("cli", GF_LOG_ERROR, "Unable to parse remove-brick CLI");
+        if (dict)
+            dict_unref(dict);
+    }
 
-        if (ret) {
-                gf_log ("cli", GF_LOG_ERROR, "Unable to parse tier CLI");
-                if (dict)
-                        dict_destroy (dict);
-        }
+    GF_FREE(tmp_brick);
+    GF_FREE(tmp_brick1);
 
-        return ret;
+    *comm = command;
+
+    return ret;
 }
 
 int32_t
-cli_cmd_volume_detach_tier_parse (const char **words, int wordcount,
-                                  dict_t **options, int *question)
+cli_cmd_brick_op_validate_bricks(const char **words, dict_t *dict, int src,
+                                 int dst)
 {
-        int      ret = -1;
-        char    *word = NULL;
-        dict_t  *dict = NULL;
-        int32_t  command = GF_OP_CMD_NONE;
-        int      force = 0;
-
-        dict = dict_new ();
-        if (!dict)
-                goto out;
-
-        ret = dict_set_str (dict, "volname", (char *)words[2]);
+    int ret = -1;
+    char *delimiter = NULL;
+
+    if (validate_brick_name((char *)words[src])) {
+        cli_err(
+            "wrong brick type: %s, use "
+            "<HOSTNAME>:<export-dir-abs-path>",
+            words[3]);
+        ret = -1;
+        goto out;
+    } else {
+        delimiter = strrchr((char *)words[src], '/');
+        ret = gf_canonicalize_path(delimiter);
         if (ret)
-                goto out;
+            goto out;
+    }
 
-        if (wordcount == 3 && !strcmp ((char *)words[2], "help")) {
-                return -1;
-        }
-
-        if (wordcount != 4) {
-                ret = -1;
-                goto out;
-        }
+    ret = dict_set_str(dict, "src-brick", (char *)words[src]);
+    if (ret)
+        goto out;
 
-        word = (char *)words[3];
+    if (dst == -1) {
+        ret = 0;
+        goto out;
+    }
 
+    if (validate_brick_name((char *)words[dst])) {
+        cli_err(
+            "wrong brick type: %s, use "
+            "<HOSTNAME>:<export-dir-abs-path>",
+            words[dst]);
         ret = -1;
-
-        if (!strcmp(word, "start")) {
-                command = GF_OP_CMD_DETACH_START;
-        } else if (!strcmp(word, "commit")) {
-                *question = 1;
-                command = GF_OP_CMD_DETACH_COMMIT;
-        } else if (!strcmp(word, "force")) {
-                *question = 1;
-                command = GF_OP_CMD_DETACH_COMMIT_FORCE;
-        } else if (!strcmp(word, "stop"))
-                command = GF_OP_CMD_STOP_DETACH_TIER;
-        else if (!strcmp(word, "status"))
-                command = GF_OP_CMD_STATUS;
-        else
-                goto out;
-
-        ret = dict_set_int32 (dict, "command", command);
+        goto out;
+    } else {
+        delimiter = strrchr((char *)words[dst], '/');
+        ret = gf_canonicalize_path(delimiter);
         if (ret)
-                goto out;
+            goto out;
+    }
 
-        *options = dict;
-        ret = 0;
+    ret = dict_set_str(dict, "dst-brick", (char *)words[dst]);
+    if (ret)
+        goto out;
+    ret = 0;
 out:
-        if (ret) {
-                gf_log ("cli", GF_LOG_ERROR, "Unable to parse detach-tier CLI");
-                if (dict)
-                        dict_unref (dict);
-        }
-
-        return ret;
+    return ret;
 }
 
 int32_t
-cli_cmd_volume_remove_brick_parse (const char **words, int wordcount,
-                                   dict_t **options, int *question)
+cli_cmd_volume_reset_brick_parse(const char **words, int wordcount,
+                                 dict_t **options)
 {
-        dict_t  *dict = NULL;
-        char    *volname = NULL;
-        char    *delimiter = NULL;
-        int     ret = -1;
-        char    key[50];
-        int     brick_count = 0, brick_index = 0;
-        int32_t tmp_index = 0;
-        int32_t j = 0;
-        char    *tmp_brick = NULL;
-        char    *tmp_brick1 = NULL;
-        char    *type_opword[] = { "replica", NULL };
-        char    *opwords[] = { "start", "commit", "stop", "status",
-                               "force", NULL };
-        char    *w = NULL;
-        int32_t  command = GF_OP_CMD_NONE;
-        long     count = 0;
-
-        GF_ASSERT (words);
-        GF_ASSERT (options);
-
-        if (wordcount < 5)
-                goto out;
+    int ret = -1;
+    char *volname = NULL;
+    dict_t *dict = NULL;
 
-        dict = dict_new ();
-        if (!dict)
-                goto out;
+    if (wordcount < 5 || wordcount > 7)
+        goto out;
 
-        volname = (char *)words[2];
+    dict = dict_new();
 
-        GF_ASSERT (volname);
+    if (!dict)
+        goto out;
 
-        ret = dict_set_str (dict, "volname", volname);
-        if (ret)
-                goto out;
+    volname = (char *)words[2];
 
-        brick_index = 3;
-        w = str_getunamb (words[3], type_opword);
-        if (w && !strcmp ("replica", w)) {
-                if (wordcount < 6) {
-                        ret = -1;
-                        goto out;
-                }
-                count = strtol (words[4], NULL, 0);
-                if (count < 1) {
-                        cli_err ("replica count should be greater than 0 in "
-                                 "case of remove-brick");
-                        ret = -1;
-                        goto out;
-                }
+    ret = dict_set_str(dict, "volname", volname);
+    if (ret)
+        goto out;
 
-                ret = dict_set_int32 (dict, "replica-count", count);
-                if (ret)
-                        goto out;
-                brick_index = 5;
-        } else if (w) {
-                GF_ASSERT (!"opword mismatch");
+    if (wordcount == 5) {
+        if (strcmp(words[4], "start")) {
+            cli_err(
+                "Invalid option '%s' for reset-brick. Please "
+                "enter valid reset-brick command",
+                words[4]);
+            ret = -1;
+            goto out;
         }
 
-        w = str_getunamb (words[wordcount - 1], opwords);
-        if (!w) {
-                ret = -1;
-                goto out;
-        } else {
-                /* handled this option */
-                wordcount--;
-                if (!strcmp ("start", w)) {
-                        command = GF_OP_CMD_START;
-                } else if (!strcmp ("commit", w)) {
-                        command = GF_OP_CMD_COMMIT;
-                        if (question)
-                                *question = 1;
-                } else if (!strcmp ("stop", w)) {
-                        command = GF_OP_CMD_STOP;
-                } else if (!strcmp ("status", w)) {
-                        command = GF_OP_CMD_STATUS;
-                } else if (!strcmp ("force", w)) {
-                        command = GF_OP_CMD_COMMIT_FORCE;
-                        if (question)
-                                *question = 1;
-                } else {
-                        GF_ASSERT (!"opword mismatch");
-                        ret = -1;
-                        goto out;
-                }
-        }
-
-        ret = dict_set_int32 (dict, "command", command);
+        ret = cli_cmd_brick_op_validate_bricks(words, dict, 3, -1);
         if (ret)
-                gf_log ("cli", GF_LOG_INFO, "failed to set 'command' %d",
-                        command);
-
-
-        tmp_index = brick_index;
-        tmp_brick = GF_MALLOC(2048 * sizeof(*tmp_brick), gf_common_mt_char);
-
-        if (!tmp_brick) {
-                gf_log ("",GF_LOG_ERROR,"cli_cmd_volume_remove_brick_parse: "
-                        "Unable to get memory");
-                ret = -1;
-                goto out;
-        }
-
-        tmp_brick1 = GF_MALLOC(2048 * sizeof(*tmp_brick1), gf_common_mt_char);
-
-        if (!tmp_brick1) {
-                gf_log ("",GF_LOG_ERROR,"cli_cmd_volume_remove_brick_parse: "
-                        "Unable to get memory");
-                ret = -1;
-                goto out;
-        }
-
-        while (brick_index < wordcount) {
-                if (validate_brick_name ((char *)words[brick_index])) {
-                        cli_err ("wrong brick type: %s, use <HOSTNAME>:"
-                                 "<export-dir-abs-path>", words[brick_index]);
-                        ret = -1;
-                        goto out;
-                } else {
-                        delimiter = strrchr(words[brick_index], ':');
-                        ret = gf_canonicalize_path (delimiter + 1);
-                        if (ret)
-                                goto out;
-                }
+            goto out;
 
-                j = tmp_index;
-                strcpy(tmp_brick, words[brick_index]);
-                while ( j < brick_index) {
-                        strcpy(tmp_brick1, words[j]);
-                        if (!(strcmp (tmp_brick, tmp_brick1))) {
-                                gf_log("",GF_LOG_ERROR, "Duplicate bricks"
-                                       " found %s", words[brick_index]);
-                                cli_err("Duplicate bricks found %s",
-                                        words[brick_index]);
-                                ret = -1;
-                                goto out;
-                        }
-                        j++;
-                }
-                snprintf (key, 50, "brick%d", ++brick_count);
-                ret = dict_set_str (dict, key, (char *)words[brick_index++]);
+        ret = dict_set_str(dict, "operation", "GF_RESET_OP_START");
+        if (ret)
+            goto out;
+    } else if (wordcount == 6) {
+        if (strcmp(words[5], "commit")) {
+            cli_err(
+                "Invalid option '%s' for reset-brick. Please "
+                "enter valid reset-brick command",
+                words[5]);
+            ret = -1;
+            goto out;
+        }
+
+        ret = cli_cmd_brick_op_validate_bricks(words, dict, 3, 4);
+        if (ret)
+            goto out;
 
-                if (ret)
-                        goto out;
-        }
+        ret = dict_set_str(dict, "operation", "GF_RESET_OP_COMMIT");
+        if (ret)
+            goto out;
+    } else if (wordcount == 7) {
+        if (strcmp(words[5], "commit") || strcmp(words[6], "force")) {
+            cli_err(
+                "Invalid option '%s %s' for reset-brick. Please "
+                "enter valid reset-brick command",
+                words[5], words[6]);
+            ret = -1;
+            goto out;
+        }
+
+        ret = cli_cmd_brick_op_validate_bricks(words, dict, 3, 4);
+        if (ret)
+            goto out;
 
-        if (command != GF_OP_CMD_STATUS && command != GF_OP_CMD_STOP) {
-                ret = dict_set_int32 (dict, "count", brick_count);
-                if (ret)
-                        goto out;
-        }
+        ret = dict_set_str(dict, "operation", "GF_RESET_OP_COMMIT_FORCE");
+        if (ret)
+            goto out;
+    }
 
-        *options = dict;
+    *options = dict;
 
 out:
-        if (ret) {
-                gf_log ("cli", GF_LOG_ERROR, "Unable to parse remove-brick CLI");
-                if (dict)
-                        dict_destroy (dict);
-        }
+    if (ret) {
+        gf_log("cli", GF_LOG_ERROR, "Unable to parse reset-brick CLI");
+        if (dict)
+            dict_unref(dict);
+    }
 
-        GF_FREE (tmp_brick);
-        GF_FREE (tmp_brick1);
-
-        return ret;
+    return ret;
 }
 
-
 int32_t
-cli_cmd_volume_replace_brick_parse (const char **words, int wordcount,
-                                    dict_t **options)
+cli_cmd_volume_replace_brick_parse(const char **words, int wordcount,
+                                   dict_t **options)
 {
-        int                   ret        = -1;
-        char                 *volname    = NULL;
-        char                 *delimiter  = NULL;
-        dict_t               *dict       = NULL;
+    int ret = -1;
+    char *volname = NULL;
+    dict_t *dict = NULL;
 
-        GF_ASSERT (words);
-        GF_ASSERT (options);
+    GF_ASSERT(words);
+    GF_ASSERT(options);
 
-        if (wordcount != 7) {
-                ret = -1;
-                goto out;
-        }
-
-        dict = dict_new ();
-
-        if (!dict) {
-                gf_log ("cli", GF_LOG_ERROR, "Failed to allocate dictionary");
-                goto out;
-        }
+    if (wordcount != 7) {
+        ret = -1;
+        goto out;
+    }
 
-        volname = (char *)words[2];
+    dict = dict_new();
 
-        GF_ASSERT (volname);
+    if (!dict) {
+        gf_log("cli", GF_LOG_ERROR, "Failed to allocate dictionary");
+        goto out;
+    }
 
-        ret = dict_set_str (dict, "volname", volname);
-        if (ret)
-                goto out;
+    volname = (char *)words[2];
 
-        if (validate_brick_name ((char *)words[3])) {
-                cli_err ("wrong brick type: %s, use "
-                         "<HOSTNAME>:<export-dir-abs-path>", words[3]);
-                ret = -1;
-                goto out;
-        } else {
-                delimiter = strrchr ((char *)words[3], ':');
-                ret = gf_canonicalize_path (delimiter + 1);
-                if (ret)
-                        goto out;
-        }
+    GF_ASSERT(volname);
 
-        ret = dict_set_str (dict, "src-brick", (char *)words[3]);
-        if (ret)
-                goto out;
+    ret = dict_set_str(dict, "volname", volname);
+    if (ret)
+        goto out;
 
-        if (validate_brick_name ((char *)words[4])) {
-                cli_err ("wrong brick type: %s, use "
-                         "<HOSTNAME>:<export-dir-abs-path>", words[4]);
-                ret = -1;
-                goto out;
-        } else {
-                delimiter = strrchr ((char *)words[4], ':');
-                ret = gf_canonicalize_path (delimiter + 1);
-                if (ret)
-                        goto out;
-        }
+    ret = cli_cmd_brick_op_validate_bricks(words, dict, 3, 4);
+    if (ret)
+        goto out;
 
-        ret = dict_set_str (dict, "dst-brick", (char *)words[4]);
-        if (ret)
-                goto out;
+    /* commit force option */
+    if (strcmp("commit", words[5]) || strcmp("force", words[6])) {
+        cli_err(
+            "Invalid option '%s' '%s' for replace-brick. Please "
+            "enter valid replace-brick command",
+            words[5], words[6]);
+        ret = -1;
+        goto out;
+    }
 
-        /* commit force option */
-        if (strcmp ("commit", words[5]) || strcmp ("force", words[6])) {
-                cli_err ("Invalid option '%s' '%s' for replace-brick. Please "
-                         "enter valid replace-brick command", words[5],
-                         words[6]);
-                ret = -1;
-                goto out;
-        }
+    ret = dict_set_str(dict, "operation", "GF_REPLACE_OP_COMMIT_FORCE");
+    if (ret)
+        goto out;
 
-        ret = dict_set_str (dict, "operation", "GF_REPLACE_OP_COMMIT_FORCE");
-        if (ret)
-                goto out;
-
-        *options = dict;
+    *options = dict;
 
 out:
-        if (ret) {
-                gf_log ("cli", GF_LOG_ERROR, "Unable to parse replace-brick CLI");
-                if (dict)
-                        dict_destroy (dict);
-        }
+    if (ret) {
+        gf_log("cli", GF_LOG_ERROR, "Unable to parse reset-brick CLI");
+        if (dict)
+            dict_unref(dict);
+    }
 
-        return ret;
+    return ret;
 }
 
 int32_t
-cli_cmd_log_filename_parse (const char **words, int wordcount, dict_t **options)
+cli_cmd_log_filename_parse(const char **words, int wordcount, dict_t **options)
 {
-        dict_t  *dict = NULL;
-        char    *volname = NULL;
-        char    *str = NULL;
-        int     ret = -1;
-        char    *delimiter = NULL;
+    dict_t *dict = NULL;
+    char *volname = NULL;
+    char *str = NULL;
+    int ret = -1;
+    char *delimiter = NULL;
 
-        GF_ASSERT (words);
-        GF_ASSERT (options);
+    GF_ASSERT(words);
+    GF_ASSERT(options);
 
-        dict = dict_new ();
-        if (!dict)
-                goto out;
+    dict = dict_new();
+    if (!dict)
+        goto out;
 
-        volname = (char *)words[3];
-        GF_ASSERT (volname);
+    volname = (char *)words[3];
+    GF_ASSERT(volname);
 
-        ret = dict_set_str (dict, "volname", volname);
-        if (ret)
-                goto out;
+    ret = dict_set_str(dict, "volname", volname);
+    if (ret)
+        goto out;
 
-        str = (char *)words[4];
-        if (strchr (str, ':')) {
-                delimiter = strchr (words[4], ':');
-                if (!delimiter || delimiter == words[4]
-                    || *(delimiter+1) != '/') {
-                        cli_err ("wrong brick type: %s, use <HOSTNAME>:"
-                                 "<export-dir-abs-path>", words[4]);
-                        ret = -1;
-                        goto out;
-                } else {
-                        ret = gf_canonicalize_path (delimiter + 1);
-                        if (ret)
-                                goto out;
-                }
-                ret = dict_set_str (dict, "brick", str);
-                if (ret)
-                        goto out;
-                /* Path */
-                str = (char *)words[5];
-                ret = dict_set_str (dict, "path", str);
-                if (ret)
-                        goto out;
+    str = (char *)words[4];
+    if (strchr(str, ':')) {
+        delimiter = strchr(words[4], ':');
+        if (!delimiter || delimiter == words[4] || *(delimiter + 1) != '/') {
+            cli_err(
+                "wrong brick type: %s, use <HOSTNAME>:"
+                "<export-dir-abs-path>",
+                words[4]);
+            ret = -1;
+            goto out;
         } else {
-                ret = dict_set_str (dict, "path", str);
-                if (ret)
-                        goto out;
+            ret = gf_canonicalize_path(delimiter + 1);
+            if (ret)
+                goto out;
         }
+        ret = dict_set_str(dict, "brick", str);
+        if (ret)
+            goto out;
+        /* Path */
+        str = (char *)words[5];
+        ret = dict_set_str(dict, "path", str);
+        if (ret)
+            goto out;
+    } else {
+        ret = dict_set_str(dict, "path", str);
+        if (ret)
+            goto out;
+    }
 
-        *options = dict;
+    *options = dict;
 
 out:
-        if (ret && dict)
-                dict_destroy (dict);
+    if (ret && dict)
+        dict_unref(dict);
 
-        return ret;
+    return ret;
 }
 
 int32_t
-cli_cmd_log_level_parse (const char **words, int worcount, dict_t **options)
+cli_cmd_log_level_parse(const char **words, int worcount, dict_t **options)
 {
-        dict_t *dict            = NULL;
-        int     ret             = -1;
-
-        GF_ASSERT (words);
-        GF_ASSERT (options);
-
-        /*
-         * loglevel command format:
-         *  > volume log level <VOL> <XLATOR[*]> <LOGLEVEL>
-         *  > volume log level colon-o posix WARNING
-         *  > volume log level colon-o replicate* DEBUG
-         *  > volume log level coon-o * TRACE
-         */
-
-        GF_ASSERT ((strncmp(words[0], "volume", 6) == 0));
-        GF_ASSERT ((strncmp(words[1], "log", 3) == 0));
-        GF_ASSERT ((strncmp(words[2], "level", 5) == 0));
-
-        ret = glusterd_check_log_level(words[5]);
-        if (ret == -1) {
-                cli_err("Invalid log level [%s] specified", words[5]);
-                cli_err("Valid values for loglevel: (DEBUG|WARNING|ERROR"
-                        "|CRITICAL|NONE|TRACE)");
-                goto out;
-        }
+    dict_t *dict = NULL;
+    int ret = -1;
+
+    GF_ASSERT(words);
+    GF_ASSERT(options);
+
+    /*
+     * loglevel command format:
+     *  > volume log level <VOL> <XLATOR[*]> <LOGLEVEL>
+     *  > volume log level colon-o posix WARNING
+     *  > volume log level colon-o replicate* DEBUG
+     *  > volume log level coon-o * TRACE
+     */
+
+    GF_ASSERT((strncmp(words[0], "volume", 6) == 0));
+    GF_ASSERT((strncmp(words[1], "log", 3) == 0));
+    GF_ASSERT((strncmp(words[2], "level", 5) == 0));
+
+    ret = glusterd_check_log_level(words[5]);
+    if (ret == -1) {
+        cli_err("Invalid log level [%s] specified", words[5]);
+        cli_err(
+            "Valid values for loglevel: (DEBUG|WARNING|ERROR"
+            "|CRITICAL|NONE|TRACE)");
+        goto out;
+    }
 
-        dict = dict_new ();
-        if (!dict)
-                goto out;
+    dict = dict_new();
+    if (!dict)
+        goto out;
 
-        GF_ASSERT(words[3]);
-        GF_ASSERT(words[4]);
+    GF_ASSERT(words[3]);
+    GF_ASSERT(words[4]);
 
-        ret = dict_set_str (dict, "volname", (char *)words[3]);
-        if (ret)
-                goto out;
+    ret = dict_set_str(dict, "volname", (char *)words[3]);
+    if (ret)
+        goto out;
 
-        ret = dict_set_str (dict, "xlator", (char *)words[4]);
-        if (ret)
-                goto out;
+    ret = dict_set_str(dict, "xlator", (char *)words[4]);
+    if (ret)
+        goto out;
 
-        ret = dict_set_str (dict, "loglevel", (char *)words[5]);
-        if (ret)
-                goto out;
+    ret = dict_set_str(dict, "loglevel", (char *)words[5]);
+    if (ret)
+        goto out;
 
-        *options = dict;
+    *options = dict;
 
- out:
-        if (ret && dict)
-                dict_destroy (dict);
+out:
+    if (ret && dict)
+        dict_unref(dict);
 
-        return ret;
+    return ret;
 }
 
 int32_t
-cli_cmd_log_locate_parse (const char **words, int wordcount, dict_t **options)
+cli_cmd_log_locate_parse(const char **words, int wordcount, dict_t **options)
 {
-        dict_t  *dict = NULL;
-        char    *volname = NULL;
-        char    *str = NULL;
-        int     ret = -1;
-        char    *delimiter = NULL;
+    dict_t *dict = NULL;
+    char *volname = NULL;
+    char *str = NULL;
+    int ret = -1;
+    char *delimiter = NULL;
 
-        GF_ASSERT (words);
-        GF_ASSERT (options);
+    GF_ASSERT(words);
+    GF_ASSERT(options);
 
-        dict = dict_new ();
-        if (!dict)
-                goto out;
+    dict = dict_new();
+    if (!dict)
+        goto out;
 
-        volname = (char *)words[3];
-        GF_ASSERT (volname);
+    volname = (char *)words[3];
+    GF_ASSERT(volname);
 
-        ret = dict_set_str (dict, "volname", volname);
-        if (ret)
-                goto out;
+    ret = dict_set_str(dict, "volname", volname);
+    if (ret)
+        goto out;
 
-        if (words[4]) {
-                delimiter = strchr (words[4], ':');
-                if (!delimiter || delimiter == words[4]
-                    || *(delimiter+1) != '/') {
-                        cli_err ("wrong brick type: %s, use <HOSTNAME>:"
-                                 "<export-dir-abs-path>", words[4]);
-                        ret = -1;
-                        goto out;
-                } else {
-                        ret = gf_canonicalize_path (delimiter + 1);
-                        if (ret)
-                                goto out;
-                }
-                str = (char *)words[4];
-                ret = dict_set_str (dict, "brick", str);
-                if (ret)
-                        goto out;
+    if (words[4]) {
+        delimiter = strchr(words[4], ':');
+        if (!delimiter || delimiter == words[4] || *(delimiter + 1) != '/') {
+            cli_err(
+                "wrong brick type: %s, use <HOSTNAME>:"
+                "<export-dir-abs-path>",
+                words[4]);
+            ret = -1;
+            goto out;
+        } else {
+            ret = gf_canonicalize_path(delimiter + 1);
+            if (ret)
+                goto out;
         }
+        str = (char *)words[4];
+        ret = dict_set_str(dict, "brick", str);
+        if (ret)
+            goto out;
+    }
 
-        *options = dict;
+    *options = dict;
 
 out:
-        if (ret && dict)
-                dict_destroy (dict);
+    if (ret && dict)
+        dict_unref(dict);
 
-        return ret;
+    return ret;
 }
 
 int32_t
-cli_cmd_log_rotate_parse (const char **words, int wordcount, dict_t **options)
+cli_cmd_log_rotate_parse(const char **words, int wordcount, dict_t **options)
 {
-        dict_t  *dict = NULL;
-        char    *volname = NULL;
-        char    *str = NULL;
-        int     ret = -1;
-        char    *delimiter = NULL;
+    dict_t *dict = NULL;
+    char *volname = NULL;
+    char *str = NULL;
+    int ret = -1;
+    char *delimiter = NULL;
 
-        GF_ASSERT (words);
-        GF_ASSERT (options);
+    GF_ASSERT(words);
+    GF_ASSERT(options);
 
-        dict = dict_new ();
-        if (!dict)
-                goto out;
+    dict = dict_new();
+    if (!dict)
+        goto out;
 
-        if (strcmp ("rotate", words[3]) == 0)
-                volname = (char *)words[2];
-        else if (strcmp ("rotate", words[2]) == 0)
-                volname = (char *)words[3];
-        GF_ASSERT (volname);
+    if (strcmp("rotate", words[3]) == 0)
+        volname = (char *)words[2];
+    GF_ASSERT(volname);
 
-        ret = dict_set_str (dict, "volname", volname);
-        if (ret)
-                goto out;
+    ret = dict_set_str(dict, "volname", volname);
+    if (ret)
+        goto out;
 
-        if (words[4]) {
-                delimiter = strchr (words[4], ':');
-                if (!delimiter || delimiter == words[4]
-                    || *(delimiter+1) != '/') {
-                        cli_err ("wrong brick type: %s, use <HOSTNAME>:"
-                                 "<export-dir-abs-path>", words[4]);
-                        ret = -1;
-                        goto out;
-                } else {
-                        ret = gf_canonicalize_path (delimiter + 1);
-                        if (ret)
-                                goto out;
-                }
-                str = (char *)words[4];
-                ret = dict_set_str (dict, "brick", str);
-                if (ret)
-                        goto out;
+    if (words[4]) {
+        delimiter = strchr(words[4], ':');
+        if (!delimiter || delimiter == words[4] || *(delimiter + 1) != '/') {
+            cli_err(
+                "wrong brick type: %s, use <HOSTNAME>:"
+                "<export-dir-abs-path>",
+                words[4]);
+            ret = -1;
+            goto out;
+        } else {
+            ret = gf_canonicalize_path(delimiter + 1);
+            if (ret)
+                goto out;
         }
+        str = (char *)words[4];
+        ret = dict_set_str(dict, "brick", str);
+        if (ret)
+            goto out;
+    }
 
-        *options = dict;
+    *options = dict;
 
 out:
-        if (ret && dict)
-                dict_destroy (dict);
+    if (ret && dict)
+        dict_unref(dict);
 
-        return ret;
+    return ret;
 }
 
 static gf_boolean_t
-gsyncd_url_check (const char *w)
+gsyncd_url_check(const char *w)
 {
-        return !!strpbrk (w, ":/");
+    return !!strpbrk(w, ":/");
 }
 
 static gf_boolean_t
-gsyncd_glob_check (const char *w)
+valid_slave_gsyncd_url(const char *w)
 {
-        return !!strpbrk (w, "*?[");
+    if (strstr(w, ":::"))
+        return _gf_false;
+    else if (strstr(w, "::"))
+        return _gf_true;
+    else
+        return _gf_false;
 }
 
-static int
-config_parse (const char **words, int wordcount, dict_t *dict,
-              unsigned cmdi, unsigned glob)
+static gf_boolean_t
+gsyncd_glob_check(const char *w)
 {
-        int32_t            ret     = -1;
-        int32_t            i       = -1;
-        char               *append_str = NULL;
-        size_t             append_len = 0;
-        char               *subop = NULL;
+    return !!strpbrk(w, "*?[");
+}
 
-        switch ((wordcount - 1) - cmdi) {
+static int
+config_parse(const char **words, int wordcount, dict_t *dict, unsigned cmdi,
+             unsigned glob)
+{
+    int32_t ret = -1;
+    int32_t i = -1;
+    char *append_str = NULL;
+    size_t append_len = 0;
+    char *subop = NULL;
+    char *ret_chkpt = NULL;
+    struct tm checkpoint_time;
+    char chkpt_buf[20] = "";
+
+    switch ((wordcount - 1) - cmdi) {
         case 0:
-                subop = gf_strdup ("get-all");
-                break;
+            subop = gf_strdup("get-all");
+            break;
         case 1:
-                if (words[cmdi + 1][0] == '!') {
-                        (words[cmdi + 1])++;
-                        if (gf_asprintf (&subop, "del%s",
-                                         glob ? "-glob" : "") == -1)
-                                subop = NULL;
-                } else
-                        subop = gf_strdup ("get");
+            if (words[cmdi + 1][0] == '!') {
+                (words[cmdi + 1])++;
+                if (gf_asprintf(&subop, "del%s", glob ? "-glob" : "") == -1)
+                    subop = NULL;
+            } else
+                subop = gf_strdup("get");
 
-                ret = dict_set_str (dict, "op_name", ((char *)words[cmdi + 1]));
-                if (ret < 0)
-                        goto out;
-                break;
+            ret = dict_set_str(dict, "op_name", ((char *)words[cmdi + 1]));
+            if (ret < 0)
+                goto out;
+            break;
         default:
-                if (gf_asprintf (&subop, "set%s", glob ? "-glob" : "") == -1)
-                        subop = NULL;
+            if (gf_asprintf(&subop, "set%s", glob ? "-glob" : "") == -1)
+                subop = NULL;
 
-                ret = dict_set_str (dict, "op_name", ((char *)words[cmdi + 1]));
-                if (ret < 0)
-                        goto out;
+            ret = dict_set_str(dict, "op_name", ((char *)words[cmdi + 1]));
+            if (ret < 0)
+                goto out;
 
-                /* join the varargs by spaces to get the op_value */
+            /* join the varargs by spaces to get the op_value */
 
-                for (i = cmdi + 2; i < wordcount; i++)
-                        append_len += (strlen (words[i]) + 1);
-                /* trailing strcat will add two bytes, make space for that */
-                append_len++;
+            for (i = cmdi + 2; i < wordcount; i++)
+                append_len += (strlen(words[i]) + 1);
+            /* trailing strcat will add two bytes, make space for that */
+            append_len++;
 
-                append_str = GF_CALLOC (1, append_len, cli_mt_append_str);
-                if (!append_str) {
-                        ret = -1;
-                        goto out;
-                }
+            /* strcat is used on this allocation and hence expected to be
+             * initiatlized to 0. So GF_CALLOC is used.
+             */
+            append_str = GF_CALLOC(1, append_len, cli_mt_append_str);
+            if (!append_str) {
+                ret = -1;
+                goto out;
+            }
+
+            for (i = cmdi + 2; i < wordcount; i++) {
+                strcat(append_str, words[i]);
+                strcat(append_str, " ");
+            }
+            append_str[append_len - 2] = '\0';
+            /* "checkpoint now" is special: we resolve that "now" */
+            if ((strcmp(words[cmdi + 1], "checkpoint") == 0) &&
+                (strcmp(append_str, "now") == 0)) {
+                struct timeval tv = {
+                    0,
+                };
+
+                ret = gettimeofday(&tv, NULL);
+                if (ret == -1)
+                    goto out;
 
-                for (i = cmdi + 2; i < wordcount; i++) {
-                        strcat (append_str, words[i]);
-                        strcat (append_str, " ");
-                }
-                append_str[append_len - 2] = '\0';
-                /* "checkpoint now" is special: we resolve that "now" */
-                if ((strcmp (words[cmdi + 1], "checkpoint") == 0) &&
-                    (strcmp (append_str, "now") == 0)) {
-                        struct timeval tv = {0,};
-
-                        ret = gettimeofday (&tv, NULL);
-                        if (ret == -1)
-                                goto out;
-
-                        GF_FREE (append_str);
-                        append_str = GF_CALLOC (1, 300, cli_mt_append_str);
-                        if (!append_str) {
-                                ret = -1;
-                                goto out;
-                        }
-                        snprintf (append_str, 300, "%" GF_PRI_SECOND,
-                                  tv.tv_sec);
+                GF_FREE(append_str);
+                append_str = GF_MALLOC(300, cli_mt_append_str);
+                if (!append_str) {
+                    ret = -1;
+                    goto out;
+                }
+                snprintf(append_str, 300, "%" GF_PRI_SECOND, tv.tv_sec);
+            } else if ((strcmp(words[cmdi + 1], "checkpoint") == 0) &&
+                       (strcmp(append_str, "now") != 0)) {
+                memset(&checkpoint_time, 0, sizeof(struct tm));
+                ret_chkpt = strptime(append_str, "%Y-%m-%d %H:%M:%S",
+                                     &checkpoint_time);
+
+                if (ret_chkpt == NULL || *ret_chkpt != '\0') {
+                    ret = -1;
+                    cli_err(
+                        "Invalid Checkpoint label. Use format "
+                        "\"Y-m-d H:M:S\", Example: 2016-10-25 15:30:45");
+                    goto out;
+                }
+                GF_FREE(append_str);
+                append_str = GF_MALLOC(300, cli_mt_append_str);
+                if (!append_str) {
+                    ret = -1;
+                    goto out;
                 }
+                strftime(chkpt_buf, sizeof(chkpt_buf), "%s", &checkpoint_time);
+                snprintf(append_str, 300, "%s", chkpt_buf);
+            }
 
-                ret = dict_set_dynstr (dict, "op_value", append_str);
-        }
+            ret = dict_set_dynstr(dict, "op_value", append_str);
+            if (ret != 0) {
+                goto out;
+            }
+            append_str = NULL;
+    }
 
-        ret = -1;
-        if (subop) {
-                ret = dict_set_dynstr (dict, "subop", subop);
-                if (!ret)
-                      subop = NULL;
-        }
+    ret = -1;
+    if (subop) {
+        ret = dict_set_dynstr(dict, "subop", subop);
+        if (!ret)
+            subop = NULL;
+    }
 
 out:
-        if (ret && append_str)
-                GF_FREE (append_str);
-
-        GF_FREE (subop);
+    GF_FREE(append_str);
+    GF_FREE(subop);
 
-        gf_log ("cli", GF_LOG_DEBUG, "Returning %d", ret);
-        return ret;
+    gf_log("cli", GF_LOG_DEBUG, "Returning %d", ret);
+    return ret;
 }
 
 /* ssh_port_parse: Parses and validates when ssh_port is given.
@@ -2426,1389 +2771,1467 @@ out:
  */
 
 static int32_t
-parse_ssh_port (const char **words, int wordcount, dict_t *dict,
-                unsigned *cmdi, int ssh_index, char *type) {
-
-        int        ret         = 0;
-        char      *end_ptr     = NULL;
-        int64_t    limit       = 0;
-
-        if (!strcmp ((char *)words[ssh_index], "ssh-port")) {
-                if (strcmp ((char *)words[ssh_index-1], "create")) {
-                        ret = -1;
-                        goto out;
-                }
-                (*cmdi)++;
-                limit = strtol (words[ssh_index+1], &end_ptr, 10);
-                if (errno == ERANGE || errno == EINVAL || limit <= 0
-                                    || strcmp (end_ptr, "") != 0) {
-                        ret = -1;
-                        cli_err ("Please enter an interger value for ssh_port ");
-                        goto out;
-                }
+parse_ssh_port(const char **words, int wordcount, dict_t *dict, unsigned *cmdi,
+               int ssh_index, char *type)
+{
+    int ret = 0;
+    char *end_ptr = NULL;
+    int64_t limit = 0;
 
-                ret = dict_set_int32 (dict, "ssh_port", limit);
-                if (ret)
-                        goto out;
-                (*cmdi)++;
-        } else if (strcmp ((char *)words[ssh_index+1], "create")) {
-                ret = -1;
-                goto out;
+    if (!strcmp((char *)words[ssh_index], "ssh-port")) {
+        if (strcmp((char *)words[ssh_index - 1], "create")) {
+            ret = -1;
+            goto out;
+        }
+        (*cmdi)++;
+        limit = strtol(words[ssh_index + 1], &end_ptr, 10);
+        if (errno == ERANGE || errno == EINVAL || limit <= 0 ||
+            strcmp(end_ptr, "") != 0) {
+            ret = -1;
+            cli_err("Please enter an integer value for ssh_port ");
+            goto out;
         }
 
-        ret = dict_set_int32 (dict, type, 1);
+        ret = dict_set_int32(dict, "ssh_port", limit);
         if (ret)
-                goto out;
+            goto out;
         (*cmdi)++;
+    } else if (strcmp((char *)words[ssh_index + 1], "create")) {
+        ret = -1;
+        goto out;
+    }
 
- out:
-        return ret;
+    ret = dict_set_int32(dict, type, 1);
+    if (ret)
+        goto out;
+    (*cmdi)++;
+
+out:
+    return ret;
 }
 
 static int32_t
-force_push_pem_no_verify_parse (const char **words, int wordcount,
-                      dict_t *dict, unsigned *cmdi)
+force_push_pem_no_verify_parse(const char **words, int wordcount, dict_t *dict,
+                               unsigned *cmdi)
 {
-        int32_t            ret     = 0;
-
-        if (!strcmp ((char *)words[wordcount-1], "force")) {
-                if ((strcmp ((char *)words[wordcount-2], "start")) &&
-                    (strcmp ((char *)words[wordcount-2], "stop")) &&
-                    (strcmp ((char *)words[wordcount-2], "create")) &&
-                    (strcmp ((char *)words[wordcount-2], "no-verify")) &&
-                    (strcmp ((char *)words[wordcount-2], "push-pem")) &&
-                    (strcmp ((char *)words[wordcount-2], "pause")) &&
-                    (strcmp ((char *)words[wordcount-2], "resume"))) {
-                        ret = -1;
-                        goto out;
-                }
-                ret = dict_set_uint32 (dict, "force",
-                                       _gf_true);
-                if (ret)
-                        goto out;
-                (*cmdi)++;
-
-                if (!strcmp ((char *)words[wordcount-2], "push-pem")) {
-                        ret = parse_ssh_port (words, wordcount, dict, cmdi,
-                                              wordcount-4, "push_pem");
-                        if (ret)
-                                goto out;
-                } else if (!strcmp ((char *)words[wordcount-2], "no-verify")) {
-                        ret = parse_ssh_port (words, wordcount, dict, cmdi,
-                                              wordcount-4, "no_verify");
-                        if (ret)
-                                goto out;
-                }
-        } else if (!strcmp ((char *)words[wordcount-1], "push-pem")) {
-                ret = parse_ssh_port (words, wordcount, dict, cmdi, wordcount-3,
-                                      "push_pem");
-                if (ret)
-                        goto out;
-        } else if (!strcmp ((char *)words[wordcount-1], "no-verify")) {
-                ret = parse_ssh_port (words, wordcount, dict, cmdi, wordcount-3,
-                                      "no_verify");
-                if (ret)
-                        goto out;
+    int32_t ret = 0;
+
+    if (!strcmp((char *)words[wordcount - 1], "force")) {
+        if ((strcmp((char *)words[wordcount - 2], "start")) &&
+            (strcmp((char *)words[wordcount - 2], "stop")) &&
+            (strcmp((char *)words[wordcount - 2], "create")) &&
+            (strcmp((char *)words[wordcount - 2], "no-verify")) &&
+            (strcmp((char *)words[wordcount - 2], "push-pem")) &&
+            (strcmp((char *)words[wordcount - 2], "pause")) &&
+            (strcmp((char *)words[wordcount - 2], "resume"))) {
+            ret = -1;
+            goto out;
+        }
+        ret = dict_set_int32n(dict, "force", SLEN("force"), 1);
+        if (ret)
+            goto out;
+        (*cmdi)++;
+
+        if (!strcmp((char *)words[wordcount - 2], "push-pem")) {
+            ret = parse_ssh_port(words, wordcount, dict, cmdi, wordcount - 4,
+                                 "push_pem");
+            if (ret)
+                goto out;
+        } else if (!strcmp((char *)words[wordcount - 2], "no-verify")) {
+            ret = parse_ssh_port(words, wordcount, dict, cmdi, wordcount - 4,
+                                 "no_verify");
+            if (ret)
+                goto out;
         }
+    } else if (!strcmp((char *)words[wordcount - 1], "push-pem")) {
+        ret = parse_ssh_port(words, wordcount, dict, cmdi, wordcount - 3,
+                             "push_pem");
+        if (ret)
+            goto out;
+    } else if (!strcmp((char *)words[wordcount - 1], "no-verify")) {
+        ret = parse_ssh_port(words, wordcount, dict, cmdi, wordcount - 3,
+                             "no_verify");
+        if (ret)
+            goto out;
+    }
 
 out:
-        gf_log ("cli", GF_LOG_DEBUG, "Returning %d", ret);
-        return ret;
+    gf_log("cli", GF_LOG_DEBUG, "Returning %d", ret);
+    return ret;
 }
 
-
 int32_t
-cli_cmd_gsync_set_parse (const char **words, int wordcount, dict_t **options)
+cli_cmd_gsync_set_parse(struct cli_state *state, const char **words,
+                        int wordcount, dict_t **options, char **errstr)
 {
-        int32_t            ret     = -1;
-        dict_t             *dict   = NULL;
-        gf1_cli_gsync_set  type    = GF_GSYNC_OPTION_TYPE_NONE;
-        int                i       = 0;
-        unsigned           masteri = 0;
-        unsigned           slavei  = 0;
-        unsigned           glob    = 0;
-        unsigned           cmdi    = 0;
-        char               *opwords[] = { "create", "status", "start", "stop",
-                                          "config", "force", "delete",
-                                          "ssh-port", "no-verify", "push-pem",
-                                          "detail", "pause", "resume", NULL };
-        char               *w = NULL;
-        char               *save_ptr   = NULL;
-        char               *slave_temp = NULL;
-        char               *token      = NULL;
-
-        GF_ASSERT (words);
-        GF_ASSERT (options);
-
-        dict = dict_new ();
-        if (!dict)
-                goto out;
-
-        /* new syntax:
-         *
-         * volume geo-replication $m $s create [[ssh-port n] [[no-verify] | [push-pem]]] [force]
-         * volume geo-replication [$m [$s]] status [detail]
-         * volume geo-replication [$m] $s config [[!]$opt [$val]]
-         * volume geo-replication $m $s start|stop [force]
-         * volume geo-replication $m $s delete
-         * volume geo-replication $m $s pause [force]
-         * volume geo-replication $m $s resume [force]
-         */
+    int32_t ret = -1;
+    dict_t *dict = NULL;
+    gf1_cli_gsync_set type = GF_GSYNC_OPTION_TYPE_NONE;
+    int i = 0;
+    unsigned masteri = 0;
+    unsigned slavei = 0;
+    unsigned glob = 0;
+    unsigned cmdi = 0;
+    static char *opwords[] = {"create",    "status",   "start",  "stop",
+                              "config",    "force",    "delete", "ssh-port",
+                              "no-verify", "push-pem", "detail", "pause",
+                              "resume",    NULL};
+    char *w = NULL;
+    char *save_ptr = NULL;
+    char *slave_temp = NULL;
+    char *token = NULL;
+    gf_answer_t answer = GF_ANSWER_NO;
+    const char *question = NULL;
+
+    GF_ASSERT(words);
+    GF_ASSERT(options);
+
+    dict = dict_new();
+    if (!dict)
+        goto out;
 
-        if (wordcount < 3)
-                goto out;
+    /* new syntax:
+     *
+     * volume geo-replication $m $s create [[ssh-port n] [[no-verify] |
+     * [push-pem]]] [force] volume geo-replication [$m [$s]] status [detail]
+     * volume geo-replication [$m] $s config [[!]$opt [$val]]
+     * volume geo-replication $m $s start|stop [force]
+     * volume geo-replication $m $s delete [reset-sync-time]
+     * volume geo-replication $m $s pause [force]
+     * volume geo-replication $m $s resume [force]
+     */
+
+    if (wordcount < 3)
+        goto out;
 
-        for (i = 2; i <= 3 && i < wordcount - 1; i++) {
-                if (gsyncd_glob_check (words[i]))
-                        glob = i;
-                if (gsyncd_url_check (words[i])) {
-                        slavei = i;
-                        break;
-                }
+    for (i = 2; i <= 3 && i < wordcount - 1; i++) {
+        if (gsyncd_glob_check(words[i]))
+            glob = i;
+        if (gsyncd_url_check(words[i])) {
+            slavei = i;
+            break;
         }
+    }
 
-        if (glob && !slavei)
-                /* glob is allowed only for config, thus it implies there is a
-                 * slave argument; but that might have not been recognized on
-                 * the first scan as it's url characteristics has been covered
-                 * by the glob syntax.
-                 *
-                 * In this case, the slave is perforce the last glob-word -- the
-                 * upcoming one is neither glob, nor url, so it's definitely not
-                 * the slave.
-                 */
-                slavei = glob;
-        if (slavei) {
-                cmdi = slavei + 1;
-                if (slavei == 3)
-                        masteri = 2;
-        } else if (i <= 3) {
-                if (!strcmp ((char *)words[wordcount-1], "detail")) {
-                        /* For status detail it is mandatory to provide
-                         * both master and slave */
-                        ret = -1;
-                        goto out;
-                }
-
-                /* no $s, can only be status cmd
-                 * (with either a single $m before it or nothing)
-                 * -- these conditions imply that i <= 3 after
-                 * the iteration and that i is the successor of
-                 * the (0 or 1 length) sequence of $m-s.
-                 */
-                cmdi = i;
-                if (i == 3)
-                        masteri = 2;
-        } else
-                goto out;
-
-        /* now check if input really complies syntax
-         * (in a somewhat redundant way, in favor
-         * transparent soundness)
+    if (glob && !slavei)
+        /* glob is allowed only for config, thus it implies there is a
+         * slave argument; but that might have not been recognized on
+         * the first scan as it's url characteristics has been covered
+         * by the glob syntax.
+         *
+         * In this case, the slave is perforce the last glob-word -- the
+         * upcoming one is neither glob, nor url, so it's definitely not
+         * the slave.
          */
+        slavei = glob;
+    if (slavei) {
+        cmdi = slavei + 1;
+        if (slavei == 3)
+            masteri = 2;
+    } else if (i <= 4) {
+        if (strtail("detail", (char *)words[wordcount - 1])) {
+            cmdi = wordcount - 2;
+            if (i == 4)
+                masteri = 2;
+        } else {
+            /* no $s, can only be status cmd
+             * (with either a single $m before it or nothing)
+             * -- these conditions imply that i <= 3 after
+             * the iteration and that i is the successor of
+             * the (0 or 1 length) sequence of $m-s.
+             */
+            cmdi = i;
+            if (i == 3)
+                masteri = 2;
+        }
+    } else
+        goto out;
 
-        if (masteri && gsyncd_url_check (words[masteri]))
-                goto out;
-        if (slavei && !glob && !gsyncd_url_check (words[slavei]))
-                goto out;
-
-        w = str_getunamb (words[cmdi], opwords);
-        if (!w)
-                goto out;
-
-        if (strcmp (w, "create") == 0) {
-                type = GF_GSYNC_OPTION_TYPE_CREATE;
-
-                if (!masteri || !slavei)
-                        goto out;
-        } else if (strcmp (w, "status") == 0) {
-                type = GF_GSYNC_OPTION_TYPE_STATUS;
-
-                if (slavei && !masteri)
-                        goto out;
-        } else if (strcmp (w, "config") == 0) {
-                type = GF_GSYNC_OPTION_TYPE_CONFIG;
-
-                if (!slavei)
-                        goto out;
-        } else if (strcmp (w, "start") == 0) {
-                type = GF_GSYNC_OPTION_TYPE_START;
+    /* now check if input really complies syntax
+     * (in a somewhat redundant way, in favor
+     * transparent soundness)
+     */
 
-                if (!masteri || !slavei)
-                        goto out;
-        } else if (strcmp (w, "stop") == 0) {
-                type = GF_GSYNC_OPTION_TYPE_STOP;
+    if (masteri && gsyncd_url_check(words[masteri]))
+        goto out;
 
-                if (!masteri || !slavei)
-                        goto out;
-        } else if (strcmp (w, "delete") == 0) {
-                type = GF_GSYNC_OPTION_TYPE_DELETE;
+    if (slavei && !glob && !valid_slave_gsyncd_url(words[slavei])) {
+        gf_asprintf(errstr, "Invalid slave url: %s", words[slavei]);
+        goto out;
+    }
 
-                if (!masteri || !slavei)
-                        goto out;
-        } else if (strcmp (w, "pause") == 0) {
-                type = GF_GSYNC_OPTION_TYPE_PAUSE;
+    w = str_getunamb(words[cmdi], opwords);
+    if (!w)
+        goto out;
 
-                if (!masteri || !slavei)
-                        goto out;
-        } else if (strcmp (w, "resume") == 0) {
-                type = GF_GSYNC_OPTION_TYPE_RESUME;
+    if (strcmp(w, "create") == 0) {
+        type = GF_GSYNC_OPTION_TYPE_CREATE;
+
+        if (!masteri || !slavei)
+            goto out;
+    } else if (strcmp(w, "status") == 0) {
+        type = GF_GSYNC_OPTION_TYPE_STATUS;
+
+        if (slavei && !masteri)
+            goto out;
+    } else if (strcmp(w, "config") == 0) {
+        type = GF_GSYNC_OPTION_TYPE_CONFIG;
+
+        if (!slavei)
+            goto out;
+    } else if (strcmp(w, "start") == 0) {
+        type = GF_GSYNC_OPTION_TYPE_START;
+
+        if (!masteri || !slavei)
+            goto out;
+    } else if (strcmp(w, "stop") == 0) {
+        type = GF_GSYNC_OPTION_TYPE_STOP;
+
+        if (!masteri || !slavei)
+            goto out;
+    } else if (strcmp(w, "delete") == 0) {
+        type = GF_GSYNC_OPTION_TYPE_DELETE;
+
+        if (!masteri || !slavei)
+            goto out;
+    } else if (strcmp(w, "pause") == 0) {
+        type = GF_GSYNC_OPTION_TYPE_PAUSE;
+
+        if (!masteri || !slavei)
+            goto out;
+    } else if (strcmp(w, "resume") == 0) {
+        type = GF_GSYNC_OPTION_TYPE_RESUME;
+
+        if (!masteri || !slavei)
+            goto out;
+    } else
+        GF_ASSERT(!"opword mismatch");
+
+    ret = force_push_pem_no_verify_parse(words, wordcount, dict, &cmdi);
+    if (ret)
+        goto out;
 
-                if (!masteri || !slavei)
-                        goto out;
-        } else
-                GF_ASSERT (!"opword mismatch");
+    if (strtail("detail", (char *)words[wordcount - 1])) {
+        if (!strtail("status", (char *)words[wordcount - 2])) {
+            ret = -1;
+            goto out;
+        }
 
-        ret = force_push_pem_no_verify_parse (words, wordcount, dict, &cmdi);
+        ret = dict_set_uint32(dict, "status-detail", _gf_true);
         if (ret)
-                goto out;
+            goto out;
+        cmdi++;
+    }
 
-        if (!strcmp ((char *)words[wordcount-1], "detail")) {
-                if (strcmp ((char *)words[wordcount-2], "status")) {
-                        ret = -1;
-                        goto out;
-                }
-                if (!slavei || !masteri) {
-                        ret = -1;
-                        goto out;
-                }
-                ret = dict_set_uint32 (dict, "status-detail", _gf_true);
-                if (ret)
-                        goto out;
-                cmdi++;
+    if (type == GF_GSYNC_OPTION_TYPE_DELETE &&
+        !strcmp((char *)words[wordcount - 1], "reset-sync-time")) {
+        if (strcmp((char *)words[wordcount - 2], "delete")) {
+            ret = -1;
+            goto out;
         }
+        ret = dict_set_uint32(dict, "reset-sync-time", _gf_true);
+        if (ret)
+            goto out;
+        cmdi++;
+    }
 
-        if (type != GF_GSYNC_OPTION_TYPE_CONFIG &&
-            (cmdi < wordcount - 1 || glob))
-                goto out;
+    if (type != GF_GSYNC_OPTION_TYPE_CONFIG && (cmdi < wordcount - 1 || glob)) {
+        ret = -1;
+        goto out;
+    }
 
-        /* If got so far, input is valid, assemble the message */
+    /* If got so far, input is valid, assemble the message */
 
-        ret = 0;
+    ret = 0;
 
-        if (masteri) {
-                ret = dict_set_str (dict, "master", (char *)words[masteri]);
-                if (!ret)
-                        ret = dict_set_str (dict, "volname",
-                                            (char *)words[masteri]);
-        }
-        if (!ret && slavei) {
-                /* If geo-rep is created with root user using the syntax
-                 * gluster vol geo-rep <mastervol> root@<slavehost> ...
-                 * pass down only <slavehost> else pass as it is.
-                 */
-                slave_temp = gf_strdup (words[slavei]);
-                token = strtok_r (slave_temp, "@", &save_ptr);
-                if (token && !strcmp (token, "root")) {
-                        ret = dict_set_str (dict, "slave",
-                                            (char *)words[slavei]+5);
-                } else {
-                        ret = dict_set_str (dict, "slave",
-                                            (char *)words[slavei]);
-                }
-        }
+    if (masteri) {
+        ret = dict_set_str(dict, "master", (char *)words[masteri]);
         if (!ret)
-                ret = dict_set_int32 (dict, "type", type);
-        if (!ret && type == GF_GSYNC_OPTION_TYPE_CONFIG)
-                ret = config_parse (words, wordcount, dict, cmdi, glob);
+            ret = dict_set_str(dict, "volname", (char *)words[masteri]);
+    }
+    if (!ret && slavei) {
+        /* If geo-rep is created with root user using the syntax
+         * gluster vol geo-rep <mastervol> root@<slavehost> ...
+         * pass down only <slavehost> else pass as it is.
+         */
+        slave_temp = gf_strdup(words[slavei]);
+        if (slave_temp == NULL) {
+            ret = -1;
+            goto out;
+        }
+        token = strtok_r(slave_temp, "@", &save_ptr);
+        if (token && !strcmp(token, "root")) {
+            ret = dict_set_str(dict, "slave", (char *)words[slavei] + 5);
+        } else {
+            ret = dict_set_str(dict, "slave", (char *)words[slavei]);
+        }
+    }
+    if (!ret)
+        ret = dict_set_int32(dict, "type", type);
+    if (!ret && type == GF_GSYNC_OPTION_TYPE_CONFIG) {
+        if (!strcmp((char *)words[wordcount - 2], "ignore-deletes") &&
+            !strcmp((char *)words[wordcount - 1], "true")) {
+            question =
+                "There exists ~15 seconds delay for the option to take"
+                " effect from stime of the corresponding brick. Please"
+                " check the log for the time, the option is effective."
+                " Proceed";
+
+            answer = cli_cmd_get_confirmation(state, question);
+
+            if (GF_ANSWER_NO == answer) {
+                gf_log("cli", GF_LOG_INFO,
+                       "Operation "
+                       "cancelled, exiting");
+                *errstr = gf_strdup("Aborted by user.");
+                ret = -1;
+                goto out;
+            }
+        }
 
-out:
-        if (slave_temp)
-                GF_FREE (slave_temp);
-        if (ret) {
-                if (dict)
-                        dict_destroy (dict);
-        } else
-                *options = dict;
+        ret = config_parse(words, wordcount, dict, cmdi, glob);
+    }
 
+out:
+    if (slave_temp)
+        GF_FREE(slave_temp);
+    if (ret && dict)
+        dict_unref(dict);
+    else
+        *options = dict;
 
-        return ret;
+    return ret;
 }
 
 int32_t
-cli_cmd_volume_profile_parse (const char **words, int wordcount,
-                              dict_t **options)
+cli_cmd_volume_profile_parse(const char **words, int wordcount,
+                             dict_t **options)
 {
-        dict_t              *dict       = NULL;
-        char                *volname    = NULL;
-        int                 ret         = -1;
-        gf1_cli_stats_op    op          = GF_CLI_STATS_NONE;
-        gf1_cli_info_op     info_op     = GF_CLI_INFO_NONE;
-        gf_boolean_t        is_peek     = _gf_false;
-
-        char      *opwords[] = { "start", "stop", "info", NULL };
-        char      *w = NULL;
+    dict_t *dict = NULL;
+    char *volname = NULL;
+    int ret = -1;
+    gf1_cli_stats_op op = GF_CLI_STATS_NONE;
+    gf1_cli_info_op info_op = GF_CLI_INFO_NONE;
+    gf_boolean_t is_peek = _gf_false;
 
-        GF_ASSERT (words);
-        GF_ASSERT (options);
+    static char *opwords[] = {"start", "stop", "info", NULL};
+    char *w = NULL;
 
-        dict = dict_new ();
-        if (!dict)
-                goto out;
+    GF_ASSERT(words);
+    GF_ASSERT(options);
 
-        if (wordcount < 4)
-                goto out;
+    dict = dict_new();
+    if (!dict)
+        goto out;
 
-        volname = (char *)words[2];
+    if (wordcount < 4)
+        goto out;
 
-        ret = dict_set_str (dict, "volname", volname);
-        if (ret)
-                goto out;
+    volname = (char *)words[2];
 
-        w = str_getunamb (words[3], opwords);
-        if (!w) {
-                ret = -1;
-                goto out;
-        }
+    ret = dict_set_str(dict, "volname", volname);
+    if (ret)
+        goto out;
 
-        if ((strcmp (w, "start") == 0 || strcmp (w, "stop") == 0) &&
-            wordcount > 5)
-                goto out;
+    w = str_getunamb(words[3], opwords);
+    if (!w) {
+        ret = -1;
+        goto out;
+    }
 
-        if (strcmp (w, "info") == 0 && wordcount > 7)
-                goto out;
+    if ((strcmp(w, "start") == 0 || strcmp(w, "stop") == 0) && wordcount > 5) {
+        ret = -1;
+        goto out;
+    }
 
-        if (strcmp (w, "start") == 0) {
-                op = GF_CLI_STATS_START;
-        } else if (strcmp (w, "stop") == 0) {
-                op = GF_CLI_STATS_STOP;
-        } else if (strcmp (w, "info") == 0) {
-                op = GF_CLI_STATS_INFO;
-                info_op = GF_CLI_INFO_ALL;
-                if (wordcount > 4) {
-                        if (strcmp (words[4], "incremental") == 0) {
-                                info_op = GF_CLI_INFO_INCREMENTAL;
-                                if (wordcount > 5 &&
-                                    strcmp (words[5], "peek") == 0) {
-                                        is_peek = _gf_true;
-                                }
-                        } else if (strcmp (words[4], "cumulative") == 0) {
-                                info_op = GF_CLI_INFO_CUMULATIVE;
-                        } else if (strcmp (words[4], "clear") == 0) {
-                                info_op = GF_CLI_INFO_CLEAR;
-                        } else if (strcmp (words[4], "peek") == 0) {
-                                is_peek = _gf_true;
-                        }
-                }
-        } else
-                GF_ASSERT (!"opword mismatch");
+    if (strcmp(w, "info") == 0 && wordcount > 7) {
+        ret = -1;
+        goto out;
+    }
+
+    if (strcmp(w, "start") == 0) {
+        op = GF_CLI_STATS_START;
+    } else if (strcmp(w, "stop") == 0) {
+        op = GF_CLI_STATS_STOP;
+    } else if (strcmp(w, "info") == 0) {
+        op = GF_CLI_STATS_INFO;
+        info_op = GF_CLI_INFO_ALL;
+        if (wordcount > 4) {
+            if (strcmp(words[4], "incremental") == 0) {
+                info_op = GF_CLI_INFO_INCREMENTAL;
+                if (wordcount > 5 && strcmp(words[5], "peek") == 0) {
+                    is_peek = _gf_true;
+                }
+            } else if (strcmp(words[4], "cumulative") == 0) {
+                info_op = GF_CLI_INFO_CUMULATIVE;
+            } else if (strcmp(words[4], "clear") == 0) {
+                info_op = GF_CLI_INFO_CLEAR;
+            } else if (strcmp(words[4], "peek") == 0) {
+                is_peek = _gf_true;
+            }
+        }
+    } else
+        GF_ASSERT(!"opword mismatch");
+
+    ret = dict_set_int32(dict, "op", (int32_t)op);
+    if (ret)
+        goto out;
 
-        ret = dict_set_int32 (dict, "op", (int32_t)op);
-        if (ret)
-                goto out;
+    ret = dict_set_int32(dict, "info-op", (int32_t)info_op);
+    if (ret)
+        goto out;
 
-        ret = dict_set_int32 (dict, "info-op", (int32_t)info_op);
-        if (ret)
-                goto out;
+    ret = dict_set_int32(dict, "peek", is_peek);
+    if (ret)
+        goto out;
 
-        ret = dict_set_int32 (dict, "peek", is_peek);
+    if (!strcmp(words[wordcount - 1], "nfs")) {
+        ret = dict_set_int32(dict, "nfs", _gf_true);
         if (ret)
-                goto out;
-
-        if (!strcmp (words[wordcount - 1], "nfs")) {
-                ret = dict_set_int32 (dict, "nfs", _gf_true);
-                if (ret)
-                        goto out;
-        }
+            goto out;
+    }
 
-        *options = dict;
+    *options = dict;
 out:
-        if (ret && dict)
-                dict_destroy (dict);
-        return ret;
+    if (ret && dict)
+        dict_unref(dict);
+    return ret;
 }
 
 int32_t
-cli_cmd_volume_top_parse (const char **words, int wordcount,
-                              dict_t **options)
+cli_cmd_volume_top_parse(const char **words, int wordcount, dict_t **options)
 {
-        dict_t  *dict           = NULL;
-        char    *volname        = NULL;
-        char    *value          = NULL;
-        char    *key            = NULL;
-        int      ret            = -1;
-        gf1_cli_stats_op op = GF_CLI_STATS_NONE;
-        gf1_cli_top_op    top_op = GF_CLI_TOP_NONE;
-        int32_t  list_cnt       = -1;
-        int      index          = 0;
-        int      perf           = 0;
-        int32_t  blk_size      = 0;
-        uint32_t  count         = 0;
-        gf_boolean_t nfs        = _gf_false;
-        char    *delimiter      = NULL;
-        char    *opwords[]      = { "open", "read", "write", "opendir",
-                                    "readdir", "read-perf", "write-perf",
-                                    "clear", NULL };
-        char    *w = NULL;
-
-        GF_ASSERT (words);
-        GF_ASSERT (options);
-
-        dict = dict_new ();
-        if (!dict)
-                goto out;
-
-        if (wordcount < 4)
-                goto out;
+    dict_t *dict = NULL;
+    char *volname = NULL;
+    char *value = NULL;
+    char *key = NULL;
+    int ret = -1;
+    gf1_cli_stats_op op = GF_CLI_STATS_NONE;
+    gf1_cli_top_op top_op = GF_CLI_TOP_NONE;
+    int32_t list_cnt = -1;
+    int index = 0;
+    int perf = 0;
+    int32_t blk_size = 0;
+    int count = 0;
+    gf_boolean_t nfs = _gf_false;
+    char *delimiter = NULL;
+    static char *opwords[] = {"open",       "read",    "write",
+                              "opendir",    "readdir", "read-perf",
+                              "write-perf", "clear",   NULL};
+    char *w = NULL;
+
+    GF_ASSERT(words);
+    GF_ASSERT(options);
+
+    dict = dict_new();
+    if (!dict)
+        goto out;
 
-        volname = (char *)words[2];
+    if (wordcount < 4)
+        goto out;
 
-        ret = dict_set_str (dict, "volname", volname);
-        if (ret)
-                goto out;
+    volname = (char *)words[2];
 
-        op = GF_CLI_STATS_TOP;
-        ret = dict_set_int32 (dict, "op", (int32_t)op);
-        if (ret)
-                goto out;
+    ret = dict_set_str(dict, "volname", volname);
+    if (ret)
+        goto out;
 
-        w = str_getunamb (words[3], opwords);
-        if (!w) {
-                ret = -1;
-                goto out;
-        }
-        if (strcmp (w, "open") == 0) {
-                top_op = GF_CLI_TOP_OPEN;
-        } else if (strcmp (w, "read") == 0) {
-                top_op = GF_CLI_TOP_READ;
-        } else if (strcmp (w, "write") == 0) {
-                top_op = GF_CLI_TOP_WRITE;
-        } else if (strcmp (w, "opendir") == 0) {
-                top_op = GF_CLI_TOP_OPENDIR;
-        } else if (strcmp (w, "readdir") == 0) {
-                top_op = GF_CLI_TOP_READDIR;
-        } else if (strcmp (w, "read-perf") == 0) {
-                top_op = GF_CLI_TOP_READ_PERF;
-                perf = 1;
-        } else if (strcmp (w, "write-perf") == 0) {
-                top_op = GF_CLI_TOP_WRITE_PERF;
-                perf = 1;
-        } else if (strcmp (w, "clear") == 0) {
-                ret = dict_set_int32 (dict, "clear-stats", 1);
-                if (ret) {
-                        gf_log ("cli", GF_LOG_ERROR,
-                                "Could not set clear-stats in dict");
-                        goto out;
-                }
-        } else
-                GF_ASSERT (!"opword mismatch");
-        ret = dict_set_int32 (dict, "top-op", (int32_t)top_op);
-        if (ret)
-                goto out;
+    op = GF_CLI_STATS_TOP;
+    ret = dict_set_int32(dict, "op", (int32_t)op);
+    if (ret)
+        goto out;
 
-        if ((wordcount > 4) && !strcmp (words[4], "nfs")) {
-                nfs = _gf_true;
-                ret = dict_set_int32 (dict, "nfs", nfs);
-                if (ret)
-                        goto out;
-                index = 5;
-        } else {
-                index = 4;
+    w = str_getunamb(words[3], opwords);
+    if (!w) {
+        ret = -1;
+        goto out;
+    }
+    if (strcmp(w, "open") == 0) {
+        top_op = GF_CLI_TOP_OPEN;
+    } else if (strcmp(w, "read") == 0) {
+        top_op = GF_CLI_TOP_READ;
+    } else if (strcmp(w, "write") == 0) {
+        top_op = GF_CLI_TOP_WRITE;
+    } else if (strcmp(w, "opendir") == 0) {
+        top_op = GF_CLI_TOP_OPENDIR;
+    } else if (strcmp(w, "readdir") == 0) {
+        top_op = GF_CLI_TOP_READDIR;
+    } else if (strcmp(w, "read-perf") == 0) {
+        top_op = GF_CLI_TOP_READ_PERF;
+        perf = 1;
+    } else if (strcmp(w, "write-perf") == 0) {
+        top_op = GF_CLI_TOP_WRITE_PERF;
+        perf = 1;
+    } else if (strcmp(w, "clear") == 0) {
+        ret = dict_set_int32(dict, "clear-stats", 1);
+        if (ret) {
+            gf_log("cli", GF_LOG_ERROR, "Could not set clear-stats in dict");
+            goto out;
         }
+    } else
+        GF_ASSERT(!"opword mismatch");
+    ret = dict_set_int32(dict, "top-op", (int32_t)top_op);
+    if (ret)
+        goto out;
 
-        for (; index < wordcount; index+=2) {
+    if ((wordcount > 4) && !strcmp(words[4], "nfs")) {
+        nfs = _gf_true;
+        ret = dict_set_int32(dict, "nfs", nfs);
+        if (ret)
+            goto out;
+        index = 5;
+    } else {
+        index = 4;
+    }
 
-                key = (char *) words[index];
-                value = (char *) words[index+1];
+    for (; index < wordcount; index += 2) {
+        key = (char *)words[index];
+        value = (char *)words[index + 1];
 
-                if ( key && !value ) {
-                        ret = -1;
-                        goto out;
-                }
-                if (!strcmp (key, "brick")) {
-                        delimiter = strchr (value, ':');
-                        if (!delimiter || delimiter == value
-                            || *(delimiter+1) != '/') {
-                                cli_err ("wrong brick type: %s, use <HOSTNAME>:"
-                                         "<export-dir-abs-path>", value);
-                                ret = -1;
-                                goto out;
-                        } else {
-                                ret = gf_canonicalize_path (delimiter + 1);
-                                if (ret)
-                                        goto out;
-                        }
-                        ret = dict_set_str (dict, "brick", value);
-
-                } else if (!strcmp (key, "list-cnt")) {
-                        ret = gf_is_str_int (value);
-                        if (!ret)
-                                list_cnt = atoi (value);
-                        if (ret || (list_cnt < 0) || (list_cnt > 100)) {
-                                cli_err ("list-cnt should be between 0 to 100");
-                                ret = -1;
-                                goto out;
-                        }
-                } else if (perf && !nfs && !strcmp (key, "bs")) {
-                        ret = gf_is_str_int (value);
-                        if (!ret)
-                                blk_size = atoi (value);
-                        if (ret || (blk_size <= 0)) {
-                                if (blk_size < 0)
-                                        cli_err ("block size is an invalid"
-                                                 " number");
-                                else
-                                        cli_err ("block size should be an "
-                                                 "integer greater than zero");
-                                ret = -1;
-                                goto out;
-                        }
-                        ret = dict_set_uint32 (dict, "blk-size",
-                                                        (uint32_t)blk_size);
-                } else if (perf && !nfs && !strcmp (key, "count")) {
-                        ret = gf_is_str_int (value);
-                        if (!ret)
-                                count = atoi(value);
-                        if (ret || (count <= 0)) {
-                                if (count < 0)
-                                        cli_err ("count is an invalid number");
-                                else
-                                        cli_err ("count should be an integer "
-                                                 "greater than zero");
-
-                                ret = -1;
-                                goto out;
-                        }
-                        ret = dict_set_uint32 (dict, "blk-cnt", count);
-                } else {
-                        ret = -1;
-                        goto out;
-                }
-                if (ret) {
-                        gf_log ("", GF_LOG_WARNING, "Dict set failed for "
-                                "key %s", key);
-                        goto out;
-                }
-        }
-        if (list_cnt == -1)
-                list_cnt = 100;
-        ret = dict_set_int32 (dict, "list-cnt", list_cnt);
-        if (ret) {
-                gf_log ("", GF_LOG_WARNING, "Dict set failed for list_cnt");
+        if (!key || !value) {
+            ret = -1;
+            goto out;
+        }
+        if (!strcmp(key, "brick")) {
+            delimiter = strchr(value, ':');
+            if (!delimiter || delimiter == value || *(delimiter + 1) != '/') {
+                cli_err(
+                    "wrong brick type: %s, use <HOSTNAME>:"
+                    "<export-dir-abs-path>",
+                    value);
+                ret = -1;
                 goto out;
-        }
-
-        if ((blk_size > 0) ^ (count > 0)) {
-                cli_err ("Need to give both 'bs' and 'count'");
+            } else {
+                ret = gf_canonicalize_path(delimiter + 1);
+                if (ret)
+                    goto out;
+            }
+            ret = dict_set_str(dict, "brick", value);
+
+        } else if (!strcmp(key, "list-cnt")) {
+            ret = gf_is_str_int(value);
+            if (!ret)
+                list_cnt = atoi(value);
+            if (ret || (list_cnt < 0) || (list_cnt > 100)) {
+                cli_err("list-cnt should be between 0 to 100");
                 ret = -1;
                 goto out;
-        } else if (((uint64_t)blk_size * count) > (10 * GF_UNIT_GB)) {
-                cli_err ("'bs * count' value %"PRIu64" is greater than "
-                         "maximum allowed value of 10GB",
-                         ((uint64_t)blk_size * count));
+            }
+        } else if (perf && !nfs && !strcmp(key, "bs")) {
+            ret = gf_is_str_int(value);
+            if (!ret)
+                blk_size = atoi(value);
+            if (ret || (blk_size <= 0)) {
+                if (blk_size < 0)
+                    cli_err(
+                        "block size is an invalid"
+                        " number");
+                else
+                    cli_err(
+                        "block size should be an "
+                        "integer greater than zero");
+                ret = -1;
+                goto out;
+            }
+            ret = dict_set_uint32(dict, "blk-size", (uint32_t)blk_size);
+        } else if (perf && !nfs && !strcmp(key, "count")) {
+            ret = gf_is_str_int(value);
+            if (!ret)
+                count = atoi(value);
+            if (ret || (count <= 0)) {
+                if (count < 0)
+                    cli_err("count is an invalid number");
+                else
+                    cli_err(
+                        "count should be an integer "
+                        "greater than zero");
+
                 ret = -1;
                 goto out;
+            }
+            ret = dict_set_uint32(dict, "blk-cnt", count);
+        } else {
+            ret = -1;
+            goto out;
         }
+        if (ret) {
+            gf_log("", GF_LOG_WARNING,
+                   "Dict set failed for "
+                   "key %s",
+                   key);
+            goto out;
+        }
+    }
+    if (list_cnt == -1)
+        list_cnt = 100;
+    ret = dict_set_int32(dict, "list-cnt", list_cnt);
+    if (ret) {
+        gf_log("", GF_LOG_WARNING, "Dict set failed for list_cnt");
+        goto out;
+    }
 
-        *options = dict;
+    if ((blk_size > 0) ^ (count > 0)) {
+        cli_err("Need to give both 'bs' and 'count'");
+        ret = -1;
+        goto out;
+    } else if (((uint64_t)blk_size * count) > (10 * GF_UNIT_GB)) {
+        cli_err("'bs * count' value %" PRIu64
+                " is greater than "
+                "maximum allowed value of 10GB",
+                ((uint64_t)blk_size * count));
+        ret = -1;
+        goto out;
+    }
+
+    *options = dict;
 out:
-        if (ret && dict)
-                dict_destroy (dict);
-        return ret;
+    if (ret && dict)
+        dict_unref(dict);
+    return ret;
 }
 
 uint32_t
-cli_cmd_get_statusop (const char *arg)
+cli_cmd_get_statusop(const char *arg)
 {
-        int        i         = 0;
-        uint32_t   ret       = GF_CLI_STATUS_NONE;
-        char      *w         = NULL;
-        char      *opwords[] = {"detail", "mem", "clients", "fd",
-                                "inode", "callpool", "tasks", NULL};
-        struct {
-                char      *opname;
-                uint32_t   opcode;
-        } optable[] = {
-                { "detail",   GF_CLI_STATUS_DETAIL   },
-                { "mem",      GF_CLI_STATUS_MEM      },
-                { "clients",  GF_CLI_STATUS_CLIENTS  },
-                { "fd",       GF_CLI_STATUS_FD       },
-                { "inode",    GF_CLI_STATUS_INODE    },
-                { "callpool", GF_CLI_STATUS_CALLPOOL },
-                { "tasks",    GF_CLI_STATUS_TASKS    },
-                { NULL }
-        };
-
-        w = str_getunamb (arg, opwords);
-        if (!w) {
-                gf_log ("cli", GF_LOG_DEBUG,
-                        "Not a status op  %s", arg);
-                goto out;
-        }
+    int i = 0;
+    uint32_t ret = GF_CLI_STATUS_NONE;
+    char *w = NULL;
+    static char *opwords[] = {"detail",   "mem",   "clients",     "fd", "inode",
+                              "callpool", "tasks", "client-list", NULL};
+    static struct {
+        char *opname;
+        uint32_t opcode;
+    } optable[] = {{"detail", GF_CLI_STATUS_DETAIL},
+                   {"mem", GF_CLI_STATUS_MEM},
+                   {"clients", GF_CLI_STATUS_CLIENTS},
+                   {"fd", GF_CLI_STATUS_FD},
+                   {"inode", GF_CLI_STATUS_INODE},
+                   {"callpool", GF_CLI_STATUS_CALLPOOL},
+                   {"tasks", GF_CLI_STATUS_TASKS},
+                   {"client-list", GF_CLI_STATUS_CLIENT_LIST},
+                   {NULL}};
+
+    w = str_getunamb(arg, opwords);
+    if (!w) {
+        gf_log("cli", GF_LOG_DEBUG, "Not a status op  %s", arg);
+        goto out;
+    }
 
-        for (i = 0; optable[i].opname; i++) {
-                if (!strcmp (w, optable[i].opname)) {
-                        ret = optable[i].opcode;
-                        break;
-                }
+    for (i = 0; optable[i].opname; i++) {
+        if (!strcmp(w, optable[i].opname)) {
+            ret = optable[i].opcode;
+            break;
         }
+    }
 
- out:
-        return ret;
+out:
+    return ret;
 }
 
 int
-cli_cmd_volume_status_parse (const char **words, int wordcount,
-                             dict_t **options)
+cli_cmd_volume_status_parse(const char **words, int wordcount, dict_t **options)
 {
-        dict_t    *dict            = NULL;
-        int        ret             = -1;
-        uint32_t   cmd             = 0;
-
-        GF_ASSERT (options);
+    dict_t *dict = NULL;
+    int ret = -1;
+    uint32_t cmd = 0;
 
-        dict = dict_new ();
-        if (!dict)
-                goto out;
+    GF_ASSERT(options);
 
-        switch (wordcount) {
+    dict = dict_new();
+    if (!dict)
+        goto out;
 
+    switch (wordcount) {
         case 2:
-                cmd = GF_CLI_STATUS_ALL;
-                ret = 0;
-                break;
+            cmd = GF_CLI_STATUS_ALL;
+            ret = 0;
+            break;
 
         case 3:
-                if (!strcmp (words[2], "all")) {
-                        cmd = GF_CLI_STATUS_ALL;
-                        ret = 0;
+            if (!strcmp(words[2], "all")) {
+                cmd = GF_CLI_STATUS_ALL;
+                ret = 0;
 
-                } else {
-                        cmd = GF_CLI_STATUS_VOL;
-                        ret = dict_set_str (dict, "volname", (char *)words[2]);
-                }
+            } else {
+                cmd = GF_CLI_STATUS_VOL;
+                ret = dict_set_str(dict, "volname", (char *)words[2]);
+            }
 
-                break;
+            break;
 
         case 4:
-                cmd = cli_cmd_get_statusop (words[3]);
-
-                if (!strcmp (words[2], "all")) {
-                        if (cmd == GF_CLI_STATUS_NONE) {
-                                cli_err ("%s is not a valid status option",
-                                         words[3]);
-                                ret = -1;
-                                goto out;
-                        }
-                        cmd |= GF_CLI_STATUS_ALL;
-                        ret  = 0;
-
-                } else {
-                        ret = dict_set_str (dict, "volname",
-                                            (char *)words[2]);
-                        if (ret)
-                                goto out;
-
-                        if (cmd == GF_CLI_STATUS_NONE) {
-                                if (!strcmp (words[3], "nfs")) {
-                                        cmd |= GF_CLI_STATUS_NFS;
-                                } else if (!strcmp (words[3], "shd")) {
-                                        cmd |= GF_CLI_STATUS_SHD;
-                                } else if (!strcmp (words[3], "quotad")) {
-                                        cmd |= GF_CLI_STATUS_QUOTAD;
-                                } else if (!strcmp (words[3], "snapd")) {
-                                        cmd |= GF_CLI_STATUS_SNAPD;
-                                } else if (!strcmp (words[3], "bitd")) {
-                                        cmd |= GF_CLI_STATUS_BITD;
-                                } else if (!strcmp (words[3], "scrub")) {
-                                        cmd |= GF_CLI_STATUS_SCRUB;
-                                } else {
-                                        cmd = GF_CLI_STATUS_BRICK;
-                                        ret = dict_set_str (dict, "brick",
-                                                            (char *)words[3]);
-                                }
-
-                        } else {
-                                cmd |= GF_CLI_STATUS_VOL;
-                                ret  = 0;
-                        }
-                }
-
-                break;
-
-        case 5:
-                if (!strcmp (words[2], "all")) {
-                        cli_err ("Cannot specify brick/nfs for \"all\"");
-                        ret = -1;
-                        goto out;
-                }
+            cmd = cli_cmd_get_statusop(words[3]);
 
-                cmd = cli_cmd_get_statusop (words[4]);
+            if (!strcmp(words[2], "all")) {
                 if (cmd == GF_CLI_STATUS_NONE) {
-                        cli_err ("%s is not a valid status option",
-                                 words[4]);
-                        ret = -1;
-                        goto out;
+                    cli_err("%s is not a valid status option", words[3]);
+                    ret = -1;
+                    goto out;
                 }
+                cmd |= GF_CLI_STATUS_ALL;
+                ret = 0;
 
-
-                ret = dict_set_str (dict, "volname", (char *)words[2]);
+            } else {
+                ret = dict_set_str(dict, "volname", (char *)words[2]);
                 if (ret)
-                        goto out;
+                    goto out;
 
-                if (!strcmp (words[3], "nfs")) {
-                        if (cmd == GF_CLI_STATUS_FD ||
-                            cmd == GF_CLI_STATUS_DETAIL ||
-                            cmd == GF_CLI_STATUS_TASKS) {
-                                cli_err ("Detail/FD/Tasks status not available"
-                                         " for NFS Servers");
-                                ret = -1;
-                                goto out;
-                        }
+                if (cmd == GF_CLI_STATUS_NONE) {
+                    if (!strcmp(words[3], "nfs")) {
                         cmd |= GF_CLI_STATUS_NFS;
-                } else if (!strcmp (words[3], "shd")){
-                        if (cmd == GF_CLI_STATUS_FD ||
-                            cmd == GF_CLI_STATUS_CLIENTS ||
-                            cmd == GF_CLI_STATUS_DETAIL ||
-                            cmd == GF_CLI_STATUS_TASKS) {
-                                cli_err ("Detail/FD/Clients/Tasks status not "
-                                         "available for Self-heal Daemons");
-                                ret = -1;
-                                goto out;
-                        }
+                    } else if (!strcmp(words[3], "shd")) {
                         cmd |= GF_CLI_STATUS_SHD;
-                } else if (!strcmp (words[3], "quotad")) {
-                        if (cmd == GF_CLI_STATUS_FD ||
-                            cmd == GF_CLI_STATUS_CLIENTS ||
-                            cmd == GF_CLI_STATUS_DETAIL ||
-                            cmd == GF_CLI_STATUS_INODE) {
-                                cli_err ("Detail/FD/Clients/Inode status not "
-                                         "available for Quota Daemon");
-                                ret = -1;
-                                goto out;
-                        }
+                    } else if (!strcmp(words[3], "quotad")) {
                         cmd |= GF_CLI_STATUS_QUOTAD;
-                } else if  (!strcmp (words[3], "snapd")) {
-                        if (cmd == GF_CLI_STATUS_FD ||
-                            cmd == GF_CLI_STATUS_CLIENTS ||
-                            cmd == GF_CLI_STATUS_DETAIL ||
-                            cmd == GF_CLI_STATUS_INODE) {
-                                cli_err ("Detail/FD/Clients/Inode status not "
-                                         "available for snap daemon");
-                                ret = -1;
-                                goto out;
-                        }
+                    } else if (!strcmp(words[3], "snapd")) {
                         cmd |= GF_CLI_STATUS_SNAPD;
+                    } else if (!strcmp(words[3], "bitd")) {
+                        cmd |= GF_CLI_STATUS_BITD;
+                    } else if (!strcmp(words[3], "scrub")) {
+                        cmd |= GF_CLI_STATUS_SCRUB;
+                    } else {
+                        cmd = GF_CLI_STATUS_BRICK;
+                        ret = dict_set_str(dict, "brick", (char *)words[3]);
+                    }
+
                 } else {
-                        if (cmd == GF_CLI_STATUS_TASKS) {
-                                cli_err ("Tasks status not available for "
-                                         "bricks");
-                                ret = -1;
-                                goto out;
-                        }
-                        cmd |= GF_CLI_STATUS_BRICK;
-                        ret = dict_set_str (dict, "brick", (char *)words[3]);
+                    cmd |= GF_CLI_STATUS_VOL;
+                    ret = 0;
                 }
-                break;
+            }
 
-        default:
-                goto out;
-        }
+            break;
 
-        if (ret)
+        case 5:
+            if (!strcmp(words[2], "all")) {
+                cli_err("Cannot specify brick/nfs for \"all\"");
+                ret = -1;
                 goto out;
+            }
 
-        ret = dict_set_int32 (dict, "cmd", cmd);
-        if (ret)
+            cmd = cli_cmd_get_statusop(words[4]);
+            if (cmd == GF_CLI_STATUS_NONE) {
+                cli_err("%s is not a valid status option", words[4]);
+                ret = -1;
                 goto out;
+            }
+
+            ret = dict_set_str(dict, "volname", (char *)words[2]);
+            if (ret)
+                goto out;
+
+            if (!strcmp(words[3], "nfs")) {
+                if (cmd == GF_CLI_STATUS_FD || cmd == GF_CLI_STATUS_DETAIL ||
+                    cmd == GF_CLI_STATUS_TASKS) {
+                    cli_err(
+                        "Detail/FD/Tasks status not available"
+                        " for NFS Servers");
+                    ret = -1;
+                    goto out;
+                }
+                cmd |= GF_CLI_STATUS_NFS;
+            } else if (!strcmp(words[3], "shd")) {
+                if (cmd == GF_CLI_STATUS_FD || cmd == GF_CLI_STATUS_CLIENTS ||
+                    cmd == GF_CLI_STATUS_DETAIL || cmd == GF_CLI_STATUS_TASKS) {
+                    cli_err(
+                        "Detail/FD/Clients/Tasks status not "
+                        "available for Self-heal Daemons");
+                    ret = -1;
+                    goto out;
+                }
+                cmd |= GF_CLI_STATUS_SHD;
+            } else if (!strcmp(words[3], "quotad")) {
+                if (cmd == GF_CLI_STATUS_FD || cmd == GF_CLI_STATUS_CLIENTS ||
+                    cmd == GF_CLI_STATUS_DETAIL || cmd == GF_CLI_STATUS_INODE) {
+                    cli_err(
+                        "Detail/FD/Clients/Inode status not "
+                        "available for Quota Daemon");
+                    ret = -1;
+                    goto out;
+                }
+                cmd |= GF_CLI_STATUS_QUOTAD;
+            } else if (!strcmp(words[3], "snapd")) {
+                if (cmd == GF_CLI_STATUS_FD || cmd == GF_CLI_STATUS_CLIENTS ||
+                    cmd == GF_CLI_STATUS_DETAIL || cmd == GF_CLI_STATUS_INODE) {
+                    cli_err(
+                        "Detail/FD/Clients/Inode status not "
+                        "available for snap daemon");
+                    ret = -1;
+                    goto out;
+                }
+                cmd |= GF_CLI_STATUS_SNAPD;
+            } else {
+                if (cmd == GF_CLI_STATUS_TASKS) {
+                    cli_err(
+                        "Tasks status not available for "
+                        "bricks");
+                    ret = -1;
+                    goto out;
+                }
+                cmd |= GF_CLI_STATUS_BRICK;
+                ret = dict_set_str(dict, "brick", (char *)words[3]);
+            }
+            break;
 
-        *options = dict;
+        default:
+            goto out;
+    }
 
- out:
-        if (ret && dict)
-                dict_destroy (dict);
+    if (ret)
+        goto out;
 
-        return ret;
+    ret = dict_set_int32(dict, "cmd", cmd);
+    if (ret)
+        goto out;
+
+    *options = dict;
+
+out:
+    if (ret && dict)
+        dict_unref(dict);
+
+    return ret;
 }
 
 gf_boolean_t
-cli_cmd_validate_dumpoption (const char *arg, char **option)
+cli_cmd_validate_dumpoption(const char *arg, char **option)
 {
-        char    *opwords[] = {"all", "nfs", "mem", "iobuf", "callpool", "priv",
-                              "fd", "inode", "history", "inodectx", "fdctx",
-                              "quotad", NULL};
-        char    *w = NULL;
-
-        w = str_getunamb (arg, opwords);
-        if (!w) {
-                gf_log ("cli", GF_LOG_DEBUG, "Unknown statedump option  %s",
-                        arg);
-                return _gf_false;
-        }
-        *option = w;
-        return _gf_true;
+    static char *opwords[] = {"all",   "nfs",    "mem",   "iobuf",   "callpool",
+                              "priv",  "fd",     "inode", "history", "inodectx",
+                              "fdctx", "quotad", NULL};
+    char *w = NULL;
+
+    w = str_getunamb(arg, opwords);
+    if (!w) {
+        gf_log("cli", GF_LOG_DEBUG, "Unknown statedump option  %s", arg);
+        return _gf_false;
+    }
+    *option = w;
+    return _gf_true;
 }
 
 int
-cli_cmd_volume_statedump_options_parse (const char **words, int wordcount,
-                                        dict_t **options)
+cli_cmd_volume_statedump_options_parse(const char **words, int wordcount,
+                                       dict_t **options)
 {
-        int     ret = 0;
-        int     i = 0;
-        dict_t  *dict = NULL;
-        int     option_cnt = 0;
-        char    *option = NULL;
-        char    option_str[100] = {0,};
-
-        for (i = 3; i < wordcount; i++, option_cnt++) {
-                if (!cli_cmd_validate_dumpoption (words[i], &option)) {
-                        ret = -1;
-                        goto out;
-                }
-                strncat (option_str, option, strlen (option));
-                strncat (option_str, " ", 1);
+    int ret = 0;
+    int i = 0;
+    dict_t *dict = NULL;
+    int option_cnt = 0;
+    char *option = NULL;
+    char *option_str = NULL;
+    char *tmp_str = NULL;
+    char *tmp = NULL;
+    char *ip_addr = NULL;
+    char *pid = NULL;
+
+    if ((wordcount >= 5) && ((strcmp(words[3], "client")) == 0)) {
+        tmp = gf_strdup(words[4]);
+        if (!tmp) {
+            ret = -1;
+            goto out;
+        }
+        ip_addr = strtok(tmp, ":");
+        pid = strtok(NULL, ":");
+        if (valid_internet_address(ip_addr, _gf_true, _gf_false) && pid &&
+            gf_valid_pid(pid, strlen(pid))) {
+            ret = gf_asprintf(&option_str, "%s %s %s", words[3], ip_addr, pid);
+            if (ret < 0) {
+                goto out;
+            }
+            option_cnt = 3;
+        } else {
+            ret = -1;
+            goto out;
         }
-        if((strstr (option_str, "nfs")) && strstr (option_str, "quotad")) {
+    } else {
+        for (i = 3; i < wordcount; i++, option_cnt++) {
+            if (!cli_cmd_validate_dumpoption(words[i], &option)) {
                 ret = -1;
                 goto out;
+            }
+            tmp_str = option_str;
+            option_str = NULL;
+            ret = gf_asprintf(&option_str, "%s%s ", tmp_str ? tmp_str : "",
+                              option);
+            GF_FREE(tmp_str);
+            if (ret < 0) {
+                goto out;
+            }
         }
+        if (option_str && (strstr(option_str, "nfs")) &&
+            strstr(option_str, "quotad")) {
+            ret = -1;
+            goto out;
+        }
+    }
 
-        dict = dict_new ();
-        if (!dict)
-                goto out;
+    dict = dict_new();
+    if (!dict) {
+        ret = -1;
+        goto out;
+    }
 
-        ret = dict_set_dynstr (dict, "options", gf_strdup (option_str));
-        if (ret)
-                goto out;
+    /* dynamic string in dict is freed up when dict is freed up, and hence
+    if option_str is NULL pass in an duplicate empty string to the same */
+    ret = dict_set_dynstr(dict, "options",
+                          (option_str ? option_str : gf_strdup("")));
+    if (ret)
+        goto out;
+    option_str = NULL;
 
-        ret = dict_set_int32 (dict, "option_cnt", option_cnt);
-        if (ret)
-                goto out;
+    ret = dict_set_int32(dict, "option_cnt", option_cnt);
+    if (ret)
+        goto out;
 
-        *options = dict;
+    *options = dict;
 out:
-        if (ret && dict)
-                dict_destroy (dict);
-        if (ret)
-                gf_log ("cli", GF_LOG_ERROR, "Error parsing dumpoptions");
-        return ret;
+    GF_FREE(tmp);
+    GF_FREE(option_str);
+    if (ret && dict)
+        dict_unref(dict);
+    if (ret)
+        gf_log("cli", GF_LOG_ERROR, "Error parsing dumpoptions");
+    return ret;
 }
 
 int
-cli_cmd_volume_clrlks_opts_parse (const char **words, int wordcount,
-                                  dict_t **options)
+cli_cmd_volume_clrlks_opts_parse(const char **words, int wordcount,
+                                 dict_t **options)
 {
-        int     ret = -1;
-        int     i = 0;
-        dict_t  *dict = NULL;
-        char    *kind_opts[4] = {"blocked", "granted", "all", NULL};
-        char    *types[4] = {"inode", "entry", "posix", NULL};
-        char    *free_ptr = NULL;
-
-        dict = dict_new ();
-        if (!dict)
-                goto out;
+    int ret = -1;
+    int i = 0;
+    dict_t *dict = NULL;
+    char *kind_opts[4] = {"blocked", "granted", "all", NULL};
+    char *types[4] = {"inode", "entry", "posix", NULL};
+    char *free_ptr = NULL;
+
+    dict = dict_new();
+    if (!dict)
+        goto out;
 
-        if (strcmp (words[4], "kind"))
-                goto out;
+    if (strcmp(words[4], "kind"))
+        goto out;
 
-        for (i = 0; kind_opts[i]; i++) {
-               if (!strcmp (words[5], kind_opts[i])) {
-                       free_ptr = gf_strdup (words[5]);
-                       ret = dict_set_dynstr (dict, "kind", free_ptr);
-                       if (ret)
-                               goto out;
-                       free_ptr = NULL;
-                       break;
-               }
-        }
-        if (i == 3)
+    for (i = 0; kind_opts[i]; i++) {
+        if (!strcmp(words[5], kind_opts[i])) {
+            free_ptr = gf_strdup(words[5]);
+            ret = dict_set_dynstr(dict, "kind", free_ptr);
+            if (ret)
                 goto out;
+            free_ptr = NULL;
+            break;
+        }
+    }
+    if (i == 3)
+        goto out;
 
-        ret = -1;
-        for (i = 0; types[i]; i++) {
-               if (!strcmp (words[6], types[i])) {
-                       free_ptr = gf_strdup (words[6]);
-                       ret = dict_set_dynstr (dict, "type", free_ptr);
-                       if (ret)
-                               goto out;
-                       free_ptr = NULL;
-                       break;
-               }
-        }
-        if (i == 3)
+    ret = -1;
+    for (i = 0; types[i]; i++) {
+        if (!strcmp(words[6], types[i])) {
+            free_ptr = gf_strdup(words[6]);
+            ret = dict_set_dynstr(dict, "type", free_ptr);
+            if (ret)
                 goto out;
-
-        if (wordcount == 8) {
-                free_ptr = gf_strdup (words[7]);
-                ret = dict_set_dynstr (dict, "opts", free_ptr);
-                if (ret)
-                        goto out;
-                free_ptr = NULL;
+            free_ptr = NULL;
+            break;
         }
+    }
+    if (i == 3)
+        goto out;
 
-        ret = 0;
-        *options = dict;
+    if (wordcount == 8) {
+        free_ptr = gf_strdup(words[7]);
+        ret = dict_set_dynstr(dict, "opts", free_ptr);
+        if (ret)
+            goto out;
+        free_ptr = NULL;
+    }
+
+    ret = 0;
+    *options = dict;
 out:
-       if (ret) {
-               GF_FREE (free_ptr);
-               dict_unref (dict);
-       }
+    if (ret) {
+        GF_FREE(free_ptr);
+        dict_unref(dict);
+    }
 
-       return ret;
+    return ret;
 }
 
 static int
-extract_hostname_path_from_token (const char *tmp_words, char **hostname,
-                                  char **path)
+extract_hostname_path_from_token(const char *tmp_words, char **hostname,
+                                 char **path)
 {
-        int ret = 0;
-        char *delimiter = NULL;
-        char *tmp_host = NULL;
-        char *host_name = NULL;
-        char *words = NULL;
-
-        *hostname = NULL;
-        *path = NULL;
-
-        words = GF_CALLOC (1, strlen (tmp_words) + 1, gf_common_mt_char);
-        if (!words){
-                ret = -1;
-                goto out;
-        }
+    int ret = 0;
+    char *delimiter = NULL;
+    char *tmp_host = NULL;
+    char *host_name = NULL;
+    char *words = NULL;
+    int str_len = 0;
+    *hostname = NULL;
+    *path = NULL;
+
+    str_len = strlen(tmp_words) + 1;
+    words = GF_MALLOC(str_len, gf_common_mt_char);
+    if (!words) {
+        ret = -1;
+        goto out;
+    }
 
-        strncpy (words, tmp_words, strlen (tmp_words) + 1);
+    snprintf(words, str_len, "%s", tmp_words);
 
-        if (validate_brick_name (words)) {
-                cli_err ("Wrong brick type: %s, use <HOSTNAME>:"
-                        "<export-dir-abs-path>", words);
-                ret = -1;
-                goto out;
+    if (validate_brick_name(words)) {
+        cli_err(
+            "Wrong brick type: %s, use <HOSTNAME>:"
+            "<export-dir-abs-path>",
+            words);
+        ret = -1;
+        goto out;
+    } else {
+        delimiter = strrchr(words, ':');
+        ret = gf_canonicalize_path(delimiter + 1);
+        if (ret) {
+            goto out;
         } else {
-                delimiter = strrchr (words, ':');
-                ret = gf_canonicalize_path (delimiter + 1);
-                if (ret) {
-                        goto out;
-                } else {
-                        *path = GF_CALLOC (1, strlen (delimiter+1) +1,
-                                           gf_common_mt_char);
-                        if (!*path) {
-                           ret = -1;
-                                goto out;
-
-                        }
-                        strncpy (*path, delimiter +1,
-                                 strlen(delimiter + 1) + 1);
-                }
-        }
-
-        tmp_host = gf_strdup (words);
-        if (!tmp_host) {
-                gf_log ("cli", GF_LOG_ERROR, "Out of memory");
-                ret = -1;
-                goto out;
-        }
-        get_host_name (tmp_host, &host_name);
-        if (!host_name) {
-                ret = -1;
-                gf_log("cli",GF_LOG_ERROR, "Unable to allocate "
-                        "memory");
-                goto out;
-        }
-        if (!(strcmp (host_name, "localhost") &&
-            strcmp (host_name, "127.0.0.1") &&
-            strncmp (host_name, "0.", 2))) {
-                cli_err ("Please provide a valid hostname/ip other "
-                         "than localhost, 127.0.0.1 or loopback "
-                         "address (0.0.0.0 to 0.255.255.255).");
-                ret = -1;
-                goto out;
-        }
-        if (!valid_internet_address (host_name, _gf_false)) {
-                cli_err ("internet address '%s' does not conform to "
-                          "standards", host_name);
+            str_len = strlen(delimiter + 1) + 1;
+            *path = GF_MALLOC(str_len, gf_common_mt_char);
+            if (!*path) {
                 ret = -1;
                 goto out;
+            }
+            snprintf(*path, str_len, "%s", delimiter + 1);
         }
+    }
 
-        *hostname = GF_CALLOC (1, strlen (host_name) + 1,
-                                       gf_common_mt_char);
-        if (!*hostname) {
-                ret = -1;
-                goto out;
-        }
-        strncpy (*hostname, host_name, strlen (host_name) + 1);
-        ret = 0;
+    tmp_host = gf_strdup(words);
+    if (!tmp_host) {
+        gf_log("cli", GF_LOG_ERROR, "Out of memory");
+        ret = -1;
+        goto out;
+    }
+    get_host_name(tmp_host, &host_name);
+    if (!host_name) {
+        ret = -1;
+        gf_log("cli", GF_LOG_ERROR,
+               "Unable to allocate "
+               "memory");
+        goto out;
+    }
+    if (!(strcmp(host_name, "localhost") && strcmp(host_name, "127.0.0.1") &&
+          strncmp(host_name, "0.", 2))) {
+        cli_err(
+            "Please provide a valid hostname/ip other "
+            "than localhost, 127.0.0.1 or loopback "
+            "address (0.0.0.0 to 0.255.255.255).");
+        ret = -1;
+        goto out;
+    }
+    if (!valid_internet_address(host_name, _gf_false, _gf_false)) {
+        cli_err(
+            "internet address '%s' does not conform to "
+            "standards",
+            host_name);
+        ret = -1;
+        goto out;
+    }
+
+    str_len = strlen(host_name) + 1;
+    *hostname = GF_MALLOC(str_len, gf_common_mt_char);
+    if (!*hostname) {
+        ret = -1;
+        goto out;
+    }
+    snprintf(*hostname, str_len, "%s", host_name);
+    ret = 0;
 
 out:
-        GF_FREE (words);
-        GF_FREE (tmp_host);
-        return ret;
+    GF_FREE(words);
+    GF_FREE(tmp_host);
+    return ret;
 }
 
 static int
-set_hostname_path_in_dict (const char *token, dict_t *dict, int heal_op)
+set_hostname_path_in_dict(const char *token, dict_t *dict, int heal_op)
 {
-        char *hostname = NULL;
-        char *path     = NULL;
-        int   ret      = 0;
+    char *hostname = NULL;
+    char *path = NULL;
+    int ret = 0;
 
-        ret = extract_hostname_path_from_token (token, &hostname, &path);
-        if (ret)
-                goto out;
+    ret = extract_hostname_path_from_token(token, &hostname, &path);
+    if (ret)
+        goto out;
 
-        switch (heal_op) {
+    switch (heal_op) {
         case GF_SHD_OP_SBRAIN_HEAL_FROM_BRICK:
-                ret = dict_set_dynstr (dict, "heal-source-hostname",
-                                       hostname);
-                if (ret)
-                        goto out;
-                ret = dict_set_dynstr (dict, "heal-source-brickpath",
-                                       path);
-                break;
+            ret = dict_set_dynstr(dict, "heal-source-hostname", hostname);
+            if (ret)
+                goto out;
+            hostname = NULL;
+            ret = dict_set_dynstr(dict, "heal-source-brickpath", path);
+            if (ret) {
+                goto out;
+            }
+            path = NULL;
+            break;
         case GF_SHD_OP_STATISTICS_HEAL_COUNT_PER_REPLICA:
-                ret = dict_set_dynstr (dict, "per-replica-cmd-hostname",
-                                       hostname);
-                if (ret)
-                        goto out;
-                ret = dict_set_dynstr (dict, "per-replica-cmd-path",
-                                       path);
-                break;
+            ret = dict_set_dynstr(dict, "per-replica-cmd-hostname", hostname);
+            if (ret)
+                goto out;
+            hostname = NULL;
+            ret = dict_set_dynstr(dict, "per-replica-cmd-path", path);
+            if (ret) {
+                goto out;
+            }
+            path = NULL;
+            break;
         default:
-                ret = -1;
-                break;
-        }
+            ret = -1;
+            break;
+    }
 
 out:
-        return ret;
+    GF_FREE(hostname);
+    GF_FREE(path);
+    return ret;
 }
 
 static int
-heal_command_type_get (const char *command)
+heal_command_type_get(const char *command)
 {
-        int     i = 0;
-        /* subcommands are set as NULL */
-        char    *heal_cmds[GF_SHD_OP_HEAL_DISABLE + 1] = {
-                [GF_SHD_OP_INVALID]                            = NULL,
-                [GF_SHD_OP_HEAL_INDEX]                         = NULL,
-                [GF_SHD_OP_HEAL_FULL]                          = "full",
-                [GF_SHD_OP_INDEX_SUMMARY]                      = "info",
-                [GF_SHD_OP_HEALED_FILES]                       = NULL,
-                [GF_SHD_OP_HEAL_FAILED_FILES]                  = NULL,
-                [GF_SHD_OP_SPLIT_BRAIN_FILES]                  = NULL,
-                [GF_SHD_OP_STATISTICS]                         = "statistics",
-                [GF_SHD_OP_STATISTICS_HEAL_COUNT]              = NULL,
-                [GF_SHD_OP_STATISTICS_HEAL_COUNT_PER_REPLICA]  = NULL,
-                [GF_SHD_OP_SBRAIN_HEAL_FROM_BIGGER_FILE]       = NULL,
-                [GF_SHD_OP_SBRAIN_HEAL_FROM_BRICK]             = NULL,
-                [GF_SHD_OP_HEAL_ENABLE]                        = "enable",
-                [GF_SHD_OP_HEAL_DISABLE]                       = "disable",
-        };
-
-        for (i = 0; i <= GF_SHD_OP_HEAL_DISABLE; i++) {
-                if (heal_cmds[i] && (strcmp (heal_cmds[i], command) == 0))
-                        return i;
-        }
-
-        return GF_SHD_OP_INVALID;
+    int i = 0;
+    /* subcommands are set as NULL */
+    char *heal_cmds[GF_SHD_OP_HEAL_DISABLE + 1] = {
+        [GF_SHD_OP_INVALID] = NULL,
+        [GF_SHD_OP_HEAL_INDEX] = NULL,
+        [GF_SHD_OP_HEAL_FULL] = "full",
+        [GF_SHD_OP_INDEX_SUMMARY] = "info",
+        [GF_SHD_OP_SPLIT_BRAIN_FILES] = NULL,
+        [GF_SHD_OP_STATISTICS] = "statistics",
+        [GF_SHD_OP_STATISTICS_HEAL_COUNT] = NULL,
+        [GF_SHD_OP_STATISTICS_HEAL_COUNT_PER_REPLICA] = NULL,
+        [GF_SHD_OP_SBRAIN_HEAL_FROM_BIGGER_FILE] = NULL,
+        [GF_SHD_OP_SBRAIN_HEAL_FROM_BRICK] = NULL,
+        [GF_SHD_OP_HEAL_ENABLE] = "enable",
+        [GF_SHD_OP_HEAL_DISABLE] = "disable",
+    };
+
+    for (i = 0; i <= GF_SHD_OP_HEAL_DISABLE; i++) {
+        if (heal_cmds[i] && (strcmp(heal_cmds[i], command) == 0))
+            return i;
+    }
+
+    return GF_SHD_OP_INVALID;
 }
 
 int
-cli_cmd_volume_heal_options_parse (const char **words, int wordcount,
-                                   dict_t **options)
+cli_cmd_volume_heal_options_parse(const char **words, int wordcount,
+                                  dict_t **options)
 {
-        int     ret = 0;
-        dict_t  *dict = NULL;
-        char    *hostname = NULL;
-        char    *path = NULL;
-        gf_xl_afr_op_t op = GF_SHD_OP_INVALID;
-
-        dict = dict_new ();
-        if (!dict)
-                goto out;
+    int ret = 0;
+    dict_t *dict = NULL;
+    gf_xl_afr_op_t op = GF_SHD_OP_INVALID;
 
-        ret = dict_set_str (dict, "volname", (char *) words[2]);
-        if (ret) {
-                gf_log (THIS->name, GF_LOG_ERROR, "failed to set volname");
-                goto out;
-        }
+    dict = dict_new();
+    if (!dict) {
+        gf_log(THIS->name, GF_LOG_ERROR, "Failed to create the dict");
+        ret = -1;
+        goto out;
+    }
 
-        if (wordcount == 3) {
-                ret = dict_set_int32 (dict, "heal-op", GF_SHD_OP_HEAL_INDEX);
-                goto done;
+    ret = dict_set_str(dict, "volname", (char *)words[2]);
+    if (ret) {
+        gf_log(THIS->name, GF_LOG_ERROR, "failed to set volname");
+        goto out;
+    }
+
+    if (wordcount == 3) {
+        ret = dict_set_int32(dict, "heal-op", GF_SHD_OP_HEAL_INDEX);
+        goto done;
+    }
+
+    if (wordcount == 4) {
+        op = heal_command_type_get(words[3]);
+        if (op == GF_SHD_OP_INVALID) {
+            ret = -1;
+            goto out;
         }
 
-        if (wordcount == 4) {
-                op = heal_command_type_get (words[3]);
-                if (op == GF_SHD_OP_INVALID) {
-                        ret = -1;
-                        goto out;
-                }
+        ret = dict_set_int32(dict, "heal-op", op);
+        goto done;
+    }
+
+    if (wordcount == 5) {
+        if (strcmp(words[3], "info") && strcmp(words[3], "statistics") &&
+            strcmp(words[3], "granular-entry-heal")) {
+            ret = -1;
+            goto out;
+        }
 
-                ret = dict_set_int32 (dict, "heal-op", op);
+        if (!strcmp(words[3], "info")) {
+            if (!strcmp(words[4], "split-brain")) {
+                ret = dict_set_int32(dict, "heal-op",
+                                     GF_SHD_OP_SPLIT_BRAIN_FILES);
+                goto done;
+            }
+            if (!strcmp(words[4], "summary")) {
+                ret = dict_set_int32(dict, "heal-op", GF_SHD_OP_HEAL_SUMMARY);
                 goto done;
+            }
         }
 
-        if (wordcount == 5) {
-                if (strcmp (words[3], "info") &&
-                    strcmp (words[3], "statistics")) {
-                        ret = -1;
-                        goto out;
-                }
+        if (!strcmp(words[3], "statistics")) {
+            if (!strcmp(words[4], "heal-count")) {
+                ret = dict_set_int32(dict, "heal-op",
+                                     GF_SHD_OP_STATISTICS_HEAL_COUNT);
+                goto done;
+            }
+        }
 
-                if (!strcmp (words[3], "info")) {
-                        if (!strcmp (words[4], "healed")) {
-                                ret = dict_set_int32 (dict, "heal-op",
-                                                      GF_SHD_OP_HEALED_FILES);
-                                goto done;
-                        }
-                        if (!strcmp (words[4], "heal-failed")) {
-                                ret = dict_set_int32 (dict, "heal-op",
-                                                   GF_SHD_OP_HEAL_FAILED_FILES);
-                                goto done;
-                        }
-                        if (!strcmp (words[4], "split-brain")) {
-                                ret = dict_set_int32 (dict, "heal-op",
-                                                   GF_SHD_OP_SPLIT_BRAIN_FILES);
-                                goto done;
-                        }
-                }
+        if (!strcmp(words[3], "granular-entry-heal")) {
+            if (!strcmp(words[4], "enable")) {
+                ret = dict_set_int32(dict, "heal-op",
+                                     GF_SHD_OP_GRANULAR_ENTRY_HEAL_ENABLE);
+                goto done;
+            } else if (!strcmp(words[4], "disable")) {
+                ret = dict_set_int32(dict, "heal-op",
+                                     GF_SHD_OP_GRANULAR_ENTRY_HEAL_DISABLE);
+                goto done;
+            }
+        }
 
-                if (!strcmp (words[3], "statistics")) {
-                        if (!strcmp (words[4], "heal-count")) {
-                                ret = dict_set_int32 (dict, "heal-op",
-                                               GF_SHD_OP_STATISTICS_HEAL_COUNT);
-                                goto done;
-                        }
-                }
-                ret = -1;
+        ret = -1;
+        goto out;
+    }
+    if (wordcount == 6) {
+        if (strcmp(words[3], "split-brain")) {
+            ret = -1;
+            goto out;
+        }
+        if (!strcmp(words[4], "bigger-file")) {
+            ret = dict_set_int32(dict, "heal-op",
+                                 GF_SHD_OP_SBRAIN_HEAL_FROM_BIGGER_FILE);
+            if (ret)
                 goto out;
+            ret = dict_set_str(dict, "file", (char *)words[5]);
+            if (ret)
+                goto out;
+            goto done;
         }
-        if (wordcount == 6) {
-                if (strcmp (words[3], "split-brain")) {
-                        ret = -1;
-                        goto out;
-                }
-                if (!strcmp (words[4], "bigger-file")) {
-                        ret = dict_set_int32 (dict, "heal-op",
-                                        GF_SHD_OP_SBRAIN_HEAL_FROM_BIGGER_FILE);
-                        if (ret)
-                                goto out;
-                        ret = dict_set_str (dict, "file", (char *)words[5]);
-                        if (ret)
-                                goto out;
-                        goto done;
-                }
-                if (!strcmp (words[4], "source-brick")) {
-                        ret = dict_set_int32 (dict, "heal-op",
-                                              GF_SHD_OP_SBRAIN_HEAL_FROM_BRICK);
-                        if (ret)
-                                goto out;
-                        ret = set_hostname_path_in_dict (words[5], dict,
-                                              GF_SHD_OP_SBRAIN_HEAL_FROM_BRICK);
-                        if (ret)
-                                goto out;
-                        goto done;
-                }
-                ret = -1;
+        if (!strcmp(words[4], "latest-mtime")) {
+            ret = dict_set_int32(dict, "heal-op",
+                                 GF_SHD_OP_SBRAIN_HEAL_FROM_LATEST_MTIME);
+            if (ret)
+                goto out;
+            ret = dict_set_str(dict, "file", (char *)words[5]);
+            if (ret)
                 goto out;
+            goto done;
         }
-        if (wordcount == 7) {
-                if (!strcmp (words[3], "statistics")
-                    && !strcmp (words[4], "heal-count")
-                    && !strcmp (words[5], "replica")) {
-
-                        ret = dict_set_int32 (dict, "heal-op",
-                                   GF_SHD_OP_STATISTICS_HEAL_COUNT_PER_REPLICA);
-                        if (ret)
-                                goto out;
-                        ret = set_hostname_path_in_dict (words[6], dict,
-                                   GF_SHD_OP_STATISTICS_HEAL_COUNT_PER_REPLICA);
-                        if (ret)
-                                goto out;
-                        goto done;
-
-                }
-                if (!strcmp (words[3], "split-brain") &&
-                    !strcmp (words[4], "source-brick")) {
-                        ret = dict_set_int32 (dict, "heal-op",
-                                              GF_SHD_OP_SBRAIN_HEAL_FROM_BRICK);
-                        ret = set_hostname_path_in_dict (words[5], dict,
-                                              GF_SHD_OP_SBRAIN_HEAL_FROM_BRICK);
-                        if (ret)
-                                goto out;
-                        ret = dict_set_str (dict, "file",
-                                            (char *) words[6]);
-                        if (ret)
-                                goto out;
-                        goto done;
-                }
+        if (!strcmp(words[4], "source-brick")) {
+            ret = dict_set_int32(dict, "heal-op",
+                                 GF_SHD_OP_SBRAIN_HEAL_FROM_BRICK);
+            if (ret)
+                goto out;
+            ret = set_hostname_path_in_dict(words[5], dict,
+                                            GF_SHD_OP_SBRAIN_HEAL_FROM_BRICK);
+            if (ret)
+                goto out;
+            goto done;
         }
         ret = -1;
         goto out;
+    }
+    if (wordcount == 7) {
+        if (!strcmp(words[3], "statistics") &&
+            !strcmp(words[4], "heal-count") && !strcmp(words[5], "replica")) {
+            ret = dict_set_int32(dict, "heal-op",
+                                 GF_SHD_OP_STATISTICS_HEAL_COUNT_PER_REPLICA);
+            if (ret)
+                goto out;
+            ret = set_hostname_path_in_dict(
+                words[6], dict, GF_SHD_OP_STATISTICS_HEAL_COUNT_PER_REPLICA);
+            if (ret)
+                goto out;
+            goto done;
+        }
+        if (!strcmp(words[3], "split-brain") &&
+            !strcmp(words[4], "source-brick")) {
+            ret = dict_set_int32(dict, "heal-op",
+                                 GF_SHD_OP_SBRAIN_HEAL_FROM_BRICK);
+            ret = set_hostname_path_in_dict(words[5], dict,
+                                            GF_SHD_OP_SBRAIN_HEAL_FROM_BRICK);
+            if (ret)
+                goto out;
+            ret = dict_set_str(dict, "file", (char *)words[6]);
+            if (ret)
+                goto out;
+            goto done;
+        }
+    }
+    ret = -1;
+    goto out;
 done:
-        *options = dict;
+    *options = dict;
 out:
-        if (ret && dict) {
-                dict_unref (dict);
-                *options = NULL;
-        }
+    if (ret && dict) {
+        dict_unref(dict);
+        *options = NULL;
+    }
 
-        return ret;
+    return ret;
 }
 
 int
-cli_cmd_volume_defrag_parse (const char **words, int wordcount,
-                             dict_t **options)
+cli_cmd_volume_defrag_parse(const char **words, int wordcount, dict_t **options)
 {
-        dict_t                 *dict = NULL;
-        int                      ret = -1;
-        char                *option  = NULL;
-        char                *volname = NULL;
-        char                *command = NULL;
-        gf_cli_defrag_type       cmd = 0;
-
-        GF_ASSERT (words);
-        GF_ASSERT (options);
-
-        dict = dict_new ();
-        if (!dict)
-                goto out;
-
-        if (!((wordcount == 4) || (wordcount == 5)))
-                goto out;
-
-        if (wordcount == 4) {
-                if (strcmp (words[3], "start") && strcmp (words[3], "stop") &&
-                    strcmp (words[3], "status"))
-                            goto out;
-        } else if ((strcmp (words[3], "tier") == 0) &&
-                   (strcmp (words[4], "start") == 0)) {
-                volname = (char *) words[2];
-                cmd = GF_DEFRAG_CMD_START_TIER;
-                goto done;
-        } else if ((strcmp (words[3], "tier") == 0) &&
-               (strcmp (words[4], "status") == 0)) {
-                volname = (char *) words[2];
-                cmd = GF_DEFRAG_CMD_STATUS_TIER;
-                goto done;
-        } else {
-                if (strcmp (words[3], "fix-layout") &&
-                    strcmp (words[3], "start"))
-                        goto out;
-        }
-
-        volname = (char *) words[2];
-
-        if (wordcount == 4) {
-                command = (char *) words[3];
-        }
-        if (wordcount == 5) {
-               if ((strcmp (words[3], "fix-layout") ||
-                    strcmp (words[4], "start")) &&
-                    (strcmp (words[3], "start") ||
-                    strcmp (words[4], "force"))) {
-                        ret = -1;
-                        goto out;
-                }
-                command = (char *) words[3];
-                option = (char *) words[4];
-        }
+    dict_t *dict = NULL;
+    int ret = -1;
+    char *option = NULL;
+    char *volname = NULL;
+    char *command = NULL;
+    gf_cli_defrag_type cmd = 0;
+
+    GF_ASSERT(words);
+    GF_ASSERT(options);
+
+    dict = dict_new();
+    if (!dict)
+        goto out;
 
-        if (strcmp (command, "start") == 0) {
-                cmd = GF_DEFRAG_CMD_START;
-                if (option && strcmp (option, "force") == 0) {
-                                cmd = GF_DEFRAG_CMD_START_FORCE;
-                        }
-                goto done;
-        }
+    if (!((wordcount == 4) || (wordcount == 5)))
+        goto out;
 
-        if (strcmp (command, "fix-layout") == 0) {
-                cmd = GF_DEFRAG_CMD_START_LAYOUT_FIX;
-                goto done;
-        }
-        if (strcmp (command, "stop") == 0) {
-                cmd = GF_DEFRAG_CMD_STOP;
-                goto done;
-        }
-        if (strcmp (command, "status") == 0) {
-                cmd = GF_DEFRAG_CMD_STATUS;
-        }
+    if (wordcount == 4) {
+        if (strcmp(words[3], "start") && strcmp(words[3], "stop") &&
+            strcmp(words[3], "status"))
+            goto out;
+    } else {
+        if (strcmp(words[3], "fix-layout") && strcmp(words[3], "start"))
+            goto out;
+    }
+
+    volname = (char *)words[2];
+
+    if (wordcount == 4) {
+        command = (char *)words[3];
+    }
+    if (wordcount == 5) {
+        if ((strcmp(words[3], "fix-layout") || strcmp(words[4], "start")) &&
+            (strcmp(words[3], "start") || strcmp(words[4], "force"))) {
+            ret = -1;
+            goto out;
+        }
+        command = (char *)words[3];
+        option = (char *)words[4];
+    }
+
+    if (strcmp(command, "start") == 0) {
+        cmd = GF_DEFRAG_CMD_START;
+        if (option && strcmp(option, "force") == 0) {
+            cmd = GF_DEFRAG_CMD_START_FORCE;
+        }
+        goto done;
+    }
+
+    if (strcmp(command, "fix-layout") == 0) {
+        cmd = GF_DEFRAG_CMD_START_LAYOUT_FIX;
+        goto done;
+    }
+    if (strcmp(command, "stop") == 0) {
+        cmd = GF_DEFRAG_CMD_STOP;
+        goto done;
+    }
+    if (strcmp(command, "status") == 0) {
+        cmd = GF_DEFRAG_CMD_STATUS;
+    }
 
 done:
-        ret = dict_set_str (dict, "volname", volname);
+    ret = dict_set_str(dict, "volname", volname);
 
-        if (ret) {
-                gf_log (THIS->name, GF_LOG_ERROR, "failed to set dict");
-                goto out;
-        }
+    if (ret) {
+        gf_log(THIS->name, GF_LOG_ERROR, "failed to set dict");
+        goto out;
+    }
 
-        ret = dict_set_int32 (dict, "rebalance-command", (int32_t) cmd);
+    ret = dict_set_int32(dict, "rebalance-command", (int32_t)cmd);
 
-        if (ret) {
-                gf_log (THIS->name, GF_LOG_ERROR, "failed to set dict");
-                goto out;
-        }
+    if (ret) {
+        gf_log(THIS->name, GF_LOG_ERROR, "failed to set dict");
+        goto out;
+    }
 
-        *options = dict;
+    *options = dict;
 
 out:
-        if (ret && dict)
-                dict_destroy (dict);
+    if (ret && dict)
+        dict_unref(dict);
 
-        return ret;
+    return ret;
 }
 
 int32_t
-cli_snap_create_desc_parse (dict_t *dict, const char **words,
-                            size_t wordcount, int32_t desc_opt_loc)
+cli_snap_create_desc_parse(dict_t *dict, const char **words, size_t wordcount,
+                           int32_t desc_opt_loc)
 {
-        int32_t        ret      = -1;
-        char          *desc     = NULL;
-        int32_t        desc_len = 0;
-
-        desc = GF_CALLOC (MAX_SNAP_DESCRIPTION_LEN + 1, sizeof(char),
-                          gf_common_mt_char);
-        if (!desc) {
-                ret = -1;
-                goto out;
-        }
-
-
-        if (strlen (words[desc_opt_loc]) >= MAX_SNAP_DESCRIPTION_LEN) {
-                cli_out ("snapshot create: description truncated: "
-                         "Description provided is longer than 1024 characters");
-                desc_len = MAX_SNAP_DESCRIPTION_LEN;
-        } else {
-                desc_len = strlen (words[desc_opt_loc]);
-        }
-
-        strncpy (desc, words[desc_opt_loc], desc_len);
-        desc[desc_len] = '\0';
-        /* Calculating the size of the description as given by the user */
+    int32_t ret = -1;
+    char *desc = NULL;
+    int32_t desc_len = 0;
+    int len;
 
-        ret = dict_set_dynstr (dict, "description", desc);
-        if (ret) {
-                gf_log ("cli", GF_LOG_ERROR, "Unable to save snap "
-                        "description");
-                goto out;
-        }
+    desc = GF_MALLOC(MAX_SNAP_DESCRIPTION_LEN + 1, gf_common_mt_char);
+    if (!desc) {
+        ret = -1;
+        goto out;
+    }
+
+    len = strlen(words[desc_opt_loc]);
+    if (len >= MAX_SNAP_DESCRIPTION_LEN) {
+        cli_out(
+            "snapshot create: description truncated: "
+            "Description provided is longer than 1024 characters");
+        desc_len = MAX_SNAP_DESCRIPTION_LEN;
+    } else {
+        desc_len = len;
+    }
+
+    snprintf(desc, desc_len + 1, "%s", words[desc_opt_loc]);
+    /* Calculating the size of the description as given by the user */
+
+    ret = dict_set_dynstr(dict, "description", desc);
+    if (ret) {
+        gf_log("cli", GF_LOG_ERROR,
+               "Unable to save snap "
+               "description");
+        goto out;
+    }
 
-        ret = 0;
+    ret = 0;
 out:
-        if (ret && desc)
-                GF_FREE (desc);
+    if (ret && desc)
+        GF_FREE(desc);
 
-        return ret;
+    return ret;
 }
 
 /* Function to check whether the Volume name is repeated */
 int
-cli_check_if_volname_repeated (const char **words, unsigned int start_index,
-                           uint64_t cur_index) {
-        uint64_t        i       =       -1;
-        int             ret     =        0;
+cli_check_if_volname_repeated(const char **words, unsigned int start_index,
+                              uint64_t cur_index)
+{
+    uint64_t i = -1;
+    int ret = 0;
 
-        GF_ASSERT (words);
+    GF_ASSERT(words);
 
-        for (i = start_index ; i < cur_index ; i++) {
-                if (strcmp (words[i], words[cur_index]) == 0) {
-                        ret = -1;
-                        goto out;
-                }
+    for (i = start_index; i < cur_index; i++) {
+        if (strcmp(words[i], words[cur_index]) == 0) {
+            ret = -1;
+            goto out;
         }
+    }
 out:
-        return ret;
+    return ret;
 }
 
 /* snapshot clone <clonename> <snapname>
@@ -3820,74 +4243,77 @@ out:
  *                 0 on success
  */
 int
-cli_snap_clone_parse (dict_t *dict, const char **words, int wordcount) {
-        uint64_t        i               =       0;
-        int             ret             =       -1;
-        char            key[PATH_MAX]   =       "";
-        char            *clonename      =       NULL;
-        unsigned int    cmdi            =       2;
-        int             flags           =       0;
-        /* cmdi is command index, here cmdi is "2" (gluster snapshot clone)*/
-
-        GF_ASSERT (words);
-        GF_ASSERT (dict);
-
-        if (wordcount == cmdi + 1) {
-                cli_err ("Invalid Syntax.");
-                gf_log ("cli", GF_LOG_ERROR,
-                        "Invalid number of  words for snap clone command");
-                goto out;
-        }
-
-        if (strlen(words[cmdi]) >= GLUSTERD_MAX_SNAP_NAME) {
-                cli_err ("snapshot clone: failed: clonename cannot exceed "
-                         "255 characters.");
-                gf_log ("cli", GF_LOG_ERROR, "Clone name too long");
-
-                goto out;
-        }
-
-        clonename = (char *) words[cmdi];
-        for (i = 0 ; i < strlen (clonename); i++) {
-                /* Following volume name convention */
-                if (!isalnum (clonename[i]) && (clonename[i] != '_'
-                                           && (clonename[i] != '-'))) {
-                        /* TODO : Is this message enough?? */
-                        cli_err ("Clonename can contain only alphanumeric, "
-                                 "\"-\" and \"_\" characters");
-                        goto out;
-                }
-        }
-
-        ret = dict_set_int32 (dict, "volcount", 1);
-        if (ret) {
-                gf_log ("cli", GF_LOG_ERROR, "Could not save volcount");
-                goto out;
-        }
-
-        ret = dict_set_str (dict, "clonename", (char *)words[cmdi]);
-        if (ret) {
-                gf_log ("cli", GF_LOG_ERROR, "Could not save clone "
-                        "name(%s)", (char *)words[cmdi]);
-                goto out;
-        }
+cli_snap_clone_parse(dict_t *dict, const char **words, int wordcount)
+{
+    uint64_t i = 0;
+    int ret = -1;
+    char *clonename = NULL;
+    unsigned int cmdi = 2;
+    /* cmdi is command index, here cmdi is "2" (gluster snapshot clone)*/
+
+    GF_ASSERT(words);
+    GF_ASSERT(dict);
+
+    if (wordcount == cmdi + 1) {
+        cli_err("Invalid Syntax.");
+        gf_log("cli", GF_LOG_ERROR,
+               "Invalid number of  words for snap clone command");
+        goto out;
+    }
 
-        /* Filling snap name in the dictionary */
-        ret = dict_set_str (dict, "snapname", (char *)words[cmdi+1]);
-        if (ret) {
-                gf_log ("cli", GF_LOG_ERROR, "Could not "
-                        "save snap name(%s)", (char *)words[cmdi+1]);
-                goto out;
-        }
+    if (strlen(words[cmdi]) >= GLUSTERD_MAX_SNAP_NAME) {
+        cli_err(
+            "snapshot clone: failed: clonename cannot exceed "
+            "255 characters.");
+        gf_log("cli", GF_LOG_ERROR, "Clone name too long");
 
+        goto out;
+    }
+
+    clonename = (char *)words[cmdi];
+    for (i = 0; i < strlen(clonename); i++) {
+        /* Following volume name convention */
+        if (!isalnum(clonename[i]) &&
+            (clonename[i] != '_' && (clonename[i] != '-'))) {
+            /* TODO : Is this message enough?? */
+            cli_err(
+                "Clonename can contain only alphanumeric, "
+                "\"-\" and \"_\" characters");
+            goto out;
+        }
+    }
+
+    ret = dict_set_int32(dict, "volcount", 1);
+    if (ret) {
+        gf_log("cli", GF_LOG_ERROR, "Could not save volcount");
+        goto out;
+    }
+
+    ret = dict_set_str(dict, "clonename", (char *)words[cmdi]);
+    if (ret) {
+        gf_log("cli", GF_LOG_ERROR,
+               "Could not save clone "
+               "name(%s)",
+               (char *)words[cmdi]);
+        goto out;
+    }
+
+    /* Filling snap name in the dictionary */
+    ret = dict_set_str(dict, "snapname", (char *)words[cmdi + 1]);
+    if (ret) {
+        gf_log("cli", GF_LOG_ERROR,
+               "Could not "
+               "save snap name(%s)",
+               (char *)words[cmdi + 1]);
+        goto out;
+    }
 
-        ret = 0;
+    ret = 0;
 
 out:
-        return ret;
+    return ret;
 }
 
-
 /* snapshot create <snapname> <vol-name(s)> [description <description>]
  *                                           [force]
  * @arg-0, dict     : Request Dictionary to be sent to server side.
@@ -3898,172 +4324,181 @@ out:
  *                 0 on success
  */
 int
-cli_snap_create_parse (dict_t *dict, const char **words, int wordcount) {
-        uint64_t        i               =       0;
-        int             ret             =       -1;
-        uint64_t        volcount        =       0;
-        char            key[PATH_MAX]   =       "";
-        char            *snapname       =       NULL;
-        unsigned int    cmdi            =       2;
-        int             flags           =       0;
-        /* cmdi is command index, here cmdi is "2" (gluster snapshot create)*/
-
-        GF_ASSERT (words);
-        GF_ASSERT (dict);
-
-        if (wordcount <= cmdi + 1) {
-                cli_err ("Invalid Syntax.");
-                gf_log ("cli", GF_LOG_ERROR,
-                        "Too less words for snap create command");
-                goto out;
-        }
+cli_snap_create_parse(dict_t *dict, const char **words, int wordcount)
+{
+    uint64_t i = 0;
+    int ret = -1;
+    uint64_t volcount = 0;
+    char key[PATH_MAX] = "";
+    char *snapname = NULL;
+    unsigned int cmdi = 2;
+    int flags = 0;
+    /* cmdi is command index, here cmdi is "2" (gluster snapshot create)*/
+
+    GF_ASSERT(words);
+    GF_ASSERT(dict);
+
+    if (wordcount <= cmdi + 1) {
+        cli_err("Invalid Syntax.");
+        gf_log("cli", GF_LOG_ERROR, "Too less words for snap create command");
+        goto out;
+    }
 
-        if (strlen(words[cmdi]) >= GLUSTERD_MAX_SNAP_NAME) {
-                cli_err ("snapshot create: failed: snapname cannot exceed "
-                         "255 characters.");
-                gf_log ("cli", GF_LOG_ERROR, "Snapname too long");
+    if (strlen(words[cmdi]) >= GLUSTERD_MAX_SNAP_NAME) {
+        cli_err(
+            "snapshot create: failed: snapname cannot exceed "
+            "255 characters.");
+        gf_log("cli", GF_LOG_ERROR, "Snapname too long");
 
-                goto out;
+        goto out;
+    }
+
+    snapname = (char *)words[cmdi];
+    for (i = 0; i < strlen(snapname); i++) {
+        /* Following volume name convention */
+        if (!isalnum(snapname[i]) &&
+            (snapname[i] != '_' && (snapname[i] != '-'))) {
+            /* TODO : Is this message enough?? */
+            cli_err(
+                "Snapname can contain only alphanumeric, "
+                "\"-\" and \"_\" characters");
+            goto out;
+        }
+    }
+
+    ret = dict_set_str(dict, "snapname", (char *)words[cmdi]);
+    if (ret) {
+        gf_log("cli", GF_LOG_ERROR,
+               "Could not save snap "
+               "name(%s)",
+               (char *)words[cmdi]);
+        goto out;
+    }
+
+    /* Filling volume name in the dictionary */
+    for (i = cmdi + 1;
+         i < wordcount && (strcmp(words[i], "description")) != 0 &&
+         (strcmp(words[i], "force") != 0) &&
+         (strcmp(words[i], "no-timestamp") != 0);
+         i++) {
+        volcount++;
+        /* volume index starts from 1 */
+        ret = snprintf(key, sizeof(key), "volname%" PRIu64, volcount);
+        if (ret < 0) {
+            goto out;
         }
 
-        snapname = (char *) words[cmdi];
-        for (i = 0 ; i < strlen (snapname); i++) {
-                /* Following volume name convention */
-                if (!isalnum (snapname[i]) && (snapname[i] != '_'
-                                           && (snapname[i] != '-'))) {
-                        /* TODO : Is this message enough?? */
-                        cli_err ("Snapname can contain only alphanumeric, "
-                                 "\"-\" and \"_\" characters");
-                        goto out;
-                }
+        ret = dict_set_str(dict, key, (char *)words[i]);
+        if (ret) {
+            gf_log("cli", GF_LOG_ERROR,
+                   "Could not "
+                   "save volume name(%s)",
+                   (char *)words[i]);
+            goto out;
         }
 
-        ret = dict_set_str (dict, "snapname", (char *)words[cmdi]);
-        if (ret) {
-                gf_log ("cli", GF_LOG_ERROR, "Could not save snap "
-                        "name(%s)", (char *)words[cmdi]);
-                goto out;
+        if (i >= cmdi + 2) {
+            ret = -1;
+            cli_err(
+                "Creating multiple volume snapshot is not "
+                "supported as of now");
+            goto out;
         }
+        /* TODO : remove this above condition check once
+         * multiple volume snapshot is supported */
+    }
 
-        /* Filling volume name in the dictionary */
-        for (i = cmdi + 1 ; i < wordcount
-                            && (strcmp (words[i], "description")) != 0
-                            && (strcmp (words[i], "force") != 0)
-                            && (strcmp (words[i], "no-timestamp") != 0);
-                            i++) {
-                volcount++;
-                /* volume index starts from 1 */
-                ret = snprintf (key, sizeof (key), "volname%"PRIu64, volcount);
-                if (ret < 0) {
-                        goto out;
-                }
+    if (volcount == 0) {
+        ret = -1;
+        cli_err("Please provide the volume name");
+        gf_log("cli", GF_LOG_ERROR, "Invalid Syntax");
+        goto out;
+    }
 
-                ret = dict_set_str (dict, key, (char *)words[i]);
-                if (ret) {
-                        gf_log ("cli", GF_LOG_ERROR, "Could not "
-                                "save volume name(%s)", (char *)words[i]);
-                        goto out;
-                }
+    ret = dict_set_int32(dict, "volcount", volcount);
+    if (ret) {
+        gf_log("cli", GF_LOG_ERROR, "Could not save volcount");
+        goto out;
+    }
+
+    /* Verify how we got out of "for" loop,
+     * if it is by reaching wordcount limit then goto "out",
+     * because we need not parse for "description","force" and
+     * "no-timestamp" after this.
+     */
+    if (i == wordcount) {
+        goto out;
+    }
 
-                if (i >= cmdi + 2) {
-                        ret = -1;
-                        cli_err("Creating multiple volume snapshot is not "
-                                "supported as of now");
-                        goto out;
-                }
-                /* TODO : remove this above condition check once
-                 * multiple volume snapshot is supported */
+    if (strcmp(words[i], "no-timestamp") == 0) {
+        ret = dict_set_int32n(dict, "no-timestamp", SLEN("no-timestamp"), 1);
+        if (ret) {
+            gf_log("cli", GF_LOG_ERROR,
+                   "Could not save "
+                   "time-stamp option");
         }
+        if (i == (wordcount - 1))
+            goto out;
+        i++;
+    }
 
-        if (volcount == 0) {
-                ret = -1;
-                cli_err ("Please provide the volume name");
-                gf_log ("cli", GF_LOG_ERROR, "Invalid Syntax");
-                goto out;
+    if ((strcmp(words[i], "description")) == 0) {
+        ++i;
+        if (i > (wordcount - 1)) {
+            ret = -1;
+            cli_err("Please provide a description");
+            gf_log("cli", GF_LOG_ERROR, "Description not provided");
+            goto out;
         }
 
-        ret = dict_set_int32 (dict, "volcount", volcount);
+        ret = cli_snap_create_desc_parse(dict, words, wordcount, i);
         if (ret) {
-                gf_log ("cli", GF_LOG_ERROR, "Could not save volcount");
-                goto out;
-        }
-
-        /* Verify how we got out of "for" loop,
-         * if it is by reaching wordcount limit then goto "out",
-         * because we need not parse for "description","force" and
-         * "no-timestamp" after this.
+            gf_log("cli", GF_LOG_ERROR,
+                   "Could not save snap "
+                   "description");
+            goto out;
+        }
+
+        if (i == (wordcount - 1))
+            goto out;
+        i++;
+        /* point the index to next word.
+         * As description might be follwed by force option.
+         * Before that, check if wordcount limit is reached
          */
-        if (i == wordcount) {
-                goto out;
-        }
-
-        if (strcmp (words[i], "no-timestamp") == 0) {
-                ret = dict_set_str (dict, "no-timestamp", "true");
-                if (ret) {
-                        gf_log ("cli", GF_LOG_ERROR, "Could not save "
-                                "time-stamp option");
-                }
-                if (i == (wordcount-1))
-                        goto out;
-                i++;
-        }
+    }
 
-        if ((strcmp (words[i], "description")) == 0) {
-                ++i;
-                if (i > (wordcount - 1)) {
-                        ret = -1;
-                        cli_err ("Please provide a description");
-                        gf_log ("cli", GF_LOG_ERROR,
-                                "Description not provided");
-                        goto out;
-                }
+    if (strcmp(words[i], "force") == 0) {
+        flags = GF_CLI_FLAG_OP_FORCE;
 
-                ret = cli_snap_create_desc_parse(dict, words, wordcount, i);
-                if (ret) {
-                        gf_log ("cli", GF_LOG_ERROR, "Could not save snap "
-                                "description");
-                        goto out;
-                }
-
-                if (i == (wordcount - 1))
-                        goto out;
-                i++;
-                /* point the index to next word.
-                 * As description might be follwed by force option.
-                 * Before that, check if wordcount limit is reached
-                 */
-        }
-
-        if (strcmp (words[i], "force") == 0) {
-                flags = GF_CLI_FLAG_OP_FORCE;
-
-        } else {
-                ret = -1;
-                cli_err ("Invalid Syntax.");
-                gf_log ("cli", GF_LOG_ERROR, "Invalid Syntax");
-                goto out;
-        }
+    } else {
+        ret = -1;
+        cli_err("Invalid Syntax.");
+        gf_log("cli", GF_LOG_ERROR, "Invalid Syntax");
+        goto out;
+    }
 
-        /* Check if the command has anything after "force" keyword */
-        if (++i < wordcount) {
-                ret = -1;
-                gf_log ("cli", GF_LOG_ERROR, "Invalid Syntax");
-                goto out;
-        }
+    /* Check if the command has anything after "force" keyword */
+    if (++i < wordcount) {
+        ret = -1;
+        gf_log("cli", GF_LOG_ERROR, "Invalid Syntax");
+        goto out;
+    }
 
-        ret = 0;
+    ret = 0;
 
 out:
-        if(ret == 0) {
-                /*Adding force flag in either of the case i.e force set
-                 * or unset*/
-                ret = dict_set_int32 (dict, "flags", flags);
-                if (ret) {
-                        gf_log ("cli", GF_LOG_ERROR, "Could not save "
-                                "snap force option");
-                }
+    if (ret == 0) {
+        /*Adding force flag in either of the case i.e force set
+         * or unset*/
+        ret = dict_set_int32(dict, "flags", flags);
+        if (ret) {
+            gf_log("cli", GF_LOG_ERROR,
+                   "Could not save "
+                   "snap force option");
         }
-        return ret;
+    }
+    return ret;
 }
 
 /* snapshot list [volname]
@@ -4075,30 +4510,30 @@ out:
  *                 0 on success
  */
 int
-cli_snap_list_parse (dict_t *dict, const char **words, int wordcount) {
-        int             ret     =       -1;
+cli_snap_list_parse(dict_t *dict, const char **words, int wordcount)
+{
+    int ret = -1;
 
-        GF_ASSERT (words);
-        GF_ASSERT (dict);
+    GF_ASSERT(words);
+    GF_ASSERT(dict);
 
-        if (wordcount < 2 || wordcount > 3) {
-                gf_log ("cli", GF_LOG_ERROR, "Invalid Syntax");
-                goto out;
-        }
+    if (wordcount < 2 || wordcount > 3) {
+        gf_log("cli", GF_LOG_ERROR, "Invalid Syntax");
+        goto out;
+    }
 
-        if (wordcount == 2) {
-                ret = 0;
-                goto out;
-        }
+    if (wordcount == 2) {
+        ret = 0;
+        goto out;
+    }
 
-        ret = dict_set_str (dict, "volname", (char *)words[2]);
-        if (ret) {
-                gf_log ("cli", GF_LOG_ERROR,
-                        "Failed to save volname in dictionary");
-                goto out;
-        }
+    ret = dict_set_str(dict, "volname", (char *)words[2]);
+    if (ret) {
+        gf_log("cli", GF_LOG_ERROR, "Failed to save volname in dictionary");
+        goto out;
+    }
 out:
-        return ret;
+    return ret;
 }
 
 /* snapshot info [(snapname |  volume <volname>)]
@@ -4110,87 +4545,88 @@ out:
  *                 0 on success
  */
 int
-cli_snap_info_parse (dict_t *dict, const char **words, int wordcount)
+cli_snap_info_parse(dict_t *dict, const char **words, int wordcount)
 {
+    int ret = -1;
+    int32_t cmd = GF_SNAP_INFO_TYPE_ALL;
+    unsigned int cmdi = 2;
+    /* cmdi is command index, here cmdi is "2" (gluster snapshot info)*/
 
-        int             ret             =       -1;
-        int32_t         cmd             =       GF_SNAP_INFO_TYPE_ALL;
-        unsigned int    cmdi            =        2;
-        /* cmdi is command index, here cmdi is "2" (gluster snapshot info)*/
+    GF_ASSERT(words);
+    GF_ASSERT(dict);
 
-        GF_ASSERT (words);
-        GF_ASSERT (dict);
+    if (wordcount > 4 || wordcount < cmdi) {
+        gf_log("cli", GF_LOG_ERROR, "Invalid syntax");
+        goto out;
+    }
 
-        if (wordcount > 4 || wordcount < cmdi) {
-                gf_log ("cli", GF_LOG_ERROR, "Invalid syntax");
-                goto out;
-        }
+    if (wordcount == cmdi) {
+        ret = 0;
+        goto out;
+    }
 
-        if (wordcount == cmdi) {
-                ret = 0;
-                goto out;
+    /* If 3rd word is not "volume", then it must
+     * be snapname.
+     */
+    if (strcmp(words[cmdi], "volume") != 0) {
+        ret = dict_set_str(dict, "snapname", (char *)words[cmdi]);
+        if (ret) {
+            gf_log("cli", GF_LOG_ERROR,
+                   "Unable to save "
+                   "snapname %s",
+                   words[cmdi]);
+            goto out;
         }
 
-        /* If 3rd word is not "volume", then it must
-         * be snapname.
+        /* Once snap name is parsed, if we encounter any other
+         * word then fail it. Invalid Syntax.
+         * example : snapshot info <snapname> word
          */
-        if (strcmp (words[cmdi], "volume") != 0) {
-                ret = dict_set_str (dict, "snapname",
-                                   (char *)words[cmdi]);
-                if (ret) {
-                        gf_log ("cli", GF_LOG_ERROR, "Unable to save "
-                                "snapname %s", words[cmdi]);
-                        goto out;
-                }
-
-                /* Once snap name is parsed, if we encounter any other
-                 * word then fail it. Invalid Syntax.
-                 * example : snapshot info <snapname> word
-                 */
-                if ((cmdi + 1) != wordcount) {
-                        ret = -1;
-                        gf_log ("cli", GF_LOG_ERROR, "Invalid Syntax");
-                        goto out;
-                }
-
-                cmd = GF_SNAP_INFO_TYPE_SNAP;
-                ret = 0;
-                goto out;
-                /* No need to continue the parsing once we
-                 * get the snapname
-                 */
+        if ((cmdi + 1) != wordcount) {
+            ret = -1;
+            gf_log("cli", GF_LOG_ERROR, "Invalid Syntax");
+            goto out;
         }
 
-        /* If 3rd word is "volume", then check if next word
-         * is present. As, "snapshot info volume" is an
-         * invalid command.
+        cmd = GF_SNAP_INFO_TYPE_SNAP;
+        ret = 0;
+        goto out;
+        /* No need to continue the parsing once we
+         * get the snapname
          */
-        if ((cmdi + 1) == wordcount) {
-                ret = -1;
-                gf_log ("cli", GF_LOG_ERROR, "Invalid Syntax");
-                goto out;
-        }
+    }
 
-        ret = dict_set_str (dict, "volname", (char *)words[wordcount - 1]);
-        if (ret) {
-                gf_log ("cli", GF_LOG_ERROR, "Could not save "
-                        "volume name %s", words[wordcount - 1]);
-                goto out;
-        }
-        cmd = GF_SNAP_INFO_TYPE_VOL;
+    /* If 3rd word is "volume", then check if next word
+     * is present. As, "snapshot info volume" is an
+     * invalid command.
+     */
+    if ((cmdi + 1) == wordcount) {
+        ret = -1;
+        gf_log("cli", GF_LOG_ERROR, "Invalid Syntax");
+        goto out;
+    }
+
+    ret = dict_set_str(dict, "volname", (char *)words[wordcount - 1]);
+    if (ret) {
+        gf_log("cli", GF_LOG_ERROR,
+               "Could not save "
+               "volume name %s",
+               words[wordcount - 1]);
+        goto out;
+    }
+    cmd = GF_SNAP_INFO_TYPE_VOL;
 out:
-        if (ret == 0) {
-                ret = dict_set_int32 (dict, "cmd", cmd);
-                if (ret) {
-                        gf_log ("cli", GF_LOG_ERROR, "Could not save "
-                                "type of snapshot info");
-                }
+    if (ret == 0) {
+        ret = dict_set_int32(dict, "sub-cmd", cmd);
+        if (ret) {
+            gf_log("cli", GF_LOG_ERROR,
+                   "Could not save "
+                   "type of snapshot info");
         }
-        return ret;
+    }
+    return ret;
 }
 
-
-
 /* snapshot restore <snapname>
  * @arg-0, dict     : Request Dictionary to be sent to server side.
  * @arg-1, words    : Contains individual words of CLI command.
@@ -4200,42 +4636,43 @@ out:
  *                 0 on success
  */
 int
-cli_snap_restore_parse (dict_t *dict, const char **words, int wordcount,
-                        struct cli_state *state)
+cli_snap_restore_parse(dict_t *dict, const char **words, int wordcount,
+                       struct cli_state *state)
 {
+    int ret = -1;
+    const char *question = NULL;
+    gf_answer_t answer = GF_ANSWER_NO;
 
-        int             ret             =       -1;
-        const char      *question       =       NULL;
-        gf_answer_t     answer          =       GF_ANSWER_NO;
-
-        GF_ASSERT (words);
-        GF_ASSERT (dict);
-
-        if (wordcount != 3) {
-                gf_log ("cli", GF_LOG_ERROR, "Invalid Syntax");
-                goto out;
-        }
-
-        ret = dict_set_str (dict, "snapname", (char *)words[2]);
-        if (ret) {
-                gf_log ("cli", GF_LOG_ERROR, "Unable to save snap-name %s",
-                        words[2]);
-                goto out;
-        }
+    GF_ASSERT(words);
+    GF_ASSERT(dict);
 
-        question = "Restore operation will replace the "
-                   "original volume with the snapshotted volume. "
-                   "Do you still want to continue?";
+    if (wordcount != 3) {
+        gf_log("cli", GF_LOG_ERROR, "Invalid Syntax");
+        goto out;
+    }
 
-        answer = cli_cmd_get_confirmation (state, question);
-        if (GF_ANSWER_NO == answer) {
-                ret = 1;
-                gf_log ("cli", GF_LOG_ERROR, "User cancelled a snapshot "
-                        "restore operation for snap %s", (char *)words[2]);
-                goto out;
-        }
+    ret = dict_set_str(dict, "snapname", (char *)words[2]);
+    if (ret) {
+        gf_log("cli", GF_LOG_ERROR, "Unable to save snap-name %s", words[2]);
+        goto out;
+    }
+
+    question =
+        "Restore operation will replace the "
+        "original volume with the snapshotted volume. "
+        "Do you still want to continue?";
+
+    answer = cli_cmd_get_confirmation(state, question);
+    if (GF_ANSWER_NO == answer) {
+        ret = 1;
+        gf_log("cli", GF_LOG_ERROR,
+               "User cancelled a snapshot "
+               "restore operation for snap %s",
+               (char *)words[2]);
+        goto out;
+    }
 out:
-        return ret;
+    return ret;
 }
 
 /* snapshot activate <snapname> [force]
@@ -4247,43 +4684,41 @@ out:
  *                 0 on success
  */
 int
-cli_snap_activate_parse (dict_t *dict, const char **words, int wordcount)
+cli_snap_activate_parse(dict_t *dict, const char **words, int wordcount)
 {
+    int ret = -1;
+    int flags = 0;
 
-        int ret = -1;
-        int flags = 0;
+    GF_ASSERT(words);
+    GF_ASSERT(dict);
 
-        GF_ASSERT (words);
-        GF_ASSERT (dict);
-
-        if ((wordcount < 3) || (wordcount > 4)) {
-                gf_log ("cli", GF_LOG_ERROR, "Invalid Syntax");
-                goto out;
-        }
+    if ((wordcount < 3) || (wordcount > 4)) {
+        gf_log("cli", GF_LOG_ERROR, "Invalid Syntax");
+        goto out;
+    }
 
-        ret = dict_set_str (dict, "snapname", (char *)words[2]);
-        if (ret) {
-                gf_log ("cli", GF_LOG_ERROR, "Unable to save snap-name %s",
-                        words[2]);
-                goto out;
-        }
+    ret = dict_set_str(dict, "snapname", (char *)words[2]);
+    if (ret) {
+        gf_log("cli", GF_LOG_ERROR, "Unable to save snap-name %s", words[2]);
+        goto out;
+    }
 
-        if (wordcount == 4) {
-                if (!strcmp("force", (char *)words[3])) {
-                        flags = GF_CLI_FLAG_OP_FORCE;
-                } else {
-                        gf_log ("cli", GF_LOG_ERROR, "Invalid option");
-                        ret = -1;
-                        goto out;
-                }
-        }
-        ret = dict_set_int32 (dict, "flags", flags);
-        if (ret) {
-                gf_log ("cli", GF_LOG_ERROR, "Unable to save force option");
-                goto out;
-        }
+    if (wordcount == 4) {
+        if (!strcmp("force", (char *)words[3])) {
+            flags = GF_CLI_FLAG_OP_FORCE;
+        } else {
+            gf_log("cli", GF_LOG_ERROR, "Invalid option");
+            ret = -1;
+            goto out;
+        }
+    }
+    ret = dict_set_int32(dict, "flags", flags);
+    if (ret) {
+        gf_log("cli", GF_LOG_ERROR, "Unable to save force option");
+        goto out;
+    }
 out:
-        return ret;
+    return ret;
 }
 
 /* snapshot deactivate <snapname>
@@ -4296,42 +4731,41 @@ out:
  *                 1 if user cancelled the request
  */
 int
-cli_snap_deactivate_parse (dict_t *dict, const char **words, int wordcount,
-                        struct cli_state *state)
+cli_snap_deactivate_parse(dict_t *dict, const char **words, int wordcount,
+                          struct cli_state *state)
 {
+    int ret = -1;
+    gf_answer_t answer = GF_ANSWER_NO;
+    const char *question =
+        "Deactivating snap will make its "
+        "data inaccessible. Do you want to "
+        "continue?";
+
+    GF_ASSERT(words);
+    GF_ASSERT(dict);
+
+    if ((wordcount != 3)) {
+        gf_log("cli", GF_LOG_ERROR, "Invalid Syntax");
+        goto out;
+    }
 
-        int             ret             = -1;
-        gf_answer_t     answer          = GF_ANSWER_NO;
-        const char     *question        = "Deactivating snap will make its "
-                                          "data inaccessible. Do you want to "
-                                          "continue?";
-
-
-        GF_ASSERT (words);
-        GF_ASSERT (dict);
-
-        if ((wordcount != 3)) {
-                gf_log ("cli", GF_LOG_ERROR, "Invalid Syntax");
-                goto out;
-        }
-
-        ret = dict_set_str (dict, "snapname", (char *)words[2]);
-        if (ret) {
-                gf_log ("cli", GF_LOG_ERROR, "Unable to save snap-name %s",
-                        words[2]);
-                goto out;
-        }
-
-        answer = cli_cmd_get_confirmation (state, question);
-        if (GF_ANSWER_NO == answer) {
-                ret = 1;
-                gf_log ("cli", GF_LOG_DEBUG, "User cancelled "
-                        "snapshot deactivate operation");
-                goto out;
-        }
+    ret = dict_set_str(dict, "snapname", (char *)words[2]);
+    if (ret) {
+        gf_log("cli", GF_LOG_ERROR, "Unable to save snap-name %s", words[2]);
+        goto out;
+    }
+
+    answer = cli_cmd_get_confirmation(state, question);
+    if (GF_ANSWER_NO == answer) {
+        ret = 1;
+        gf_log("cli", GF_LOG_DEBUG,
+               "User cancelled "
+               "snapshot deactivate operation");
+        goto out;
+    }
 
 out:
-        return ret;
+    return ret;
 }
 
 /* snapshot delete (all | snapname | volume <volname>)
@@ -4344,78 +4778,84 @@ out:
  *                 1 if user cancel the operation
  */
 int
-cli_snap_delete_parse (dict_t *dict, const char **words, int wordcount,
-                       struct cli_state *state) {
-
-        int             ret             =       -1;
-        const char      *question       =       NULL;
-        int32_t         cmd             =       -1;
-        unsigned int    cmdi            =       2;
-        gf_answer_t     answer          =       GF_ANSWER_NO;
-
-        GF_ASSERT (words);
-        GF_ASSERT (dict);
+cli_snap_delete_parse(dict_t *dict, const char **words, int wordcount,
+                      struct cli_state *state)
+{
+    int ret = -1;
+    const char *question = NULL;
+    int32_t cmd = -1;
+    unsigned int cmdi = 2;
+    gf_answer_t answer = GF_ANSWER_NO;
 
-        if (wordcount > 4 || wordcount <= cmdi) {
-                gf_log ("cli", GF_LOG_ERROR, "Invalid Syntax");
-                goto out;
-        }
+    GF_ASSERT(words);
+    GF_ASSERT(dict);
 
-        question = "Deleting snap will erase all the information about "
-                   "the snap. Do you still want to continue?";
+    if (wordcount > 4 || wordcount <= cmdi) {
+        gf_log("cli", GF_LOG_ERROR, "Invalid Syntax");
+        goto out;
+    }
 
-        if (strcmp (words [cmdi], "all") == 0) {
-                ret = 0;
-                cmd = GF_SNAP_DELETE_TYPE_ALL;
-        } else if (strcmp (words [cmdi], "volume") == 0) {
-                if (++cmdi == wordcount) {
-                        ret = -1;
-                        gf_log ("cli", GF_LOG_ERROR, "Invalid Syntax");
-                        goto out;
-                }
+    question =
+        "Deleting snap will erase all the information about "
+        "the snap. Do you still want to continue?";
 
-                ret = dict_set_str (dict, "volname",
-                                    (char *)words[cmdi]);
-                if (ret) {
-                        gf_log ("cli", GF_LOG_ERROR, "Could not save "
-                                "volume name %s", words[wordcount - 1]);
-                        goto out;
-                }
-                cmd = GF_SNAP_DELETE_TYPE_VOL;
-        } else {
-                ret = dict_set_str (dict, "snapname", (char *)words[cmdi]);
-                if (ret) {
-                        gf_log ("cli", GF_LOG_ERROR, "Unable to save "
-                                "snapname %s", words[2]);
-                        goto out;
-                }
-                cmd = GF_SNAP_DELETE_TYPE_SNAP;
+    if (strcmp(words[cmdi], "all") == 0) {
+        ret = 0;
+        cmd = GF_SNAP_DELETE_TYPE_ALL;
+    } else if (strcmp(words[cmdi], "volume") == 0) {
+        if (++cmdi == wordcount) {
+            ret = -1;
+            gf_log("cli", GF_LOG_ERROR, "Invalid Syntax");
+            goto out;
         }
 
-        if ((cmdi + 1) != wordcount) {
-                ret = -1;
-                gf_log ("cli", GF_LOG_ERROR, "Invalid Syntax");
-                goto out;
+        ret = dict_set_str(dict, "volname", (char *)words[cmdi]);
+        if (ret) {
+            gf_log("cli", GF_LOG_ERROR,
+                   "Could not save "
+                   "volume name %s",
+                   words[wordcount - 1]);
+            goto out;
+        }
+        cmd = GF_SNAP_DELETE_TYPE_VOL;
+    } else {
+        ret = dict_set_str(dict, "snapname", (char *)words[cmdi]);
+        if (ret) {
+            gf_log("cli", GF_LOG_ERROR,
+                   "Unable to save "
+                   "snapname %s",
+                   words[2]);
+            goto out;
         }
+        cmd = GF_SNAP_DELETE_TYPE_SNAP;
+    }
 
-        if (cmd == GF_SNAP_DELETE_TYPE_SNAP) {
-                answer = cli_cmd_get_confirmation (state, question);
-                if (GF_ANSWER_NO == answer) {
-                        ret = 1;
-                        gf_log ("cli", GF_LOG_DEBUG, "User cancelled "
-                                "snapshot delete operation for snap %s",
-                                (char *)words[2]);
-                        goto out;
-                }
-        }
+    if ((cmdi + 1) != wordcount) {
+        ret = -1;
+        gf_log("cli", GF_LOG_ERROR, "Invalid Syntax");
+        goto out;
+    }
 
-        ret = dict_set_int32 (dict, "sub-cmd", cmd);
-        if (ret) {
-                gf_log ("cli", GF_LOG_ERROR, "Could not save "
-                        "type of snapshot delete");
-        }
+    if (cmd == GF_SNAP_DELETE_TYPE_SNAP) {
+        answer = cli_cmd_get_confirmation(state, question);
+        if (GF_ANSWER_NO == answer) {
+            ret = 1;
+            gf_log("cli", GF_LOG_DEBUG,
+                   "User cancelled "
+                   "snapshot delete operation for snap %s",
+                   (char *)words[2]);
+            goto out;
+        }
+    }
+
+    ret = dict_set_int32(dict, "sub-cmd", cmd);
+    if (ret) {
+        gf_log("cli", GF_LOG_ERROR,
+               "Could not save "
+               "type of snapshot delete");
+    }
 out:
-        return ret;
+    return ret;
 }
 
 /* snapshot status [(snapname | volume <volname>)]
@@ -4427,132 +4867,138 @@ out:
  *                 0 on success
  */
 int
-cli_snap_status_parse (dict_t *dict, const char **words, int wordcount)
+cli_snap_status_parse(dict_t *dict, const char **words, int wordcount)
 {
+    int ret = -1;
+    int32_t cmd = GF_SNAP_STATUS_TYPE_ALL;
+    unsigned int cmdi = 2;
+    /* cmdi is command index, here cmdi is "2" (gluster snapshot status)*/
 
-        int             ret  =        -1;
-        int32_t         cmd  =       GF_SNAP_STATUS_TYPE_ALL;
-        unsigned int    cmdi =        2;
-        /* cmdi is command index, here cmdi is "2" (gluster snapshot status)*/
+    GF_ASSERT(words);
+    GF_ASSERT(dict);
 
-        GF_ASSERT (words);
-        GF_ASSERT (dict);
+    if (wordcount > 4 || wordcount < cmdi) {
+        gf_log("cli", GF_LOG_ERROR, "Invalid Syntax");
+        goto out;
+    }
 
-        if (wordcount > 4 || wordcount < cmdi) {
-                gf_log ("cli", GF_LOG_ERROR, "Invalid Syntax");
-                goto out;
-        }
+    if (wordcount == cmdi) {
+        ret = 0;
+        goto out;
+    }
 
-        if (wordcount == cmdi) {
-                ret = 0;
-                goto out;
+    /* if 3rd word is not "volume", then it must be "snapname"
+     */
+    if (strcmp(words[cmdi], "volume") != 0) {
+        ret = dict_set_str(dict, "snapname", (char *)words[cmdi]);
+        if (ret) {
+            gf_log("cli", GF_LOG_ERROR,
+                   "Count not save "
+                   "snap name %s",
+                   words[cmdi]);
+            goto out;
         }
 
-        /* if 3rd word is not "volume", then it must be "snapname"
-        */
-        if (strcmp (words[cmdi], "volume") != 0) {
-                ret = dict_set_str (dict, "snapname",
-                                   (char *)words[cmdi]);
-                if (ret) {
-                        gf_log ("cli", GF_LOG_ERROR, "Count not save "
-                                "snap name %s", words[cmdi]);
-                        goto out;
-                }
-
-                if ((cmdi + 1) != wordcount) {
-                        ret = -1;
-                        gf_log ("cli", GF_LOG_ERROR, "Invalid Syntax");
-                        goto out;
-                }
-
-                ret = 0;
-                cmd = GF_SNAP_STATUS_TYPE_SNAP;
-                goto out;
+        if ((cmdi + 1) != wordcount) {
+            ret = -1;
+            gf_log("cli", GF_LOG_ERROR, "Invalid Syntax");
+            goto out;
         }
 
-        /* If 3rd word is "volume", then check if next word is present.
-         * As, "snapshot info volume" is an invalid command
-         */
-        if ((cmdi + 1) == wordcount) {
-                ret = -1;
-                gf_log ("cli", GF_LOG_ERROR, "Invalid Syntax");
-                goto out;
-        }
+        ret = 0;
+        cmd = GF_SNAP_STATUS_TYPE_SNAP;
+        goto out;
+    }
 
-        ret = dict_set_str (dict, "volname", (char *)words [wordcount - 1]);
-        if (ret) {
-                gf_log ("cli", GF_LOG_ERROR, "Count not save "
-                        "volume name %s", words[wordcount - 1]);
-                goto out;
-        }
-        cmd = GF_SNAP_STATUS_TYPE_VOL;
+    /* If 3rd word is "volume", then check if next word is present.
+     * As, "snapshot info volume" is an invalid command
+     */
+    if ((cmdi + 1) == wordcount) {
+        ret = -1;
+        gf_log("cli", GF_LOG_ERROR, "Invalid Syntax");
+        goto out;
+    }
+
+    ret = dict_set_str(dict, "volname", (char *)words[wordcount - 1]);
+    if (ret) {
+        gf_log("cli", GF_LOG_ERROR,
+               "Count not save "
+               "volume name %s",
+               words[wordcount - 1]);
+        goto out;
+    }
+    cmd = GF_SNAP_STATUS_TYPE_VOL;
 
 out:
-        if (ret == 0) {
-                ret = dict_set_int32 (dict, "sub-cmd", cmd);
-                if (ret) {
-                        gf_log ("cli", GF_LOG_ERROR, "Could not save cmd "
-                                "of snapshot status");
-                }
+    if (ret == 0) {
+        ret = dict_set_int32(dict, "sub-cmd", cmd);
+        if (ret) {
+            gf_log("cli", GF_LOG_ERROR,
+                   "Could not save cmd "
+                   "of snapshot status");
         }
+    }
 
-        return ret;
+    return ret;
 }
 
-
 /* return value:
  *      -1  in case of failure.
  *       0  in case of success.
  */
 int32_t
-cli_snap_config_limit_parse (const char **words, dict_t *dict,
-                             unsigned int wordcount, unsigned int index,
-                             char *key)
+cli_snap_config_limit_parse(const char **words, dict_t *dict,
+                            unsigned int wordcount, unsigned int index,
+                            char *key)
 {
-        int             ret             = -1;
-        int             limit           = 0;
-        char            *end_ptr        = NULL;
-
-        GF_ASSERT (words);
-        GF_ASSERT (dict);
-        GF_ASSERT (key);
+    int ret = -1;
+    int limit = 0;
+    char *end_ptr = NULL;
 
-        if (index >= wordcount) {
-                ret = -1;
-                cli_err ("Please provide a value for %s.", key);
-                gf_log ("cli", GF_LOG_ERROR, "Value not provided for %s", key);
-                goto out;
-        }
+    GF_ASSERT(words);
+    GF_ASSERT(dict);
+    GF_ASSERT(key);
 
-        limit = strtol (words[index], &end_ptr, 10);
+    if (index >= wordcount) {
+        ret = -1;
+        cli_err("Please provide a value for %s.", key);
+        gf_log("cli", GF_LOG_ERROR, "Value not provided for %s", key);
+        goto out;
+    }
 
-        if (limit <= 0 || strcmp (end_ptr, "") != 0) {
-                ret = -1;
-                cli_err("Please enter an integer value "
-                        "greater than zero for %s", key);
-                goto out;
-        }
+    limit = strtol(words[index], &end_ptr, 10);
 
-        ret = dict_set_int32 (dict, key, limit);
-        if (ret) {
-                gf_log ("cli", GF_LOG_ERROR, "Could not set "
-                        "%s in dictionary", key);
-                goto out;
-        }
+    if (limit <= 0 || strcmp(end_ptr, "") != 0) {
+        ret = -1;
+        cli_err(
+            "Please enter an integer value "
+            "greater than zero for %s",
+            key);
+        goto out;
+    }
+
+    ret = dict_set_int32(dict, key, limit);
+    if (ret) {
+        gf_log("cli", GF_LOG_ERROR,
+               "Could not set "
+               "%s in dictionary",
+               key);
+        goto out;
+    }
 
-        ret = dict_set_dynstr_with_alloc (dict, "globalname", "All");
-        if (ret) {
-                gf_log ("cli", GF_LOG_ERROR, "Could not set global key");
-                goto out;
-        }
-        ret = dict_set_int32 (dict, "hold_global_locks", _gf_true);
-        if (ret) {
-                gf_log ("cli", GF_LOG_ERROR, "Could not set global locks");
-                goto out;
-        }
+    ret = dict_set_dynstr_with_alloc(dict, "globalname", "All");
+    if (ret) {
+        gf_log("cli", GF_LOG_ERROR, "Could not set global key");
+        goto out;
+    }
+    ret = dict_set_int32(dict, "hold_global_locks", _gf_true);
+    if (ret) {
+        gf_log("cli", GF_LOG_ERROR, "Could not set global locks");
+        goto out;
+    }
 
 out:
-        return ret;
+    return ret;
 }
 
 /* function cli_snap_config_parse
@@ -4568,763 +5014,933 @@ out:
   NOTE : snap-max-soft-limit can only be set for system.
 */
 int32_t
-cli_snap_config_parse (const char **words, int wordcount, dict_t *dict,
-                       struct cli_state *state)
+cli_snap_config_parse(const char **words, int wordcount, dict_t *dict,
+                      struct cli_state *state)
 {
-        int                            ret                 = -1;
-        gf_answer_t                    answer              = GF_ANSWER_NO;
-        gf_boolean_t                   vol_presence        = _gf_false;
-        struct snap_config_opt_vals_  *conf_vals           = NULL;
-        int8_t                         hard_limit          = 0;
-        int8_t                         soft_limit          = 0;
-        int8_t                         config_type         = -1;
-        const char                    *question            = NULL;
-        unsigned int                   cmdi                = 2;
-        /* cmdi is command index, here cmdi is "2" (gluster snapshot config)*/
-
-        GF_ASSERT (words);
-        GF_ASSERT (dict);
-        GF_ASSERT (state);
-
-        if ((wordcount < 2) || (wordcount > 7)) {
-                gf_log ("cli", GF_LOG_ERROR,
-                        "Invalid wordcount(%d)", wordcount);
-                goto out;
-        }
+    int ret = -1;
+    gf_answer_t answer = GF_ANSWER_NO;
+    gf_boolean_t vol_presence = _gf_false;
+    struct snap_config_opt_vals_ *conf_vals = NULL;
+    int8_t hard_limit = 0;
+    int8_t soft_limit = 0;
+    int8_t config_type = -1;
+    const char *question = NULL;
+    unsigned int cmdi = 2;
+    /* cmdi is command index, here cmdi is "2" (gluster snapshot config)*/
+
+    GF_ASSERT(words);
+    GF_ASSERT(dict);
+    GF_ASSERT(state);
+
+    if ((wordcount < 2) || (wordcount > 7)) {
+        gf_log("cli", GF_LOG_ERROR, "Invalid wordcount(%d)", wordcount);
+        goto out;
+    }
 
-        if (wordcount == 2) {
-                config_type = GF_SNAP_CONFIG_DISPLAY;
-                ret = 0;
-                goto set;
+    if (wordcount == 2) {
+        config_type = GF_SNAP_CONFIG_DISPLAY;
+        ret = 0;
+        goto set;
+    }
+
+    /* auto-delete cannot be a volume name */
+    /* Check whether the 3rd word is volname */
+    if (strcmp(words[cmdi], "snap-max-hard-limit") != 0 &&
+        strcmp(words[cmdi], "snap-max-soft-limit") != 0 &&
+        strcmp(words[cmdi], "auto-delete") != 0 &&
+        strcmp(words[cmdi], "activate-on-create") != 0) {
+        ret = dict_set_str(dict, "volname", (char *)words[cmdi]);
+        if (ret) {
+            gf_log("cli", GF_LOG_ERROR, "Failed to set volname");
+            goto out;
         }
+        cmdi++;
+        vol_presence = _gf_true;
 
-        /* auto-delete cannot be a volume name */
-        /* Check whether the 3rd word is volname */
-        if (strcmp (words[cmdi], "snap-max-hard-limit") != 0
-             && strcmp (words[cmdi], "snap-max-soft-limit") != 0
-             && strcmp (words[cmdi], "auto-delete") != 0
-             && strcmp (words[cmdi], "activate-on-create") != 0) {
-                ret = dict_set_str (dict, "volname", (char *)words[cmdi]);
-                if (ret) {
-                        gf_log ("cli", GF_LOG_ERROR, "Failed to set volname");
-                        goto out;
-                }
-                cmdi++;
-                vol_presence = _gf_true;
-
-                if (cmdi == wordcount) {
-                        config_type = GF_SNAP_CONFIG_DISPLAY;
-                        ret = 0;
-                        goto set;
-                }
+        if (cmdi == wordcount) {
+            config_type = GF_SNAP_CONFIG_DISPLAY;
+            ret = 0;
+            goto set;
         }
+    }
 
-        config_type = GF_SNAP_CONFIG_TYPE_SET;
-
-        if (strcmp (words[cmdi], "snap-max-hard-limit") == 0) {
-                ret = cli_snap_config_limit_parse (words, dict, wordcount,
-                                                ++cmdi, "snap-max-hard-limit");
-                if (ret) {
-                        gf_log ("cli", GF_LOG_ERROR, "Failed to parse snap "
-                                "config hard limit");
-                        goto out;
-                }
-                hard_limit = 1;
+    config_type = GF_SNAP_CONFIG_TYPE_SET;
 
-                if (++cmdi == wordcount) {
-                        ret = 0;
-                        goto set;
-                }
+    if (strcmp(words[cmdi], "snap-max-hard-limit") == 0) {
+        ret = cli_snap_config_limit_parse(words, dict, wordcount, ++cmdi,
+                                          "snap-max-hard-limit");
+        if (ret) {
+            gf_log("cli", GF_LOG_ERROR,
+                   "Failed to parse snap "
+                   "config hard limit");
+            goto out;
         }
+        hard_limit = 1;
 
-        if (strcmp (words[cmdi], "snap-max-soft-limit") == 0) {
-                if (vol_presence == 1) {
-                        ret = -1;
-                        cli_err ("Soft limit cannot be set to individual "
-                                  "volumes.");
-                        gf_log ("cli", GF_LOG_ERROR, "Soft limit cannot be "
-                                "set to volumes");
-                        goto out;
-                }
-
-                ret = cli_snap_config_limit_parse (words, dict, wordcount,
-                                                ++cmdi, "snap-max-soft-limit");
-                if (ret) {
-                        gf_log ("cli", GF_LOG_ERROR, "Failed to parse snap "
-                                "config soft limit");
-                        goto out;
-                }
-
-                if (++cmdi != wordcount) {
-                        ret = -1;
-                        gf_log ("cli", GF_LOG_ERROR, "Invalid Syntax");
-                        goto out;
-                }
-                soft_limit = 1;
+        if (++cmdi == wordcount) {
+            ret = 0;
+            goto set;
         }
+    }
 
-        if (hard_limit || soft_limit)
-                goto set;
+    if (strcmp(words[cmdi], "snap-max-soft-limit") == 0) {
+        if (vol_presence == 1) {
+            ret = -1;
+            cli_err(
+                "Soft limit cannot be set to individual "
+                "volumes.");
+            gf_log("cli", GF_LOG_ERROR,
+                   "Soft limit cannot be "
+                   "set to volumes");
+            goto out;
+        }
 
-        if (strcmp(words[cmdi], "auto-delete") == 0) {
-                if (vol_presence == 1) {
-                        ret = -1;
-                        cli_err ("As of now, auto-delete option cannot be set "
-                                "to volumes");
-                        gf_log ("cli", GF_LOG_ERROR, "auto-delete option "
-                                "cannot be set to volumes");
-                        goto out;
-                }
+        ret = cli_snap_config_limit_parse(words, dict, wordcount, ++cmdi,
+                                          "snap-max-soft-limit");
+        if (ret) {
+            gf_log("cli", GF_LOG_ERROR,
+                   "Failed to parse snap "
+                   "config soft limit");
+            goto out;
+        }
 
-                if (++cmdi >= wordcount) {
-                        ret = -1;
-                        gf_log ("cli", GF_LOG_ERROR, "Invalid Syntax");
-                        goto out;
-                }
+        if (++cmdi != wordcount) {
+            ret = -1;
+            gf_log("cli", GF_LOG_ERROR, "Invalid Syntax");
+            goto out;
+        }
+        soft_limit = 1;
+    }
 
-                ret = dict_set_str (dict, "auto-delete", (char *)words[cmdi]);
-                if (ret) {
-                        gf_log ("cli", GF_LOG_ERROR, "Failed to set "
-                                "value of auto-delete in request "
-                                "dictionary");
-                        goto out;
-                }
+    if (hard_limit || soft_limit)
+        goto set;
 
-                if (++cmdi != wordcount) {
-                        ret = -1;
-                        gf_log ("cli", GF_LOG_ERROR, "Invalid Syntax");
-                        goto out;
-                }
-        } else if (strcmp(words[cmdi], "activate-on-create") == 0) {
-                if (vol_presence == 1) {
-                        ret = -1;
-                        cli_err ("As of now, activate-on-create option "
-                                 "cannot be set to volumes");
-                        gf_log ("cli", GF_LOG_ERROR, "activate-on-create "
-                                "option cannot be set to volumes");
-                        goto out;
-                }
+    if (strcmp(words[cmdi], "auto-delete") == 0) {
+        if (vol_presence == 1) {
+            ret = -1;
+            cli_err(
+                "As of now, auto-delete option cannot be set "
+                "to volumes");
+            gf_log("cli", GF_LOG_ERROR,
+                   "auto-delete option "
+                   "cannot be set to volumes");
+            goto out;
+        }
 
-                if (++cmdi >= wordcount) {
-                        ret = -1;
-                        gf_log ("cli", GF_LOG_ERROR, "Invalid Syntax");
-                        goto out;
-                }
+        if (++cmdi >= wordcount) {
+            ret = -1;
+            gf_log("cli", GF_LOG_ERROR, "Invalid Syntax");
+            goto out;
+        }
 
-                ret = dict_set_str (dict, "snap-activate-on-create",
-                                    (char *)words[cmdi]);
-                if (ret) {
-                        gf_log ("cli", GF_LOG_ERROR, "Failed to set value "
-                                "of activate-on-create in request dictionary");
-                        goto out;
-                }
+        ret = dict_set_str(dict, "auto-delete", (char *)words[cmdi]);
+        if (ret) {
+            gf_log("cli", GF_LOG_ERROR,
+                   "Failed to set "
+                   "value of auto-delete in request "
+                   "dictionary");
+            goto out;
+        }
+
+        if (++cmdi != wordcount) {
+            ret = -1;
+            gf_log("cli", GF_LOG_ERROR, "Invalid Syntax");
+            goto out;
+        }
+    } else if (strcmp(words[cmdi], "activate-on-create") == 0) {
+        if (vol_presence == 1) {
+            ret = -1;
+            cli_err(
+                "As of now, activate-on-create option "
+                "cannot be set to volumes");
+            gf_log("cli", GF_LOG_ERROR,
+                   "activate-on-create "
+                   "option cannot be set to volumes");
+            goto out;
+        }
+
+        if (++cmdi >= wordcount) {
+            ret = -1;
+            gf_log("cli", GF_LOG_ERROR, "Invalid Syntax");
+            goto out;
+        }
+
+        ret = dict_set_str(dict, "snap-activate-on-create",
+                           (char *)words[cmdi]);
+        if (ret) {
+            gf_log("cli", GF_LOG_ERROR,
+                   "Failed to set value "
+                   "of activate-on-create in request dictionary");
+            goto out;
+        }
 
-                if (++cmdi != wordcount) {
-                        ret = -1;
-                        gf_log ("cli", GF_LOG_ERROR, "Invalid Syntax");
-                        goto out;
-                }
-        } else {
-                ret = -1;
-                gf_log ("cli", GF_LOG_ERROR, "Invalid Syntax");
-                goto out;
+        if (++cmdi != wordcount) {
+            ret = -1;
+            gf_log("cli", GF_LOG_ERROR, "Invalid Syntax");
+            goto out;
         }
+    } else {
+        ret = -1;
+        gf_log("cli", GF_LOG_ERROR, "Invalid Syntax");
+        goto out;
+    }
 
-        ret = 0; /* Success */
+    ret = 0; /* Success */
 
 set:
-        ret = dict_set_int32 (dict, "config-command", config_type);
-        if (ret) {
-                gf_log ("cli", GF_LOG_ERROR, "Unable to set "
-                        "config-command");
-                goto out;
-        }
+    ret = dict_set_int32(dict, "config-command", config_type);
+    if (ret) {
+        gf_log("cli", GF_LOG_ERROR,
+               "Unable to set "
+               "config-command");
+        goto out;
+    }
 
-        if (config_type == GF_SNAP_CONFIG_TYPE_SET &&
-           (hard_limit || soft_limit)) {
-                conf_vals = snap_confopt_vals;
-                if (hard_limit && soft_limit) {
-                        question = conf_vals[GF_SNAP_CONFIG_SET_BOTH].question;
-                } else if (soft_limit) {
-                        question = conf_vals[GF_SNAP_CONFIG_SET_SOFT].question;
-                } else if (hard_limit) {
-                        question = conf_vals[GF_SNAP_CONFIG_SET_HARD].question;
-                }
+    if (config_type == GF_SNAP_CONFIG_TYPE_SET && (hard_limit || soft_limit)) {
+        conf_vals = snap_confopt_vals;
+        if (hard_limit && soft_limit) {
+            question = conf_vals[GF_SNAP_CONFIG_SET_BOTH].question;
+        } else if (soft_limit) {
+            question = conf_vals[GF_SNAP_CONFIG_SET_SOFT].question;
+        } else if (hard_limit) {
+            question = conf_vals[GF_SNAP_CONFIG_SET_HARD].question;
+        }
 
-                answer = cli_cmd_get_confirmation (state, question);
-                if (GF_ANSWER_NO == answer) {
-                        ret = 1;
-                        gf_log ("cli", GF_LOG_DEBUG, "User cancelled "
-                        "snapshot config operation");
-                }
+        answer = cli_cmd_get_confirmation(state, question);
+        if (GF_ANSWER_NO == answer) {
+            ret = 1;
+            gf_log("cli", GF_LOG_DEBUG,
+                   "User cancelled "
+                   "snapshot config operation");
         }
+    }
 
 out:
-        return ret;
+    return ret;
 }
 
 int
-validate_op_name (const char *op, const char *opname, char **opwords) {
-        int     ret     =       -1;
-        int     i       =       0;
+validate_op_name(const char *op, const char *opname, char **opwords)
+{
+    int ret = -1;
+    int i = 0;
 
-        GF_ASSERT (opname);
-        GF_ASSERT (opwords);
+    GF_ASSERT(opname);
+    GF_ASSERT(opwords);
 
-        for (i = 0 ; opwords[i] != NULL; i++) {
-                if (strcmp (opwords[i], opname) == 0) {
-                        cli_out ("\"%s\" cannot be a %s", opname, op);
-                        goto out;
-                }
+    for (i = 0; opwords[i] != NULL; i++) {
+        if (strcmp(opwords[i], opname) == 0) {
+            cli_out("\"%s\" cannot be a %s", opname, op);
+            goto out;
         }
-        ret = 0;
+    }
+    ret = 0;
 out:
-        return ret;
+    return ret;
 }
 
 int32_t
-cli_cmd_snapshot_parse (const char **words, int wordcount, dict_t **options,
-                        struct cli_state *state)
+cli_cmd_snapshot_parse(const char **words, int wordcount, dict_t **options,
+                       struct cli_state *state)
 {
-        int32_t            ret        = -1;
-        dict_t             *dict      = NULL;
-        gf1_cli_snapshot   type       = GF_SNAP_OPTION_TYPE_NONE;
-        char               *w         = NULL;
-        char               *opwords[] = {"create", "delete", "restore",
-                                        "activate", "deactivate", "list",
-                                        "status", "config", "info", "clone",
+    int32_t ret = -1;
+    dict_t *dict = NULL;
+    gf1_cli_snapshot type = GF_SNAP_OPTION_TYPE_NONE;
+    char *w = NULL;
+    static char *opwords[] = {"create",     "delete", "restore", "activate",
+                              "deactivate", "list",   "status",  "config",
+                              "info",       "clone",  NULL};
+    static char *invalid_snapnames[] = {"description", "force", "volume", "all",
                                         NULL};
-        char               *invalid_snapnames[] = {"description", "force",
-                                                  "volume", "all", NULL};
-        char               *invalid_volnames[]  = {"volume", "type",
-                                                   "subvolumes", "option",
-                                                   "end-volume", "all",
-                                                   "volume_not_in_ring",
-                                                   "description", "force",
-                                                   "snap-max-hard-limit",
-                                                   "snap-max-soft-limit",
-                                                   "auto-delete",
-                                                   "activate-on-create", NULL};
-
-        GF_ASSERT (words);
-        GF_ASSERT (options);
-        GF_ASSERT (state);
-
-        dict = dict_new ();
-        if (!dict)
-                goto out;
-
-        /* Lowest wordcount possible */
-        if (wordcount < 2) {
-                gf_log ("", GF_LOG_ERROR,
-                        "Invalid command: Not enough arguments");
-                goto out;
-        }
-
-        w = str_getunamb (words[1], opwords);
-        if (!w) {
-                /* Checks if the operation is a valid operation */
-                gf_log ("", GF_LOG_ERROR, "Opword Mismatch");
-                goto out;
-        }
+    static char *invalid_volnames[] = {"volume",
+                                       "type",
+                                       "subvolumes",
+                                       "option",
+                                       "end-volume",
+                                       "all",
+                                       "volume_not_in_ring",
+                                       "description",
+                                       "force",
+                                       "snap-max-hard-limit",
+                                       "snap-max-soft-limit",
+                                       "auto-delete",
+                                       "activate-on-create",
+                                       NULL};
+
+    GF_ASSERT(words);
+    GF_ASSERT(options);
+    GF_ASSERT(state);
+
+    dict = dict_new();
+    if (!dict)
+        goto out;
 
-        if (!strcmp (w, "create")) {
-                type = GF_SNAP_OPTION_TYPE_CREATE;
-        } else if (!strcmp (w, "list")) {
-                type = GF_SNAP_OPTION_TYPE_LIST;
-        } else if (!strcmp (w, "info")) {
-                type = GF_SNAP_OPTION_TYPE_INFO;
-        } else if (!strcmp (w, "delete")) {
-                type = GF_SNAP_OPTION_TYPE_DELETE;
-        } else if (!strcmp (w, "config")) {
-                type = GF_SNAP_OPTION_TYPE_CONFIG;
-        } else if (!strcmp (w, "restore")) {
-                type = GF_SNAP_OPTION_TYPE_RESTORE;
-        } else if (!strcmp (w, "status")) {
-                type = GF_SNAP_OPTION_TYPE_STATUS;
-        } else if (!strcmp (w, "activate")) {
-                type = GF_SNAP_OPTION_TYPE_ACTIVATE;
-        } else if (!strcmp (w, "deactivate")) {
-                type = GF_SNAP_OPTION_TYPE_DEACTIVATE;
-        } else if (!strcmp(w, "clone")) {
-                type = GF_SNAP_OPTION_TYPE_CLONE;
-        }
-
-        if (type != GF_SNAP_OPTION_TYPE_CONFIG &&
-            type != GF_SNAP_OPTION_TYPE_STATUS) {
-                ret = dict_set_int32 (dict, "hold_snap_locks", _gf_true);
-                if (ret) {
-                        gf_log ("cli", GF_LOG_ERROR,
-                                "Unable to set hold-snap-locks value "
-                                "as _gf_true");
-                        goto out;
-                }
-        }
+    /* Lowest wordcount possible */
+    if (wordcount < 2) {
+        gf_log("", GF_LOG_ERROR, "Invalid command: Not enough arguments");
+        goto out;
+    }
 
-        /* Following commands does not require volume locks  */
-        if (type == GF_SNAP_OPTION_TYPE_STATUS ||
-            type == GF_SNAP_OPTION_TYPE_ACTIVATE ||
-            type == GF_SNAP_OPTION_TYPE_DEACTIVATE) {
-                ret = dict_set_int32 (dict, "hold_vol_locks", _gf_false);
-                if (ret) {
-                        gf_log ("cli", GF_LOG_ERROR, "Setting volume lock "
-                                "flag failed");
-                        goto out;
-                }
+    w = str_getunamb(words[1], opwords);
+    if (!w) {
+        /* Checks if the operation is a valid operation */
+        gf_log("", GF_LOG_ERROR, "Opword Mismatch");
+        goto out;
+    }
+
+    if (!strcmp(w, "create")) {
+        type = GF_SNAP_OPTION_TYPE_CREATE;
+    } else if (!strcmp(w, "list")) {
+        type = GF_SNAP_OPTION_TYPE_LIST;
+    } else if (!strcmp(w, "info")) {
+        type = GF_SNAP_OPTION_TYPE_INFO;
+    } else if (!strcmp(w, "delete")) {
+        type = GF_SNAP_OPTION_TYPE_DELETE;
+    } else if (!strcmp(w, "config")) {
+        type = GF_SNAP_OPTION_TYPE_CONFIG;
+    } else if (!strcmp(w, "restore")) {
+        type = GF_SNAP_OPTION_TYPE_RESTORE;
+    } else if (!strcmp(w, "status")) {
+        type = GF_SNAP_OPTION_TYPE_STATUS;
+    } else if (!strcmp(w, "activate")) {
+        type = GF_SNAP_OPTION_TYPE_ACTIVATE;
+    } else if (!strcmp(w, "deactivate")) {
+        type = GF_SNAP_OPTION_TYPE_DEACTIVATE;
+    } else if (!strcmp(w, "clone")) {
+        type = GF_SNAP_OPTION_TYPE_CLONE;
+    }
+
+    if (type != GF_SNAP_OPTION_TYPE_CONFIG &&
+        type != GF_SNAP_OPTION_TYPE_STATUS) {
+        ret = dict_set_int32(dict, "hold_snap_locks", _gf_true);
+        if (ret) {
+            gf_log("cli", GF_LOG_ERROR,
+                   "Unable to set hold-snap-locks value "
+                   "as _gf_true");
+            goto out;
+        }
+    }
+
+    /* Following commands does not require volume locks  */
+    if (type == GF_SNAP_OPTION_TYPE_STATUS ||
+        type == GF_SNAP_OPTION_TYPE_ACTIVATE ||
+        type == GF_SNAP_OPTION_TYPE_DEACTIVATE) {
+        ret = dict_set_int32(dict, "hold_vol_locks", _gf_false);
+        if (ret) {
+            gf_log("cli", GF_LOG_ERROR,
+                   "Setting volume lock "
+                   "flag failed");
+            goto out;
         }
+    }
 
-        /* Check which op is intended */
-        switch (type) {
+    /* Check which op is intended */
+    switch (type) {
         case GF_SNAP_OPTION_TYPE_CREATE:
-                /* Syntax :
-                 * gluster snapshot create <snapname> <vol-name(s)>
-                 *                         [no-timestamp]
-                 *                         [description <description>]
-                 *                         [force]
-                 */
-                /* In cases where the snapname is not given then
-                 * parsing fails & snapname cannot be "description",
-                 * "force" and "volume", that check is made here
-                 */
-                if (wordcount == 2){
-                        ret = -1;
-                        gf_log ("cli", GF_LOG_ERROR, "Invalid Syntax");
-                        goto out;
-                }
+            /* Syntax :
+             * gluster snapshot create <snapname> <vol-name(s)>
+             *                         [no-timestamp]
+             *                         [description <description>]
+             *                         [force]
+             */
+            /* In cases where the snapname is not given then
+             * parsing fails & snapname cannot be "description",
+             * "force" and "volume", that check is made here
+             */
+            if (wordcount == 2) {
+                ret = -1;
+                gf_log("cli", GF_LOG_ERROR, "Invalid Syntax");
+                goto out;
+            }
 
-                ret = validate_op_name ("snapname", words[2],
-                                        invalid_snapnames);
-                if (ret) {
-                        goto out;
-                }
+            ret = validate_op_name("snapname", words[2], invalid_snapnames);
+            if (ret) {
+                goto out;
+            }
 
-                ret = cli_snap_create_parse (dict, words, wordcount);
-                if (ret) {
-                        gf_log ("cli", GF_LOG_ERROR,
-                                "create command parsing failed.");
-                        goto out;
-                }
-                break;
+            ret = cli_snap_create_parse(dict, words, wordcount);
+            if (ret) {
+                gf_log("cli", GF_LOG_ERROR, "create command parsing failed.");
+                goto out;
+            }
+            break;
 
         case GF_SNAP_OPTION_TYPE_CLONE:
-                /* Syntax :
-                 * gluster snapshot clone <clonename> <snapname>
-                 */
-                /* In cases where the clonename is not given then
-                 * parsing fails & snapname cannot be "description",
-                 * "force" and "volume", that check is made here
-                 */
-                if (wordcount == 2) {
-                        ret = -1;
-                        gf_log ("cli", GF_LOG_ERROR, "Invalid Syntax");
-                        goto out;
-                }
-
-                ret = validate_op_name ("clonename", words[2],
-                                        invalid_volnames);
-                if (ret) {
-                        goto out;
-                }
+            /* Syntax :
+             * gluster snapshot clone <clonename> <snapname>
+             */
+            /* In cases where the clonename is not given then
+             * parsing fails & snapname cannot be "description",
+             * "force" and "volume", that check is made here
+             */
+            if (wordcount == 2) {
+                ret = -1;
+                gf_log("cli", GF_LOG_ERROR, "Invalid Syntax");
+                goto out;
+            }
 
-                ret = cli_snap_clone_parse (dict, words, wordcount);
-                if (ret) {
-                        gf_log ("cli", GF_LOG_ERROR,
-                                "clone command parsing failed.");
-                        goto out;
-                }
-                break;
+            ret = validate_op_name("clonename", words[2], invalid_volnames);
+            if (ret) {
+                goto out;
+            }
 
+            ret = cli_snap_clone_parse(dict, words, wordcount);
+            if (ret) {
+                gf_log("cli", GF_LOG_ERROR, "clone command parsing failed.");
+                goto out;
+            }
+            break;
 
         case GF_SNAP_OPTION_TYPE_INFO:
-                /* Syntax :
-                 * gluster snapshot info [(snapname] | [vol <volname>)]
-                 */
-                ret = cli_snap_info_parse (dict, words, wordcount);
-                if (ret) {
-                        gf_log ("cli", GF_LOG_ERROR, "Failed to parse "
-                                "snapshot info command");
-                        goto out;
-                }
-                break;
+            /* Syntax :
+             * gluster snapshot info [(snapname] | [vol <volname>)]
+             */
+            ret = cli_snap_info_parse(dict, words, wordcount);
+            if (ret) {
+                gf_log("cli", GF_LOG_ERROR,
+                       "Failed to parse "
+                       "snapshot info command");
+                goto out;
+            }
+            break;
 
         case GF_SNAP_OPTION_TYPE_LIST:
-                /* Syntax :
-                 * gluster snaphsot list [volname]
-                 */
+            /* Syntax :
+             * gluster snaphsot list [volname]
+             */
 
-                ret = cli_snap_list_parse (dict, words, wordcount);
-                if (ret) {
-                        gf_log ("cli", GF_LOG_ERROR, "Failed to parse "
-                                "snapshot list command");
-                        goto out;
-                }
-                break;
+            ret = cli_snap_list_parse(dict, words, wordcount);
+            if (ret) {
+                gf_log("cli", GF_LOG_ERROR,
+                       "Failed to parse "
+                       "snapshot list command");
+                goto out;
+            }
+            break;
 
         case GF_SNAP_OPTION_TYPE_DELETE:
-                /* Syntax :
-                 * snapshot delete (all | snapname | volume <volname>)
-                 */
-                ret = cli_snap_delete_parse (dict, words, wordcount, state);
-                if (ret) {
-                        /* A positive ret value means user cancelled
-                        * the command */
-                        if (ret < 0) {
-                                gf_log ("cli", GF_LOG_ERROR, "Failed to parse "
-                                        "snapshot delete command");
-                        }
-                        goto out;
+            /* Syntax :
+             * snapshot delete (all | snapname | volume <volname>)
+             */
+            ret = cli_snap_delete_parse(dict, words, wordcount, state);
+            if (ret) {
+                /* A positive ret value means user cancelled
+                 * the command */
+                if (ret < 0) {
+                    gf_log("cli", GF_LOG_ERROR,
+                           "Failed to parse "
+                           "snapshot delete command");
                 }
-                break;
+                goto out;
+            }
+            break;
 
         case GF_SNAP_OPTION_TYPE_CONFIG:
-                /* snapshot config [volname]  [snap-max-hard-limit <count>]
-                 *                            [snap-max-soft-limit <percent>] */
-                ret = cli_snap_config_parse (words, wordcount, dict, state);
-                if (ret) {
-                        if (ret < 0)
-                                gf_log ("cli", GF_LOG_ERROR,
-                                        "config command parsing failed.");
-                        goto out;
-                }
-
-                ret = dict_set_int32 (dict, "type", GF_SNAP_OPTION_TYPE_CONFIG);
-                if (ret) {
-                        gf_log ("cli", GF_LOG_ERROR, "Unable to set "
-                                "config type");
-                        ret = -1;
-                        goto out;
-                }
-                break;
-
-        case GF_SNAP_OPTION_TYPE_STATUS:
-                {
-                        /* Syntax :
-                         * gluster snapshot status [(snapname |
-                         *                         volume <volname>)]
-                         */
-                        ret = cli_snap_status_parse (dict, words, wordcount);
-                        if (ret) {
-                                gf_log ("cli", GF_LOG_ERROR, "Failed to parse "
-                                        "snapshot status command");
-                                goto out;
-                        }
-                        break;
-                }
+            /* snapshot config [volname]  [snap-max-hard-limit <count>]
+             *                            [snap-max-soft-limit <percent>] */
+            ret = cli_snap_config_parse(words, wordcount, dict, state);
+            if (ret) {
+                if (ret < 0)
+                    gf_log("cli", GF_LOG_ERROR,
+                           "config command parsing failed.");
+                goto out;
+            }
 
-        case GF_SNAP_OPTION_TYPE_RESTORE:
-                /* Syntax:
-                 * snapshot restore <snapname>
-                 */
-                ret = cli_snap_restore_parse (dict, words, wordcount, state);
-                if (ret) {
-                        gf_log ("cli", GF_LOG_ERROR, "Failed to parse "
-                                "restore command");
-                        goto out;
-                }
-                break;
-
-                case GF_SNAP_OPTION_TYPE_ACTIVATE:
-                        /* Syntax:
-                        * snapshot activate <snapname> [force]
-                        */
-                        ret = cli_snap_activate_parse (dict, words, wordcount);
-                        if (ret) {
-                                gf_log ("cli", GF_LOG_ERROR, "Failed to parse "
-                                        "start command");
-                                goto out;
-                        }
-                        break;
-                case GF_SNAP_OPTION_TYPE_DEACTIVATE:
-                        /* Syntax:
-                        * snapshot deactivate <snapname>
-                        */
-                        ret = cli_snap_deactivate_parse (dict, words, wordcount,
-                                state);
-                        if (ret) {
-                                /* A positive ret value means user cancelled
-                                 * the command */
-                                if (ret < 0) {
-                                        gf_log ("cli", GF_LOG_ERROR,
-                                                "Failed to parse deactivate "
-                                                "command");
-                                }
-                                goto out;
-                        }
-                        break;
+            ret = dict_set_int32(dict, "type", GF_SNAP_OPTION_TYPE_CONFIG);
+            if (ret) {
+                gf_log("cli", GF_LOG_ERROR,
+                       "Unable to set "
+                       "config type");
+                ret = -1;
+                goto out;
+            }
+            break;
 
-        default:
-                gf_log ("", GF_LOG_ERROR, "Opword Mismatch");
+        case GF_SNAP_OPTION_TYPE_STATUS: {
+            /* Syntax :
+             * gluster snapshot status [(snapname |
+             *                         volume <volname>)]
+             */
+            ret = cli_snap_status_parse(dict, words, wordcount);
+            if (ret) {
+                gf_log("cli", GF_LOG_ERROR,
+                       "Failed to parse "
+                       "snapshot status command");
                 goto out;
+            }
+            break;
         }
 
-        ret = dict_set_int32 (dict, "type", type);
-        if (ret) {
-                gf_log ("", GF_LOG_ERROR,
-                        "Failed to set type.");
+        case GF_SNAP_OPTION_TYPE_RESTORE:
+            /* Syntax:
+             * snapshot restore <snapname>
+             */
+            ret = cli_snap_restore_parse(dict, words, wordcount, state);
+            if (ret) {
+                gf_log("cli", GF_LOG_ERROR,
+                       "Failed to parse "
+                       "restore command");
+                goto out;
+            }
+            break;
+
+        case GF_SNAP_OPTION_TYPE_ACTIVATE:
+            /* Syntax:
+             * snapshot activate <snapname> [force]
+             */
+            ret = cli_snap_activate_parse(dict, words, wordcount);
+            if (ret) {
+                gf_log("cli", GF_LOG_ERROR,
+                       "Failed to parse "
+                       "start command");
+                goto out;
+            }
+            break;
+        case GF_SNAP_OPTION_TYPE_DEACTIVATE:
+            /* Syntax:
+             * snapshot deactivate <snapname>
+             */
+            ret = cli_snap_deactivate_parse(dict, words, wordcount, state);
+            if (ret) {
+                /* A positive ret value means user cancelled
+                 * the command */
+                if (ret < 0) {
+                    gf_log("cli", GF_LOG_ERROR,
+                           "Failed to parse deactivate "
+                           "command");
+                }
                 goto out;
-        }
-        /* If you got so far, input is valid */
-        ret = 0;
+            }
+            break;
+
+        default:
+            ret = -1;
+            gf_log("", GF_LOG_ERROR, "Opword Mismatch");
+            goto out;
+    }
+
+    ret = dict_set_int32(dict, "type", type);
+    if (ret) {
+        gf_log("", GF_LOG_ERROR, "Failed to set type.");
+        goto out;
+    }
+    /* If you got so far, input is valid */
+    ret = 0;
 out:
-        if (ret) {
-                if (dict)
-                        dict_destroy (dict);
-        } else
-                *options = dict;
+    if (ret) {
+        if (dict)
+            dict_unref(dict);
+    } else
+        *options = dict;
 
-        return ret;
+    return ret;
 }
 
 int
-cli_cmd_validate_volume (char *volname)
+cli_cmd_validate_volume(char *volname)
 {
-        int       i        =  0;
-        int       ret      = -1;
+    int i = 0;
+    int ret = -1;
+    int volname_len;
 
+    if (volname[0] == '-')
+        return ret;
 
-        if (volname[0] == '-')
-                return ret;
+    if (!strcmp(volname, "all")) {
+        cli_err("\"all\" cannot be the name of a volume.");
+        return ret;
+    }
 
-        if (!strcmp (volname, "all")) {
-                cli_err ("\"all\" cannot be the name of a volume.");
-                return ret;
-        }
+    if (strchr(volname, '/')) {
+        cli_err("Volume name should not contain \"/\" character.");
+        return ret;
+    }
 
-        if (strchr (volname, '/')) {
-                cli_err ("Volume name should not contain \"/\" character.");
-                return ret;
-        }
+    volname_len = strlen(volname);
+    if (volname_len > GD_VOLUME_NAME_MAX) {
+        cli_err("Volname can not exceed %d characters.", GD_VOLUME_NAME_MAX);
+        return ret;
+    }
 
-        if (strlen (volname) > GD_VOLUME_NAME_MAX) {
-                cli_err ("Volname can not exceed %d characters.",
-                        GD_VOLUME_NAME_MAX);
-                return ret;
+    for (i = 0; i < volname_len; i++)
+        if (!isalnum(volname[i]) && (volname[i] != '_') &&
+            (volname[i] != '-')) {
+            cli_err(
+                "Volume name should not contain \"%c\""
+                " character.\nVolume names can only"
+                "contain alphanumeric, '-' and '_' "
+                "characters.",
+                volname[i]);
+            return ret;
         }
 
-        for (i = 0; i < strlen (volname); i++)
-                if (!isalnum (volname[i]) && (volname[i] != '_') &&
-                    (volname[i] != '-')) {
-                        cli_err ("Volume name should not contain \"%c\""
-                                 " character.\nVolume names can only"
-                                 "contain alphanumeric, '-' and '_' "
-                                 "characters.", volname[i]);
-                        return ret;
-                }
-
-        ret = 0;
+    ret = 0;
 
-        return ret;
+    return ret;
 }
 
 int32_t
-cli_cmd_bitrot_parse (const char **words, int wordcount, dict_t **options)
+cli_cmd_bitrot_parse(const char **words, int wordcount, dict_t **options)
 {
-        int32_t            ret                    = -1;
-        char               *w                     = NULL;
-        char               *volname               = NULL;
-        char               *opwords[]             = {"enable", "disable",
-                                                     "scrub-throttle",
-                                                     "scrub-frequency", "scrub",
-                                                     "signing-time", NULL};
-        char               *scrub_throt_values[]  = {"lazy", "normal",
-                                                     "aggressive", NULL};
-        char               *scrub_freq_values[]   = {"hourly",
-                                                     "daily", "weekly",
-                                                     "biweekly", "monthly",
-                                                      NULL};
-        char               *scrub_values[]        = {"pause", "resume",
-                                                     "status", NULL};
-        dict_t             *dict                  = NULL;
-        gf_bitrot_type     type                   = GF_BITROT_OPTION_TYPE_NONE;
-        int32_t            expiry_time            = 0;
-
-        GF_ASSERT (words);
-        GF_ASSERT (options);
-
-        dict = dict_new ();
-        if (!dict)
-                goto out;
+    int32_t ret = -1;
+    char *w = NULL;
+    char *volname = NULL;
+    static char *opwords[] = {"enable",          "disable", "scrub-throttle",
+                              "scrub-frequency", "scrub",   "signing-time",
+                              "signer-threads",  NULL};
+    static char *scrub_throt_values[] = {"lazy", "normal", "aggressive", NULL};
+    static char *scrub_freq_values[] = {
+        "hourly", "daily", "weekly", "biweekly", "monthly", "minute", NULL};
+    static char *scrub_values[] = {"pause", "resume", "status", "ondemand",
+                                   NULL};
+    dict_t *dict = NULL;
+    gf_bitrot_type type = GF_BITROT_OPTION_TYPE_NONE;
+    int32_t expiry_time = 0;
+    int32_t signer_th_count = 0;
+
+    GF_ASSERT(words);
+    GF_ASSERT(options);
+
+    /* Hack to print out bitrot help properly */
+    if ((wordcount == 3) && !(strcmp(words[2], "help"))) {
+        ret = 1;
+        return ret;
+    }
 
-        if (wordcount < 4 || wordcount > 5) {
-                gf_log ("cli", GF_LOG_ERROR, "Invalid syntax");
-                goto out;
-        }
+    if (wordcount < 4 || wordcount > 5) {
+        gf_log("cli", GF_LOG_ERROR, "Invalid syntax");
+        goto out;
+    }
 
-        volname = (char *)words[2];
-        if (!volname) {
-                ret = -1;
-                goto out;
-        }
+    dict = dict_new();
+    if (!dict)
+        goto out;
 
-        ret = cli_cmd_validate_volume (volname);
-        if (ret) {
-                gf_log ("cli", GF_LOG_ERROR, "Failed to validate volume name");
-                goto out;
-        }
+    volname = (char *)words[2];
+    if (!volname) {
+        ret = -1;
+        goto out;
+    }
 
-        ret = dict_set_str (dict, "volname", volname);
-        if (ret) {
-                cli_out ("Failed to set volume name in dictionary ");
-                goto out;
+    ret = cli_cmd_validate_volume(volname);
+    if (ret) {
+        gf_log("cli", GF_LOG_ERROR, "Failed to validate volume name");
+        goto out;
+    }
+
+    ret = dict_set_str(dict, "volname", volname);
+    if (ret) {
+        cli_out("Failed to set volume name in dictionary ");
+        goto out;
+    }
+
+    w = str_getunamb(words[3], opwords);
+    if (!w) {
+        cli_out("Invalid bit rot option : %s", words[3]);
+        ret = -1;
+        goto out;
+    }
+
+    if (strcmp(w, "enable") == 0) {
+        if (wordcount == 4) {
+            type = GF_BITROT_OPTION_TYPE_ENABLE;
+            ret = 0;
+            goto set_type;
+        } else {
+            ret = -1;
+            goto out;
         }
+    }
 
-        w = str_getunamb (words[3], opwords);
-        if (!w) {
-                cli_out ("Invalid bit rot option : %s", words[3]);
+    if (strcmp(w, "disable") == 0) {
+        if (wordcount == 4) {
+            type = GF_BITROT_OPTION_TYPE_DISABLE;
+            ret = 0;
+            goto set_type;
+        } else {
+            ret = -1;
+            goto out;
+        }
+    }
+
+    if (!strcmp(w, "scrub-throttle")) {
+        if (!words[4]) {
+            cli_err(
+                "Missing scrub-throttle value for bitrot "
+                "option");
+            ret = -1;
+            goto out;
+        } else {
+            w = str_getunamb(words[4], scrub_throt_values);
+            if (!w) {
+                cli_err(
+                    "Invalid scrub-throttle option for "
+                    "bitrot");
                 ret = -1;
                 goto out;
-        }
-
-        if (strcmp (w, "enable") == 0) {
-                if (wordcount == 4) {
-                        type = GF_BITROT_OPTION_TYPE_ENABLE;
-                        ret = 0;
-                        goto set_type;
-                } else {
-                        ret = -1;
-                        goto out;
+            } else {
+                type = GF_BITROT_OPTION_TYPE_SCRUB_THROTTLE;
+                ret = dict_set_str(dict, "scrub-throttle-value",
+                                   (char *)words[4]);
+                if (ret) {
+                    cli_out(
+                        "Failed to set scrub-throttle "
+                        "value in the dict");
+                    goto out;
                 }
+                goto set_type;
+            }
         }
+    }
 
-        if (strcmp (w, "disable") == 0) {
-                if (wordcount == 4) {
-                        type = GF_BITROT_OPTION_TYPE_DISABLE;
-                        ret = 0;
-                        goto set_type;
-                } else {
-                        ret = -1;
-                        goto out;
+    if (!strcmp(words[3], "scrub-frequency")) {
+        if (!words[4]) {
+            cli_err("Missing scrub-frequency value");
+            ret = -1;
+            goto out;
+        } else {
+            w = str_getunamb(words[4], scrub_freq_values);
+            if (!w) {
+                cli_err("Invalid frequency option for bitrot");
+                ret = -1;
+                goto out;
+            } else {
+                type = GF_BITROT_OPTION_TYPE_SCRUB_FREQ;
+                ret = dict_set_str(dict, "scrub-frequency-value",
+                                   (char *)words[4]);
+                if (ret) {
+                    cli_out(
+                        "Failed to set dict for "
+                        "bitrot");
+                    goto out;
                 }
+                goto set_type;
+            }
         }
+    }
 
-        if (!strcmp (w, "scrub-throttle")) {
-                if (!words[4]) {
-                        cli_err ("Missing scrub-throttle value for bitrot "
-                                 "option");
-                        ret = -1;
-                        goto out;
+    if (!strcmp(words[3], "scrub")) {
+        if (!words[4]) {
+            cli_err("Missing scrub value for bitrot option");
+            ret = -1;
+            goto out;
+        } else {
+            w = str_getunamb(words[4], scrub_values);
+            if (!w) {
+                cli_err("Invalid scrub option for bitrot");
+                ret = -1;
+                goto out;
+            } else {
+                if (strcmp(words[4], "status") == 0) {
+                    type = GF_BITROT_CMD_SCRUB_STATUS;
+                } else if (strcmp(words[4], "ondemand") == 0) {
+                    type = GF_BITROT_CMD_SCRUB_ONDEMAND;
                 } else {
-                        w = str_getunamb (words[4], scrub_throt_values);
-                        if (!w) {
-                                cli_err ("Invalid scrub-throttle option for "
-                                         "bitrot");
-                                ret = -1;
-                                goto out;
-                        } else {
-                                type = GF_BITROT_OPTION_TYPE_SCRUB_THROTTLE;
-                                ret =  dict_set_str (dict,
-                                                     "scrub-throttle-value",
-                                                     (char *) words[4]);
-                                if (ret) {
-                                        cli_out ("Failed to set scrub-throttle "
-                                                 "value in the dict");
-                                        goto out;
-                                }
-                                goto set_type;
-                        }
+                    type = GF_BITROT_OPTION_TYPE_SCRUB;
                 }
-        }
-
-        if (!strcmp (words[3], "scrub-frequency")) {
-                if (!words[4]) {
-                        cli_err ("Missing scrub-frequency value");
-                        ret = -1;
-                        goto out;
-                } else {
-                        w = str_getunamb (words[4], scrub_freq_values);
-                        if (!w) {
-                                cli_err ("Invalid frequency option for bitrot");
-                                ret = -1;
-                                goto out;
-                        } else {
-                                type = GF_BITROT_OPTION_TYPE_SCRUB_FREQ;
-                                ret = dict_set_str (dict,
-                                                    "scrub-frequency-value",
-                                                    (char *) words[4]);
-                                if (ret) {
-                                        cli_out ("Failed to set dict for "
-                                                 "bitrot");
-                                        goto out;
-                                }
-                                goto set_type;
-                        }
+                ret = dict_set_str(dict, "scrub-value", (char *)words[4]);
+                if (ret) {
+                    cli_out(
+                        "Failed to set dict for "
+                        "bitrot");
+                    goto out;
                 }
+                goto set_type;
+            }
         }
+    }
 
-        if (!strcmp (words[3], "scrub")) {
-                if (!words[4]) {
-                        cli_err ("Missing scrub value for bitrot option");
-                        ret = -1;
-                        goto out;
-                } else {
-                        w = str_getunamb (words[4], scrub_values);
-                        if (!w) {
-                                cli_err ("Invalid scrub option for bitrot");
-                                ret = -1;
-                                goto out;
-                        } else {
-                                if (strcmp (words[4], "status") == 0) {
-                                        type = GF_BITROT_CMD_SCRUB_STATUS;
-                                } else {
-                                        type = GF_BITROT_OPTION_TYPE_SCRUB;
-                                }
-                                ret =  dict_set_str (dict, "scrub-value",
-                                                    (char *) words[4]);
-                                if (ret) {
-                                        cli_out ("Failed to set dict for "
-                                                 "bitrot");
-                                        goto out;
-                                }
-                                goto set_type;
-                        }
-                }
-        }
+    if (!strcmp(words[3], "signing-time")) {
+        if (!words[4]) {
+            cli_err(
+                "Missing signing-time value for bitrot "
+                "option");
+            ret = -1;
+            goto out;
+        } else {
+            type = GF_BITROT_OPTION_TYPE_EXPIRY_TIME;
 
-        if (!strcmp (words[3], "signing-time")) {
-                if (!words[4]) {
-                        cli_err ("Missing signing-time value for bitrot "
-                                 "option");
-                        ret = -1;
-                        goto out;
-                } else {
-                        type = GF_BITROT_OPTION_TYPE_EXPIRY_TIME;
-
-                        expiry_time = strtol (words[4], NULL, 0);
-                        if (expiry_time < 1) {
-                                cli_err ("Expiry time  value should not be less"
-                                         " than 1");
-                                ret = -1;
-                                goto out;
-                        }
+            expiry_time = strtol(words[4], NULL, 0);
+            if (expiry_time < 1) {
+                cli_err(
+                    "Expiry time  value should not be less"
+                    " than 1");
+                ret = -1;
+                goto out;
+            }
 
-                        ret = dict_set_uint32 (dict, "expiry-time",
-                                               (unsigned int) expiry_time);
-                        if (ret) {
-                                cli_out ("Failed to set dict for bitrot");
-                                goto out;
-                        }
-                        goto set_type;
-                }
+            ret = dict_set_uint32(dict, "expiry-time",
+                                  (unsigned int)expiry_time);
+            if (ret) {
+                cli_out("Failed to set dict for bitrot");
+                goto out;
+            }
+            goto set_type;
+        }
+    } else if (!strcmp(words[3], "signer-threads")) {
+        if (!words[4]) {
+            cli_err(
+                "Missing signer-thread value for bitrot "
+                "option");
+            ret = -1;
+            goto out;
         } else {
-                cli_err ("Invalid option %s for bitrot. Please enter valid "
-                         "bitrot option", words[3]);
+            type = GF_BITROT_OPTION_TYPE_SIGNER_THREADS;
+
+            signer_th_count = strtol(words[4], NULL, 0);
+            if (signer_th_count < 1) {
+                cli_err("signer-thread count should not be less than 1");
                 ret = -1;
                 goto out;
-        }
+            }
 
-set_type:
-        ret = dict_set_int32 (dict, "type", type);
-        if (ret < 0)
+            ret = dict_set_uint32(dict, "signer-threads",
+                                  (unsigned int)signer_th_count);
+            if (ret) {
+                cli_out("Failed to set dict for bitrot");
                 goto out;
+            }
+            goto set_type;
+        }
+    } else {
+        cli_err(
+            "Invalid option %s for bitrot. Please enter valid "
+            "bitrot option",
+            words[3]);
+        ret = -1;
+        goto out;
+    }
+set_type:
+    ret = dict_set_int32(dict, "type", type);
+    if (ret < 0)
+        goto out;
 
-        *options = dict;
+    *options = dict;
 
 out:
-        if (ret) {
-                gf_log ("cli", GF_LOG_ERROR, "Unable to parse bitrot command");
-                if (dict)
-                        dict_destroy (dict);
-        }
+    if (ret) {
+        gf_log("cli", GF_LOG_ERROR, "Unable to parse bitrot command");
+        if (dict)
+            dict_unref(dict);
+    }
 
-        return ret;
+    return ret;
+}
+
+/* Parsing global option for NFS-Ganesha config
+ *  gluster nfs-ganesha enable/disable */
+
+int32_t
+cli_cmd_ganesha_parse(struct cli_state *state, const char **words,
+                      int wordcount, dict_t **options, char **op_errstr)
+{
+    dict_t *dict = NULL;
+    int ret = -1;
+    char *key = NULL;
+    char *value = NULL;
+    char *w = NULL;
+    static char *opwords[] = {"enable", "disable", NULL};
+    const char *question = NULL;
+    gf_answer_t answer = GF_ANSWER_NO;
+
+    GF_ASSERT(words);
+    GF_ASSERT(options);
+
+    dict = dict_new();
+
+    if (!dict)
+        goto out;
+
+    if (wordcount != 2)
+        goto out;
+
+    key = (char *)words[0];
+    value = (char *)words[1];
+
+    if (!key || !value) {
+        cli_out("Usage : nfs-ganesha <enable/disable>");
+        ret = -1;
+        goto out;
+    }
+
+    ret = gf_strip_whitespace(value, strlen(value));
+    if (ret == -1)
+        goto out;
+
+    if (strcmp(key, "nfs-ganesha")) {
+        gf_asprintf(op_errstr,
+                    "Global option: error: ' %s '"
+                    "is not a valid global option.",
+                    key);
+        ret = -1;
+        goto out;
+    }
+
+    w = str_getunamb(value, opwords);
+    if (!w) {
+        cli_out(
+            "Invalid global option \n"
+            "Usage : nfs-ganesha <enable/disable>");
+        ret = -1;
+        goto out;
+    }
+
+    if (strcmp(value, "enable") == 0) {
+        question =
+            "Enabling NFS-Ganesha requires Gluster-NFS to be "
+            "disabled across the trusted pool. Do you "
+            "still want to continue?\n";
+    } else if (strcmp(value, "disable") == 0) {
+        question =
+            "Disabling NFS-Ganesha will tear down the entire "
+            "ganesha cluster across the trusted pool. Do you "
+            "still want to continue?\n";
+    } else {
+        ret = -1;
+        goto out;
+    }
+    answer = cli_cmd_get_confirmation(state, question);
+    if (GF_ANSWER_NO == answer) {
+        gf_log("cli", GF_LOG_ERROR,
+               "Global operation "
+               "cancelled, exiting");
+        ret = -1;
+        goto out;
+    }
+    cli_out("This will take a few minutes to complete. Please wait ..");
+
+    ret = dict_set_str(dict, "key", key);
+    if (ret) {
+        gf_log(THIS->name, GF_LOG_ERROR, "dict set on key failed");
+        goto out;
+    }
+
+    ret = dict_set_str(dict, "value", value);
+    if (ret) {
+        gf_log(THIS->name, GF_LOG_ERROR, "dict set on value failed");
+        goto out;
+    }
+
+    ret = dict_set_str(dict, "globalname", "All");
+    if (ret) {
+        gf_log(THIS->name, GF_LOG_ERROR,
+               "dict set on global"
+               " key failed.");
+        goto out;
+    }
+
+    ret = dict_set_int32(dict, "hold_global_locks", _gf_true);
+    if (ret) {
+        gf_log(THIS->name, GF_LOG_ERROR,
+               "dict set on global key "
+               "failed.");
+        goto out;
+    }
+
+    *options = dict;
+out:
+    if (ret)
+        dict_unref(dict);
+
+    return ret;
 }
diff --git a/cli/src/cli-cmd-peer.c b/cli/src/cli-cmd-peer.c
index d6b4ab147a4..084998701d8 100644
--- a/cli/src/cli-cmd-peer.c
+++ b/cli/src/cli-cmd-peer.c
@@ -18,286 +18,300 @@
 #include "cli-mem-types.h"
 #include "cli1-xdr.h"
 #include "protocol-common.h"
+#include <glusterfs/events.h>
 
-extern struct rpc_clnt *global_rpc;
-
-extern rpc_clnt_prog_t *cli_rpc_prog;
-
-int cli_cmd_peer_help_cbk (struct cli_state *state, struct cli_cmd_word *in_word,
+int
+cli_cmd_peer_help_cbk(struct cli_state *state, struct cli_cmd_word *in_word,
                       const char **words, int wordcount);
 
 int
-cli_cmd_peer_probe_cbk (struct cli_state *state, struct cli_cmd_word *word,
-                   const char **words, int wordcount)
+cli_cmd_peer_probe_cbk(struct cli_state *state, struct cli_cmd_word *word,
+                       const char **words, int wordcount)
 {
-        int                     ret = -1;
-        rpc_clnt_procedure_t    *proc = NULL;
-        call_frame_t            *frame = NULL;
-        dict_t                  *dict = NULL;
-        int                     sent = 0;
-        int                     parse_error = 0;
-        cli_local_t             *local = NULL;
-
-        if (!(wordcount == 3)) {
-                cli_usage_out (word->pattern);
-                parse_error = 1;
-                goto out;
-        }
-
-        proc = &cli_rpc_prog->proctable[GLUSTER_CLI_PROBE];
-
-        frame = create_frame (THIS, THIS->ctx->pool);
-        if (!frame)
-                goto out;
-
-        dict = dict_new ();
-        if (!dict)
-                goto out;
-
-        ret = dict_set_str (dict, "hostname", (char *)words[2]);
-        if (ret)
-                goto out;
-
-        ret = valid_internet_address ((char *) words[2], _gf_false);
-        if (ret == 1) {
-                ret = 0;
-        } else {
-                cli_out ("%s is an invalid address", words[2]);
-                cli_usage_out (word->pattern);
-                parse_error = 1;
-                ret = -1;
-                goto out;
-        }
-/*        if (words[3]) {
-                ret = dict_set_str (dict, "port", (char *)words[3]);
-                if (ret)
-                        goto out;
-        }
-*/
-
-        CLI_LOCAL_INIT (local, words, frame, dict);
-
-        if (proc->fn) {
-                ret = proc->fn (frame, THIS, dict);
-        }
+    int ret = -1;
+    rpc_clnt_procedure_t *proc = NULL;
+    call_frame_t *frame = NULL;
+    dict_t *dict = NULL;
+    int sent = 0;
+    int parse_error = 0;
+    cli_local_t *local = NULL;
+
+    if (!(wordcount == 3)) {
+        cli_usage_out(word->pattern);
+        parse_error = 1;
+        goto out;
+    }
+
+    proc = &cli_rpc_prog->proctable[GLUSTER_CLI_PROBE];
+
+    dict = dict_new();
+    if (!dict)
+        goto out;
+
+    ret = dict_set_str(dict, "hostname", (char *)words[2]);
+    if (ret)
+        goto out;
+
+    ret = valid_internet_address((char *)words[2], _gf_false, _gf_false);
+    if (ret == 1) {
+        ret = 0;
+    } else {
+        cli_out("%s is an invalid address", words[2]);
+        cli_usage_out(word->pattern);
+        parse_error = 1;
+        ret = -1;
+        goto out;
+    }
+    /*        if (words[3]) {
+                    ret = dict_set_str (dict, "port", (char *)words[3]);
+                    if (ret)
+                            goto out;
+            }
+    */
+
+    frame = create_frame(THIS, THIS->ctx->pool);
+    if (!frame) {
+        ret = -1;
+        goto out;
+    }
+
+    CLI_LOCAL_INIT(local, words, frame, dict);
+
+    if (proc->fn) {
+        ret = proc->fn(frame, THIS, dict);
+    }
 
 out:
-        if (ret) {
-                cli_cmd_sent_status_get (&sent);
-                if ((sent == 0) && (parse_error == 0))
-                        cli_out ("Peer probe failed");
-        }
+    if (ret) {
+        cli_cmd_sent_status_get(&sent);
+        if ((sent == 0) && (parse_error == 0))
+            cli_out("Peer probe failed");
+    }
 
-        CLI_STACK_DESTROY (frame);
+    CLI_STACK_DESTROY(frame);
 
-        return ret;
-}
+    if (ret == 0) {
+        gf_event(EVENT_PEER_ATTACH, "host=%s", (char *)words[2]);
+    }
 
+    return ret;
+}
 
 int
-cli_cmd_peer_deprobe_cbk (struct cli_state *state, struct cli_cmd_word *word,
-                          const char **words, int wordcount)
+cli_cmd_peer_deprobe_cbk(struct cli_state *state, struct cli_cmd_word *word,
+                         const char **words, int wordcount)
 {
-        int                   ret   = -1;
-        rpc_clnt_procedure_t *proc  = NULL;
-        call_frame_t         *frame = NULL;
-        dict_t               *dict  = NULL;
-        int                  flags = 0;
-        int                  sent = 0;
-        int                  parse_error = 0;
-        cli_local_t          *local = NULL;
-
-        if ((wordcount < 3) || (wordcount > 4)) {
-                cli_usage_out (word->pattern);
-                parse_error = 1;
-                goto out;
-        }
-
-        proc = &cli_rpc_prog->proctable[GLUSTER_CLI_DEPROBE];
-
-        frame = create_frame (THIS, THIS->ctx->pool);
-        if (!frame)
-                goto out;
-
-        dict = dict_new ();
-
-        ret = dict_set_str (dict, "hostname", (char *)words[2]);
-        if (ret)
-                goto out;
-
-/*        if (words[3]) {
-                ret = dict_set_str (dict, "port", (char *)words[3]);
-                if (ret)
-                        goto out;
-        }
-*/
-        if (wordcount == 4) {
-                if (!strcmp("force", words[3]))
-                        flags |= GF_CLI_FLAG_OP_FORCE;
-                else {
-                        ret = -1;
-                        cli_usage_out (word->pattern);
-                        parse_error = 1;
-                        goto out;
-                }
-        }
-        ret = dict_set_int32 (dict, "flags", flags);
-        if (ret)
-                goto out;
-
-        CLI_LOCAL_INIT (local, words, frame, dict);
-
-        if (proc->fn) {
-                ret = proc->fn (frame, THIS, dict);
+    int ret = -1;
+    rpc_clnt_procedure_t *proc = NULL;
+    call_frame_t *frame = NULL;
+    dict_t *dict = NULL;
+    int flags = 0;
+    int sent = 0;
+    int parse_error = 0;
+    cli_local_t *local = NULL;
+    gf_answer_t answer = GF_ANSWER_NO;
+    const char *question = NULL;
+
+    if ((wordcount < 3) || (wordcount > 4)) {
+        cli_usage_out(word->pattern);
+        parse_error = 1;
+        goto out;
+    }
+    question =
+        "All clients mounted through the peer which is getting detached need "
+        "to be remounted using one of the other active peers in the trusted "
+        "storage pool to ensure client gets notification on any changes done "
+        "on the gluster configuration and if the same has been done do you "
+        "want to proceed?";
+    proc = &cli_rpc_prog->proctable[GLUSTER_CLI_DEPROBE];
+
+    dict = dict_new();
+
+    ret = dict_set_str(dict, "hostname", (char *)words[2]);
+    if (ret)
+        goto out;
+
+    /*        if (words[3]) {
+                    ret = dict_set_str (dict, "port", (char *)words[3]);
+                    if (ret)
+                            goto out;
+            }
+    */
+    if (wordcount == 4) {
+        if (!strcmp("force", words[3]))
+            flags |= GF_CLI_FLAG_OP_FORCE;
+        else {
+            ret = -1;
+            cli_usage_out(word->pattern);
+            parse_error = 1;
+            goto out;
         }
+    }
+    ret = dict_set_int32(dict, "flags", flags);
+    if (ret)
+        goto out;
+    answer = cli_cmd_get_confirmation(state, question);
+    if (GF_ANSWER_NO == answer) {
+        ret = 0;
+        goto out;
+    }
+
+    frame = create_frame(THIS, THIS->ctx->pool);
+    if (!frame) {
+        ret = -1;
+        goto out;
+    }
+
+    CLI_LOCAL_INIT(local, words, frame, dict);
+
+    if (proc->fn) {
+        ret = proc->fn(frame, THIS, dict);
+    }
 
 out:
-        if (ret) {
-                cli_cmd_sent_status_get (&sent);
-                if ((sent == 0) && (parse_error == 0))
-                        cli_out ("Peer detach failed");
-        }
+    if (ret) {
+        cli_cmd_sent_status_get(&sent);
+        if ((sent == 0) && (parse_error == 0))
+            cli_out("Peer detach failed");
+    }
 
-        CLI_STACK_DESTROY (frame);
+    CLI_STACK_DESTROY(frame);
 
-        return ret;
+    if (ret == 0) {
+        gf_event(EVENT_PEER_DETACH, "host=%s", (char *)words[2]);
+    }
+
+    return ret;
 }
 
 int
-cli_cmd_peer_status_cbk (struct cli_state *state, struct cli_cmd_word *word,
+cli_cmd_peer_status_cbk(struct cli_state *state, struct cli_cmd_word *word,
                         const char **words, int wordcount)
 {
-        int                     ret = -1;
-        rpc_clnt_procedure_t    *proc = NULL;
-        call_frame_t            *frame = NULL;
-        int                     sent = 0;
-        int                     parse_error = 0;
-
-        if (wordcount != 2) {
-                cli_usage_out (word->pattern);
-                parse_error = 1;
-                goto out;
-        }
+    int ret = -1;
+    rpc_clnt_procedure_t *proc = NULL;
+    call_frame_t *frame = NULL;
+    int sent = 0;
+    int parse_error = 0;
 
-        proc = &cli_rpc_prog->proctable[GLUSTER_CLI_LIST_FRIENDS];
+    if (wordcount != 2) {
+        cli_usage_out(word->pattern);
+        parse_error = 1;
+        goto out;
+    }
 
-        frame = create_frame (THIS, THIS->ctx->pool);
-        if (!frame)
-                goto out;
+    proc = &cli_rpc_prog->proctable[GLUSTER_CLI_LIST_FRIENDS];
 
-        if (proc->fn) {
-                ret = proc->fn (frame, THIS, (void *)GF_CLI_LIST_PEERS);
-        }
+    frame = create_frame(THIS, THIS->ctx->pool);
+    if (!frame)
+        goto out;
+
+    if (proc->fn) {
+        ret = proc->fn(frame, THIS, (void *)GF_CLI_LIST_PEERS);
+    }
 
 out:
-        if (ret) {
-                cli_cmd_sent_status_get (&sent);
-                if ((sent == 0) && (parse_error == 0))
-                        cli_out ("Peer status failed");
-        }
+    if (ret) {
+        cli_cmd_sent_status_get(&sent);
+        if ((sent == 0) && (parse_error == 0))
+            cli_out("Peer status failed");
+    }
 
-        CLI_STACK_DESTROY (frame);
+    CLI_STACK_DESTROY(frame);
 
-        return ret;
+    return ret;
 }
 
 int
-cli_cmd_pool_list_cbk (struct cli_state *state, struct cli_cmd_word *word,
-                       const char **words, int wordcount)
+cli_cmd_pool_list_cbk(struct cli_state *state, struct cli_cmd_word *word,
+                      const char **words, int wordcount)
 {
-        int                     ret = -1;
-        rpc_clnt_procedure_t    *proc = NULL;
-        call_frame_t            *frame = NULL;
-        int                     sent = 0;
-        int                     parse_error = 0;
-
-        if (wordcount != 2) {
-                cli_usage_out (word->pattern);
-                parse_error = 1;
-                goto out;
-        }
+    int ret = -1;
+    rpc_clnt_procedure_t *proc = NULL;
+    call_frame_t *frame = NULL;
+    int sent = 0;
+    int parse_error = 0;
 
-        proc = &cli_rpc_prog->proctable[GLUSTER_CLI_LIST_FRIENDS];
+    if (wordcount != 2) {
+        cli_usage_out(word->pattern);
+        parse_error = 1;
+        goto out;
+    }
 
-        frame = create_frame (THIS, THIS->ctx->pool);
-        if (!frame)
-                goto out;
+    proc = &cli_rpc_prog->proctable[GLUSTER_CLI_LIST_FRIENDS];
 
-        if (proc->fn) {
-                ret = proc->fn (frame, THIS,
-                                (void *)GF_CLI_LIST_POOL_NODES);
-        }
+    frame = create_frame(THIS, THIS->ctx->pool);
+    if (!frame)
+        goto out;
+
+    if (proc->fn) {
+        ret = proc->fn(frame, THIS, (void *)GF_CLI_LIST_POOL_NODES);
+    }
 
 out:
-        if (ret) {
-                cli_cmd_sent_status_get (&sent);
-                if ((sent == 0) && (parse_error == 0))
-                        cli_err ("pool list: command execution failed");
-        }
+    if (ret) {
+        cli_cmd_sent_status_get(&sent);
+        if ((sent == 0) && (parse_error == 0))
+            cli_err("pool list: command execution failed");
+    }
 
-        CLI_STACK_DESTROY (frame);
+    CLI_STACK_DESTROY(frame);
 
-        return ret;
+    return ret;
 }
 
 struct cli_cmd cli_probe_cmds[] = {
-        { "peer probe { <HOSTNAME> | <IP-address> }",
-          cli_cmd_peer_probe_cbk,
-          "probe peer specified by <HOSTNAME>"},
+    {"peer probe { <HOSTNAME> | <IP-address> }", cli_cmd_peer_probe_cbk,
+     "probe peer specified by <HOSTNAME>"},
 
-        { "peer detach { <HOSTNAME> | <IP-address> } [force]",
-          cli_cmd_peer_deprobe_cbk,
-          "detach peer specified by <HOSTNAME>"},
+    {"peer detach { <HOSTNAME> | <IP-address> } [force]",
+     cli_cmd_peer_deprobe_cbk, "detach peer specified by <HOSTNAME>"},
 
-        { "peer status",
-          cli_cmd_peer_status_cbk,
-          "list status of peers"},
+    {"peer status", cli_cmd_peer_status_cbk, "list status of peers"},
 
-	{ "peer help",
-           cli_cmd_peer_help_cbk,
-           "Help command for peer "},
+    {"peer help", cli_cmd_peer_help_cbk, "display help for peer commands"},
 
-        { "pool list",
-          cli_cmd_pool_list_cbk,
-          "list all the nodes in the pool (including localhost)"},
+    {"pool list", cli_cmd_pool_list_cbk,
+     "list all the nodes in the pool (including localhost)"},
 
-        { NULL, NULL, NULL }
-};
+    {NULL, NULL, NULL}};
 
 int
-cli_cmd_peer_help_cbk (struct cli_state *state, struct cli_cmd_word *in_word,
+cli_cmd_peer_help_cbk(struct cli_state *state, struct cli_cmd_word *in_word,
                       const char **words, int wordcount)
 {
-        struct cli_cmd        *cmd = NULL;
-        struct cli_cmd        *probe_cmd = NULL;
-        int                   count     = 0;
+    struct cli_cmd *cmd = NULL;
+    struct cli_cmd *probe_cmd = NULL;
+    int count = 0;
 
-        cmd = GF_CALLOC (1, sizeof (cli_probe_cmds), cli_mt_cli_cmd);
-        memcpy (cmd, cli_probe_cmds, sizeof (cli_probe_cmds));
-        count = (sizeof (cli_probe_cmds) / sizeof (struct cli_cmd));
-        cli_cmd_sort (cmd, count);
+    cli_out("\ngluster peer commands");
+    cli_out("======================\n");
 
+    cmd = GF_MALLOC(sizeof(cli_probe_cmds), cli_mt_cli_cmd);
+    memcpy(cmd, cli_probe_cmds, sizeof(cli_probe_cmds));
+    count = (sizeof(cli_probe_cmds) / sizeof(struct cli_cmd));
+    cli_cmd_sort(cmd, count);
 
+    for (probe_cmd = cmd; probe_cmd->pattern; probe_cmd++)
+        cli_out("%s - %s", probe_cmd->pattern, probe_cmd->desc);
 
-        for (probe_cmd = cmd; probe_cmd->pattern; probe_cmd++)
-                cli_out ("%s - %s", probe_cmd->pattern, probe_cmd->desc);
+    GF_FREE(cmd);
 
-        GF_FREE (cmd);
-        return 0;
+    cli_out("\n");
+    return 0;
 }
 
 int
-cli_cmd_probe_register (struct cli_state *state)
+cli_cmd_probe_register(struct cli_state *state)
 {
-        int  ret = 0;
-        struct cli_cmd *cmd = NULL;
-
-        for (cmd = cli_probe_cmds; cmd->pattern; cmd++) {
+    int ret = 0;
+    struct cli_cmd *cmd = NULL;
 
-                ret = cli_cmd_register (&state->tree, cmd);
-                if (ret)
-                        goto out;
-        }
+    for (cmd = cli_probe_cmds; cmd->pattern; cmd++) {
+        ret = cli_cmd_register(&state->tree, cmd);
+        if (ret)
+            goto out;
+    }
 out:
-        return ret;
+    return ret;
 }
diff --git a/cli/src/cli-cmd-snapshot.c b/cli/src/cli-cmd-snapshot.c
index e79128c5a59..859d6b2e40d 100644
--- a/cli/src/cli-cmd-snapshot.c
+++ b/cli/src/cli-cmd-snapshot.c
@@ -17,148 +17,119 @@
 #include "cli-cmd.h"
 #include "cli-mem-types.h"
 
-extern rpc_clnt_prog_t *cli_rpc_prog;
-
 int
-cli_cmd_snapshot_help_cbk (struct cli_state *state, struct cli_cmd_word *in_word,
-                           const char **words, int wordcount);
+cli_cmd_snapshot_help_cbk(struct cli_state *state, struct cli_cmd_word *in_word,
+                          const char **words, int wordcount);
 
 int
-cli_cmd_snapshot_cbk (struct cli_state *state, struct cli_cmd_word *word,
-                      const char **words, int wordcount)
+cli_cmd_snapshot_cbk(struct cli_state *state, struct cli_cmd_word *word,
+                     const char **words, int wordcount)
 {
-        int                      ret       = 0;
-        int                      parse_err = 0;
-        dict_t                  *options   = NULL;
-        rpc_clnt_procedure_t    *proc      = NULL;
-        call_frame_t            *frame     = NULL;
-        cli_local_t             *local     = NULL;
-
-        proc = &cli_rpc_prog->proctable [GLUSTER_CLI_SNAP];
-        if (proc == NULL) {
-               ret = -1;
-                goto out;
-        }
-
-        frame = create_frame (THIS, THIS->ctx->pool);
-        if (frame == NULL) {
-                ret = -1;
-                goto out;
+    int ret = 0;
+    int parse_err = 0;
+    dict_t *options = NULL;
+    rpc_clnt_procedure_t *proc = NULL;
+    call_frame_t *frame = NULL;
+    cli_local_t *local = NULL;
+
+    proc = &cli_rpc_prog->proctable[GLUSTER_CLI_SNAP];
+
+    /* Parses the command entered by the user */
+    ret = cli_cmd_snapshot_parse(words, wordcount, &options, state);
+    if (ret) {
+        if (ret < 0) {
+            cli_usage_out(word->pattern);
+            parse_err = 1;
+        } else {
+            /* User might have cancelled the snapshot operation */
+            ret = 0;
         }
+        goto out;
+    }
 
-        /* Parses the command entered by the user */
-        ret = cli_cmd_snapshot_parse (words, wordcount, &options, state);
-        if (ret) {
-                if (ret < 0) {
-                        cli_usage_out (word->pattern);
-                        parse_err = 1;
-                } else {
-                        /* User might have cancelled the snapshot operation */
-                        ret = 0;
-                }
-                goto out;
-        }
+    frame = create_frame(THIS, THIS->ctx->pool);
+    if (frame == NULL) {
+        ret = -1;
+        goto out;
+    }
 
-        CLI_LOCAL_INIT (local, words, frame, options);
+    CLI_LOCAL_INIT(local, words, frame, options);
 
-        if (proc->fn)
-                ret = proc->fn (frame, THIS, options);
+    if (proc->fn)
+        ret = proc->fn(frame, THIS, options);
 
 out:
-        if (ret && parse_err == 0)
-                cli_out ("Snapshot command failed");
+    if (ret && parse_err == 0)
+        cli_out("Snapshot command failed");
 
-        CLI_STACK_DESTROY (frame);
+    CLI_STACK_DESTROY(frame);
 
-        return ret;
+    return ret;
 }
 
 struct cli_cmd snapshot_cmds[] = {
-        { "snapshot help",
-          cli_cmd_snapshot_help_cbk,
-          "display help for snapshot commands"
-        },
-        { "snapshot create <snapname> <volname> [no-timestamp] "
-                "[description <description>] [force]",
-          cli_cmd_snapshot_cbk,
-          "Snapshot Create."
-        },
-        { "snapshot clone <clonename> <snapname>",
-          cli_cmd_snapshot_cbk,
-          "Snapshot Clone."
-        },
-        { "snapshot restore <snapname>",
-          cli_cmd_snapshot_cbk,
-          "Snapshot Restore."
-        },
-        { "snapshot status [(snapname | volume <volname>)]",
-          cli_cmd_snapshot_cbk,
-          "Snapshot Status."
-        },
-        { "snapshot info [(snapname | volume <volname>)]",
-          cli_cmd_snapshot_cbk,
-          "Snapshot Info."
-        },
-        { "snapshot list [volname]",
-          cli_cmd_snapshot_cbk,
-          "Snapshot List."
-        },
-        {"snapshot config [volname] ([snap-max-hard-limit <count>] "
-         "[snap-max-soft-limit <percent>]) "
-         "| ([auto-delete <enable|disable>])"
-         "| ([activate-on-create <enable|disable>])",
-          cli_cmd_snapshot_cbk,
-          "Snapshot Config."
-        },
-        {"snapshot delete (all | snapname | volume <volname>)",
-          cli_cmd_snapshot_cbk,
-          "Snapshot Delete."
-        },
-        {"snapshot activate <snapname> [force]",
-          cli_cmd_snapshot_cbk,
-          "Activate snapshot volume."
-        },
-        {"snapshot deactivate <snapname>",
-          cli_cmd_snapshot_cbk,
-          "Deactivate snapshot volume."
-        },
-        { NULL, NULL, NULL }
-};
+    {"snapshot help", cli_cmd_snapshot_help_cbk,
+     "display help for snapshot commands"},
+    {"snapshot create <snapname> <volname> [no-timestamp] "
+     "[description <description>] [force]",
+     cli_cmd_snapshot_cbk, "Snapshot Create."},
+    {"snapshot clone <clonename> <snapname>", cli_cmd_snapshot_cbk,
+     "Snapshot Clone."},
+    {"snapshot restore <snapname>", cli_cmd_snapshot_cbk, "Snapshot Restore."},
+    {"snapshot status [(snapname | volume <volname>)]", cli_cmd_snapshot_cbk,
+     "Snapshot Status."},
+    {"snapshot info [(snapname | volume <volname>)]", cli_cmd_snapshot_cbk,
+     "Snapshot Info."},
+    {"snapshot list [volname]", cli_cmd_snapshot_cbk, "Snapshot List."},
+    {"snapshot config [volname] ([snap-max-hard-limit <count>] "
+     "[snap-max-soft-limit <percent>]) "
+     "| ([auto-delete <enable|disable>])"
+     "| ([activate-on-create <enable|disable>])",
+     cli_cmd_snapshot_cbk, "Snapshot Config."},
+    {"snapshot delete (all | snapname | volume <volname>)",
+     cli_cmd_snapshot_cbk, "Snapshot Delete."},
+    {"snapshot activate <snapname> [force]", cli_cmd_snapshot_cbk,
+     "Activate snapshot volume."},
+    {"snapshot deactivate <snapname>", cli_cmd_snapshot_cbk,
+     "Deactivate snapshot volume."},
+    {NULL, NULL, NULL}};
 
 int
-cli_cmd_snapshot_help_cbk (struct cli_state *state,
-                           struct cli_cmd_word *in_word,
-                           const char **words,
-                           int wordcount)
+cli_cmd_snapshot_help_cbk(struct cli_state *state, struct cli_cmd_word *in_word,
+                          const char **words, int wordcount)
 {
-        struct cli_cmd        *cmd      = NULL;
-        struct cli_cmd        *snap_cmd = NULL;
-        int                   count     = 0;
-
-        cmd = GF_CALLOC (1, sizeof (snapshot_cmds), cli_mt_cli_cmd);
-        memcpy (cmd, snapshot_cmds, sizeof (snapshot_cmds));
-        count = (sizeof (snapshot_cmds) / sizeof (struct cli_cmd));
-        cli_cmd_sort (cmd, count);
-
-        for (snap_cmd = cmd; snap_cmd->pattern; snap_cmd++)
-                if (_gf_false == snap_cmd->disable)
-                        cli_out ("%s - %s", snap_cmd->pattern, snap_cmd->desc);
-        GF_FREE (cmd);
-        return 0;
+    struct cli_cmd *cmd = NULL;
+    struct cli_cmd *snap_cmd = NULL;
+    int count = 0;
+
+    cmd = GF_MALLOC(sizeof(snapshot_cmds), cli_mt_cli_cmd);
+    memcpy(cmd, snapshot_cmds, sizeof(snapshot_cmds));
+    count = (sizeof(snapshot_cmds) / sizeof(struct cli_cmd));
+    cli_cmd_sort(cmd, count);
+
+    cli_out("\ngluster snapshot commands");
+    cli_out("=========================\n");
+
+    for (snap_cmd = cmd; snap_cmd->pattern; snap_cmd++)
+        if (_gf_false == snap_cmd->disable)
+            cli_out("%s - %s", snap_cmd->pattern, snap_cmd->desc);
+    cli_out("\n");
+
+    GF_FREE(cmd);
+    return 0;
 }
 
 int
-cli_cmd_snapshot_register (struct cli_state *state)
+cli_cmd_snapshot_register(struct cli_state *state)
 {
-        int  ret = 0;
-        struct cli_cmd *cmd = NULL;
-
-        for (cmd = snapshot_cmds; cmd->pattern; cmd++) {
-
-                ret = cli_cmd_register (&state->tree, cmd);
-                if (ret)
-                        goto out;
-        }
+    int ret = 0;
+    struct cli_cmd *cmd = NULL;
+
+    for (cmd = snapshot_cmds; cmd->pattern; cmd++) {
+        ret = cli_cmd_register(&state->tree, cmd);
+        if (ret)
+            goto out;
+    }
 out:
-        return ret;
+    return ret;
 }
diff --git a/cli/src/cli-cmd-system.c b/cli/src/cli-cmd-system.c
index 89d7d23187e..801e8f4efed 100644
--- a/cli/src/cli-cmd-system.c
+++ b/cli/src/cli-cmd-system.c
@@ -18,588 +18,607 @@
 #include "cli-mem-types.h"
 #include "protocol-common.h"
 
+int
+cli_cmd_system_help_cbk(struct cli_state *state, struct cli_cmd_word *in_word,
+                        const char **words, int wordcount);
 
-extern struct rpc_clnt *global_rpc;
-
-extern rpc_clnt_prog_t *cli_rpc_prog;
-
-int cli_cmd_system_help_cbk (struct cli_state *state, struct cli_cmd_word *in_word,
-                             const char **words, int wordcount);
-
-int cli_cmd_copy_file_cbk (struct cli_state *state, struct cli_cmd_word *word,
-                           const char **words, int wordcount);
+int
+cli_cmd_copy_file_cbk(struct cli_state *state, struct cli_cmd_word *word,
+                      const char **words, int wordcount);
 
-int cli_cmd_sys_exec_cbk (struct cli_state *state, struct cli_cmd_word *word,
-                          const char **words, int wordcount);
+int
+cli_cmd_sys_exec_cbk(struct cli_state *state, struct cli_cmd_word *word,
+                     const char **words, int wordcount);
 
 int
-cli_cmd_getspec_cbk (struct cli_state *state, struct cli_cmd_word *word,
-                     const char **words, int wordcount)
+cli_cmd_getspec_cbk(struct cli_state *state, struct cli_cmd_word *word,
+                    const char **words, int wordcount)
 {
-        int                     ret = -1;
-        rpc_clnt_procedure_t    *proc = NULL;
-        call_frame_t            *frame = NULL;
-        dict_t                  *dict = NULL;
-
-        frame = create_frame (THIS, THIS->ctx->pool);
-        if (!frame)
-                goto out;
-
-        dict = dict_new ();
-        if (!dict)
-                goto out;
-
-        if (wordcount != 3) {
-                cli_usage_out (word->pattern);
-                goto out;
-        }
-
-        ret = dict_set_str (dict, "volid", (char *)words[2]);
-        if (ret)
-                goto out;
-
-        proc = &cli_rpc_prog->proctable[GLUSTER_CLI_GETSPEC];
-        if (proc->fn) {
-                ret = proc->fn (frame, THIS, dict);
-        }
+    int ret = -1;
+    rpc_clnt_procedure_t *proc = NULL;
+    call_frame_t *frame = NULL;
+    dict_t *dict = NULL;
+
+    if (wordcount != 3) {
+        cli_usage_out(word->pattern);
+        goto out;
+    }
+
+    dict = dict_new();
+    if (!dict)
+        goto out;
+
+    ret = dict_set_str(dict, "volid", (char *)words[2]);
+    if (ret)
+        goto out;
+
+    proc = &cli_rpc_prog->proctable[GLUSTER_CLI_GETSPEC];
+    if (proc->fn) {
+        frame = create_frame(THIS, THIS->ctx->pool);
+        if (!frame) {
+            ret = -1;
+            goto out;
+        }
+        ret = proc->fn(frame, THIS, dict);
+    }
 
 out:
-        if (!proc && ret) {
-                if (dict)
-                        dict_destroy (dict);
-                if (wordcount > 1)
-                        cli_out ("Fetching spec for volume %s failed",
-                                 (char *)words[2]);
-        }
+    if (!proc && ret) {
+        if (wordcount > 1)
+            cli_out("Fetching spec for volume %s failed", (char *)words[2]);
+    }
 
-        return ret;
+    if (dict)
+        dict_unref(dict);
+
+    CLI_STACK_DESTROY(frame);
+    return ret;
 }
 
 int
-cli_cmd_pmap_b2p_cbk (struct cli_state *state, struct cli_cmd_word *word,
-                 const char **words, int wordcount)
+cli_cmd_pmap_b2p_cbk(struct cli_state *state, struct cli_cmd_word *word,
+                     const char **words, int wordcount)
 {
-        int                     ret = -1;
-        rpc_clnt_procedure_t    *proc = NULL;
-        call_frame_t            *frame = NULL;
-        dict_t                  *dict = NULL;
-
-        frame = create_frame (THIS, THIS->ctx->pool);
-        if (!frame)
-                goto out;
-
-        dict = dict_new ();
-        if (!dict)
-                goto out;
-
-        if (wordcount != 4) {
-                cli_usage_out (word->pattern);
-                goto out;
-        }
-
-        ret = dict_set_str (dict, "brick", (char *)words[3]);
-        if (ret)
-                goto out;
-
-        proc = &cli_rpc_prog->proctable[GLUSTER_CLI_PMAP_PORTBYBRICK];
-        if (proc->fn) {
-                ret = proc->fn (frame, THIS, dict);
-        }
+    int ret = -1;
+    rpc_clnt_procedure_t *proc = NULL;
+    call_frame_t *frame = NULL;
+    dict_t *dict = NULL;
+
+    if (wordcount != 4) {
+        cli_usage_out(word->pattern);
+        goto out;
+    }
+
+    dict = dict_new();
+    if (!dict)
+        goto out;
+
+    ret = dict_set_str(dict, "brick", (char *)words[3]);
+    if (ret)
+        goto out;
+
+    proc = &cli_rpc_prog->proctable[GLUSTER_CLI_PMAP_PORTBYBRICK];
+    if (proc->fn) {
+        frame = create_frame(THIS, THIS->ctx->pool);
+        if (!frame) {
+            ret = -1;
+            goto out;
+        }
+        ret = proc->fn(frame, THIS, dict);
+    }
 
 out:
-        if (!proc && ret) {
-                if (dict)
-                        dict_destroy (dict);
-                if (wordcount > 1)
-                        cli_out ("Fetching spec for volume %s failed",
-                                 (char *)words[3]);
-        }
+    if (!proc && ret) {
+        if (wordcount > 1)
+            cli_out("Fetching spec for volume %s failed", (char *)words[3]);
+    }
+
+    if (dict)
+        dict_unref(dict);
 
-        return ret;
+    CLI_STACK_DESTROY(frame);
+    return ret;
 }
 
 int
-cli_cmd_fsm_log_cbk (struct cli_state *state, struct cli_cmd_word *word,
-                     const char **words, int wordcount)
+cli_cmd_fsm_log_cbk(struct cli_state *state, struct cli_cmd_word *word,
+                    const char **words, int wordcount)
 {
-        int                             ret = -1;
-        rpc_clnt_procedure_t            *proc = NULL;
-        call_frame_t                    *frame = NULL;
-        char                            *name = "";
-
-        if ((wordcount != 4) && (wordcount != 3)) {
-                cli_usage_out (word->pattern);
-                goto out;
-        }
-
-        if (wordcount == 4)
-                name = (char*)words[3];
-        proc = &cli_rpc_prog->proctable[GLUSTER_CLI_FSM_LOG];
-        if (proc && proc->fn) {
-                frame = create_frame (THIS, THIS->ctx->pool);
-                if (!frame)
-                        goto out;
-                ret = proc->fn (frame, THIS, (void*)name);
-        }
+    int ret = -1;
+    rpc_clnt_procedure_t *proc = NULL;
+    call_frame_t *frame = NULL;
+    char *name = "";
+
+    if ((wordcount != 4) && (wordcount != 3)) {
+        cli_usage_out(word->pattern);
+        goto out;
+    }
+
+    if (wordcount == 4)
+        name = (char *)words[3];
+    proc = &cli_rpc_prog->proctable[GLUSTER_CLI_FSM_LOG];
+    if (proc && proc->fn) {
+        frame = create_frame(THIS, THIS->ctx->pool);
+        if (!frame)
+            goto out;
+        ret = proc->fn(frame, THIS, (void *)name);
+    }
 out:
-        return ret;
+    return ret;
 }
 
 int
-cli_cmd_getwd_cbk (struct cli_state *state, struct cli_cmd_word *word,
-                   const char **words, int wordcount)
+cli_cmd_getwd_cbk(struct cli_state *state, struct cli_cmd_word *word,
+                  const char **words, int wordcount)
 {
-        int                             ret = -1;
-        rpc_clnt_procedure_t            *proc = NULL;
-        call_frame_t                    *frame = NULL;
-
-        if (wordcount != 2) {
-                cli_usage_out (word->pattern);
-                goto out;
-        }
-
-        proc = &cli_rpc_prog->proctable[GLUSTER_CLI_GETWD];
-        if (proc && proc->fn) {
-                frame = create_frame (THIS, THIS->ctx->pool);
-                if (!frame)
-                        goto out;
-                ret = proc->fn (frame, THIS, NULL);
-        }
+    int ret = -1;
+    rpc_clnt_procedure_t *proc = NULL;
+    call_frame_t *frame = NULL;
+
+    if (wordcount != 2) {
+        cli_usage_out(word->pattern);
+        goto out;
+    }
+
+    proc = &cli_rpc_prog->proctable[GLUSTER_CLI_GETWD];
+    if (proc && proc->fn) {
+        frame = create_frame(THIS, THIS->ctx->pool);
+        if (!frame)
+            goto out;
+        ret = proc->fn(frame, THIS, NULL);
+    }
 out:
-        return ret;
+    return ret;
 }
 
 static dict_t *
-make_seq_dict (int argc, char **argv)
+make_seq_dict(int argc, char **argv)
 {
-        char index[] = "4294967296"; // 1<<32
-        int i        = 0;
-        int ret      = 0;
-        dict_t *dict = dict_new ();
-
-        if (!dict)
-                return NULL;
-
-        for (i = 0; i < argc; i++) {
-                snprintf(index, sizeof(index), "%d", i);
-                ret = dict_set_str (dict, index, argv[i]);
-                if (ret == -1)
-                        break;
-        }
-
-        if (ret) {
-                dict_destroy (dict);
-                dict = NULL;
-        }
-
-        return dict;
+    char index[] = "4294967296";  // 1<<32
+    int i = 0;
+    int len;
+    int ret = 0;
+    dict_t *dict = dict_new();
+
+    if (!dict)
+        return NULL;
+
+    for (i = 0; i < argc; i++) {
+        len = snprintf(index, sizeof(index), "%d", i);
+        ret = dict_set_strn(dict, index, len, argv[i]);
+        if (ret == -1)
+            break;
+    }
+
+    if (ret) {
+        dict_unref(dict);
+        dict = NULL;
+    }
+
+    return dict;
 }
 
 int
-cli_cmd_mount_cbk (struct cli_state *state, struct cli_cmd_word *word,
-                   const char **words, int wordcount)
+cli_cmd_mount_cbk(struct cli_state *state, struct cli_cmd_word *word,
+                  const char **words, int wordcount)
 {
-        rpc_clnt_procedure_t *proc = NULL;
-        call_frame_t *frame        = NULL;
-        int ret                    = -1;
-        dict_t *dict               = NULL;
-        void *dataa[]              = {NULL, NULL};
-
-        if (wordcount < 4) {
-                cli_usage_out (word->pattern);
-                goto out;
-        }
-
-        dict = make_seq_dict (wordcount - 3, (char **)words + 3);
-        if (!dict)
-                goto out;
-
-        dataa[0] = (void *)words[2];
-        dataa[1] = dict;
-
-        proc = &cli_rpc_prog->proctable[GLUSTER_CLI_MOUNT];
-        if (proc && proc->fn) {
-                frame = create_frame (THIS, THIS->ctx->pool);
-                if (!frame)
-                        goto out;
-                ret = proc->fn (frame, THIS, dataa);
-        }
+    rpc_clnt_procedure_t *proc = NULL;
+    call_frame_t *frame = NULL;
+    int ret = -1;
+    dict_t *dict = NULL;
+    void *dataa[] = {NULL, NULL};
+
+    if (wordcount < 4) {
+        cli_usage_out(word->pattern);
+        goto out;
+    }
+
+    dict = make_seq_dict(wordcount - 3, (char **)words + 3);
+    if (!dict)
+        goto out;
+
+    dataa[0] = (void *)words[2];
+    dataa[1] = dict;
+
+    proc = &cli_rpc_prog->proctable[GLUSTER_CLI_MOUNT];
+    if (proc && proc->fn) {
+        frame = create_frame(THIS, THIS->ctx->pool);
+        if (!frame)
+            goto out;
+        ret = proc->fn(frame, THIS, dataa);
+    }
 
- out:
-        if (dict)
-                dict_unref (dict);
+out:
+    if (dict)
+        dict_unref(dict);
 
-        if (!proc && ret)
-                cli_out ("Mount command failed");
+    if (!proc && ret)
+        cli_out("Mount command failed");
 
-        return ret;
+    return ret;
 }
 
 int
-cli_cmd_umount_cbk (struct cli_state *state, struct cli_cmd_word *word,
+cli_cmd_umount_cbk(struct cli_state *state, struct cli_cmd_word *word,
                    const char **words, int wordcount)
 {
-        rpc_clnt_procedure_t *proc = NULL;
-        call_frame_t *frame        = NULL;
-        int ret                    = -1;
-        dict_t *dict               = NULL;
-
-        if (!(wordcount == 3 ||
-              (wordcount == 4 && strcmp (words[3], "lazy") == 0))) {
-                cli_usage_out (word->pattern);
-                goto out;
-        }
-
-        dict = dict_new ();
-        if (!dict)
-                goto out;
+    rpc_clnt_procedure_t *proc = NULL;
+    call_frame_t *frame = NULL;
+    int ret = -1;
+    dict_t *dict = NULL;
+
+    if (!(wordcount == 3 ||
+          (wordcount == 4 && strcmp(words[3], "lazy") == 0))) {
+        cli_usage_out(word->pattern);
+        goto out;
+    }
+
+    dict = dict_new();
+    if (!dict)
+        goto out;
+
+    ret = dict_set_str(dict, "path", (char *)words[2]);
+    if (ret != 0)
+        goto out;
+    ret = dict_set_int32(dict, "lazy", wordcount == 4);
+    if (ret != 0)
+        goto out;
+
+    proc = &cli_rpc_prog->proctable[GLUSTER_CLI_UMOUNT];
+    if (proc && proc->fn) {
+        frame = create_frame(THIS, THIS->ctx->pool);
+        if (!frame) {
+            gf_log(THIS->name, GF_LOG_ERROR, "failed to create frame");
+            ret = -1;
+            goto out;
+        }
+        ret = proc->fn(frame, THIS, dict);
+    }
 
-        ret = dict_set_str (dict, "path", (char *)words[2]);
-        if (ret != 0)
-                goto out;
-        ret = dict_set_int32 (dict, "lazy", wordcount == 4);
-        if (ret != 0)
-                goto out;
-
-        proc = &cli_rpc_prog->proctable[GLUSTER_CLI_UMOUNT];
-        if (proc && proc->fn) {
-                frame = create_frame (THIS, THIS->ctx->pool);
-                if (!frame)
-                        goto out;
-                ret = proc->fn (frame, THIS, dict);
-        }
-
- out:
-        if (dict)
-                dict_unref (dict);
+out:
+    if (dict)
+        dict_unref(dict);
 
-        if (!proc && ret)
-                cli_out ("Umount command failed");
+    if (!proc && ret)
+        cli_out("Umount command failed");
 
-        return ret;
+    return ret;
 }
 
 int
-cli_cmd_uuid_get_cbk (struct cli_state *state, struct cli_cmd_word *word,
-                      const char **words, int wordcount)
+cli_cmd_uuid_get_cbk(struct cli_state *state, struct cli_cmd_word *word,
+                     const char **words, int wordcount)
 {
-        int                     ret = -1;
-        int                     sent = 0;
-        int                     parse_error = 0;
-        dict_t                  *dict  = NULL;
-        rpc_clnt_procedure_t    *proc = NULL;
-        call_frame_t            *frame = NULL;
-        cli_local_t             *local = NULL;
-        xlator_t                *this  = NULL;
-
-        this = THIS;
-        if (wordcount != 3) {
-                cli_usage_out (word->pattern);
-                parse_error = 1;
-                goto out;
-        }
-
-        proc = &cli_rpc_prog->proctable[GLUSTER_CLI_UUID_GET];
-        frame = create_frame (this, this->ctx->pool);
-        if (!frame)
-                goto out;
-
-        dict = dict_new ();
-        if (!dict)
-                goto out;
-
-        CLI_LOCAL_INIT (local, words, frame, dict);
-        if (proc->fn)
-                ret = proc->fn (frame, this, dict);
+    int ret = -1;
+    int sent = 0;
+    int parse_error = 0;
+    dict_t *dict = NULL;
+    rpc_clnt_procedure_t *proc = NULL;
+    call_frame_t *frame = NULL;
+    cli_local_t *local = NULL;
+    xlator_t *this = NULL;
+
+    this = THIS;
+    if (wordcount != 3) {
+        cli_usage_out(word->pattern);
+        parse_error = 1;
+        goto out;
+    }
+
+    proc = &cli_rpc_prog->proctable[GLUSTER_CLI_UUID_GET];
+    frame = create_frame(this, this->ctx->pool);
+    if (!frame)
+        goto out;
+
+    dict = dict_new();
+    if (!dict)
+        goto out;
+
+    CLI_LOCAL_INIT(local, words, frame, dict);
+    if (proc->fn)
+        ret = proc->fn(frame, this, dict);
 
 out:
-        if (ret) {
-                cli_cmd_sent_status_get (&sent);
-                if ((sent == 0) && (parse_error == 0))
-                        cli_out ("uuid get failed");
-        }
+    if (ret) {
+        cli_cmd_sent_status_get(&sent);
+        if ((sent == 0) && (parse_error == 0))
+            cli_out("uuid get failed");
+    }
 
-        if (dict)
-                dict_unref (dict);
+    if (dict)
+        dict_unref(dict);
 
-        CLI_STACK_DESTROY (frame);
-        return ret;
+    CLI_STACK_DESTROY(frame);
+    return ret;
 }
 
 int
-cli_cmd_uuid_reset_cbk (struct cli_state *state, struct cli_cmd_word *word,
-                        const char **words, int wordcount)
+cli_cmd_uuid_reset_cbk(struct cli_state *state, struct cli_cmd_word *word,
+                       const char **words, int wordcount)
 {
-        int                     ret = -1;
-        rpc_clnt_procedure_t    *proc = NULL;
-        call_frame_t            *frame = NULL;
-        int                     sent = 0;
-        int                     parse_error = 0;
-        gf_answer_t             answer = GF_ANSWER_NO;
-        char                    *question = NULL;
-        cli_local_t             *local = NULL;
-        dict_t                  *dict  = NULL;
-        xlator_t                *this  = NULL;
-
-        question = "Resetting uuid changes the uuid of local glusterd. "
-                   "Do you want to continue?";
-
-        if (wordcount != 3) {
-                cli_usage_out (word->pattern);
-                parse_error = 1;
-                goto out;
-        }
-
-        proc = &cli_rpc_prog->proctable[GLUSTER_CLI_UUID_RESET];
-
-        this = THIS;
-        frame = create_frame (this, this->ctx->pool);
-        if (!frame)
-                goto out;
-
-        dict = dict_new ();
-        if (!dict) {
-                ret = -1;
-                goto out;
-        }
-        CLI_LOCAL_INIT (local, words, frame, dict);
-        answer = cli_cmd_get_confirmation (state, question);
-
-        if (GF_ANSWER_NO == answer) {
-                ret = 0;
-                goto out;
-        }
-
-        //send NULL as argument since no dictionary is sent to glusterd
-        if (proc->fn) {
-                ret = proc->fn (frame, this, dict);
-        }
+    int ret = -1;
+    rpc_clnt_procedure_t *proc = NULL;
+    call_frame_t *frame = NULL;
+    int sent = 0;
+    int parse_error = 0;
+    gf_answer_t answer = GF_ANSWER_NO;
+    char *question = NULL;
+    cli_local_t *local = NULL;
+    dict_t *dict = NULL;
+    xlator_t *this = NULL;
+
+    question =
+        "Resetting uuid changes the uuid of local glusterd. "
+        "Do you want to continue?";
+
+    if (wordcount != 3) {
+        cli_usage_out(word->pattern);
+        parse_error = 1;
+        goto out;
+    }
+
+    proc = &cli_rpc_prog->proctable[GLUSTER_CLI_UUID_RESET];
+
+    this = THIS;
+    frame = create_frame(this, this->ctx->pool);
+    if (!frame)
+        goto out;
+
+    dict = dict_new();
+    if (!dict) {
+        ret = -1;
+        goto out;
+    }
+    CLI_LOCAL_INIT(local, words, frame, dict);
+    answer = cli_cmd_get_confirmation(state, question);
+
+    if (GF_ANSWER_NO == answer) {
+        ret = 0;
+        goto out;
+    }
+
+    // send NULL as argument since no dictionary is sent to glusterd
+    if (proc->fn) {
+        ret = proc->fn(frame, this, dict);
+    }
 
 out:
-        if (ret) {
-                cli_cmd_sent_status_get (&sent);
-                if ((sent == 0) && (parse_error == 0))
-                        cli_out ("uuid reset failed");
-        }
+    if (ret) {
+        cli_cmd_sent_status_get(&sent);
+        if ((sent == 0) && (parse_error == 0))
+            cli_out("uuid reset failed");
+    }
+
+    if (dict)
+        dict_unref(dict);
 
-        CLI_STACK_DESTROY (frame);
+    CLI_STACK_DESTROY(frame);
 
-        return ret;
+    return ret;
 }
 
-struct cli_cmd cli_system_cmds[] = {
-        { "system:: getspec <VOLNAME>",
-          cli_cmd_getspec_cbk,
-          "fetch the volume file for the volume <VOLNAME>"},
+static struct cli_cmd cli_system_cmds[] = {
+    {"system:: getspec <VOLNAME>", cli_cmd_getspec_cbk,
+     "fetch the volume file for the volume <VOLNAME>"},
 
-        { "system:: portmap brick2port <BRICK>",
-          cli_cmd_pmap_b2p_cbk,
-          "query which port <BRICK> listens on"},
+    {"system:: portmap brick2port <BRICK>", cli_cmd_pmap_b2p_cbk,
+     "query which port <BRICK> listens on"},
 
-        { "system:: fsm log [<peer-name>]",
-          cli_cmd_fsm_log_cbk,
-          "display fsm transitions"},
+    {"system:: fsm log [<peer-name>]", cli_cmd_fsm_log_cbk,
+     "display fsm transitions"},
 
-        { "system:: getwd",
-          cli_cmd_getwd_cbk,
-          "query glusterd work directory"},
+    {"system:: getwd", cli_cmd_getwd_cbk, "query glusterd work directory"},
 
-        { "system:: mount <label> <args...>",
-          cli_cmd_mount_cbk,
-          "request a mount"},
+    {"system:: mount <label> <args...>", cli_cmd_mount_cbk, "request a mount"},
 
-        { "system:: umount <path> [lazy]",
-          cli_cmd_umount_cbk,
-          "request an umount"},
+    {"system:: umount <path> [lazy]", cli_cmd_umount_cbk, "request an umount"},
 
-        { "system:: uuid get",
-          cli_cmd_uuid_get_cbk,
-          "get uuid of glusterd"},
+    {"system:: uuid get", cli_cmd_uuid_get_cbk, "get uuid of glusterd"},
 
-        { "system:: uuid reset",
-          cli_cmd_uuid_reset_cbk,
-          "reset the uuid of glusterd"},
+    {"system:: uuid reset", cli_cmd_uuid_reset_cbk,
+     "reset the uuid of glusterd"},
 
-        { "system:: help",
-           cli_cmd_system_help_cbk,
-           "display help for system commands"},
+    {"system:: help", cli_cmd_system_help_cbk,
+     "display help for system commands"},
 
-        { "system:: copy file [<filename>]",
-           cli_cmd_copy_file_cbk,
-           "Copy file from current node's $working_dir to "
-           "$working_dir of all cluster nodes"},
+    {"system:: copy file [<filename>]", cli_cmd_copy_file_cbk,
+     "Copy file from current node's $working_dir to "
+     "$working_dir of all cluster nodes"},
 
-        { "system:: execute <command> <args>",
-           cli_cmd_sys_exec_cbk,
-           "Execute the command on all the nodes "
-           "in the cluster and display their output."},
+    {"system:: execute <command> <args>", cli_cmd_sys_exec_cbk,
+     "Execute the command on all the nodes "
+     "in the cluster and display their output."},
 
-        { NULL, NULL, NULL }
-};
+    {NULL, NULL, NULL}};
 
 int
-cli_cmd_sys_exec_cbk (struct cli_state *state, struct cli_cmd_word *word,
-                      const char **words, int wordcount)
+cli_cmd_sys_exec_cbk(struct cli_state *state, struct cli_cmd_word *word,
+                     const char **words, int wordcount)
 {
-        char                   cmd_arg_name[PATH_MAX] = "";
-        char                  *command                = NULL;
-        char                  *saveptr                = NULL;
-        char                  *tmp                    = NULL;
-        int                    ret                    = -1;
-        int                    i                      = -1;
-        int                    cmd_args_count         = 0;
-        int                    in_cmd_args_count      = 0;
-        rpc_clnt_procedure_t  *proc                   = NULL;
-        call_frame_t          *frame                  = NULL;
-        dict_t                *dict                   = NULL;
-        cli_local_t           *local                  = NULL;
-
-        if (wordcount < 3) {
-                cli_usage_out (word->pattern);
-                goto out;
-        }
-
-        dict = dict_new ();
-        if (!dict)
-                goto out;
-
-        command = strtok_r ((char *)words[2], " ", &saveptr);
-        do {
-                tmp = strtok_r (NULL, " ", &saveptr);
-                if (tmp) {
-                        in_cmd_args_count++;
-                        memset (cmd_arg_name, '\0', sizeof(cmd_arg_name));
-                        snprintf (cmd_arg_name, sizeof(cmd_arg_name),
-                                  "cmd_arg_%d", in_cmd_args_count);
-                        ret = dict_set_str (dict, cmd_arg_name, tmp);
-                        if (ret) {
-                                gf_log ("", GF_LOG_ERROR, "Unable to set "
-                                        "%s in dict", cmd_arg_name);
-                                goto out;
-                        }
-                }
-        } while (tmp);
-
-        cmd_args_count = wordcount - 3;
-
-        ret = dict_set_str (dict, "command", command);
-        if (ret) {
-                gf_log ("", GF_LOG_ERROR, "Unable to set command in dict");
+    char cmd_arg_name[PATH_MAX] = "";
+    char *command = NULL;
+    char *saveptr = NULL;
+    char *tmp = NULL;
+    int ret = -1;
+    int i = -1;
+    int len;
+    int cmd_args_count = 0;
+    int in_cmd_args_count = 0;
+    rpc_clnt_procedure_t *proc = NULL;
+    call_frame_t *frame = NULL;
+    dict_t *dict = NULL;
+    cli_local_t *local = NULL;
+
+    if ((wordcount < 3) || (words[2] == NULL)) {
+        cli_usage_out(word->pattern);
+        goto out;
+    }
+
+    command = strtok_r((char *)words[2], " ", &saveptr);
+    if (command == NULL) {
+        gf_log("cli", GF_LOG_ERROR, "Failed to parse command");
+        goto out;
+    }
+
+    dict = dict_new();
+    if (!dict)
+        goto out;
+
+    do {
+        tmp = strtok_r(NULL, " ", &saveptr);
+        if (tmp) {
+            in_cmd_args_count++;
+            snprintf(cmd_arg_name, sizeof(cmd_arg_name), "cmd_arg_%d",
+                     in_cmd_args_count);
+            ret = dict_set_str(dict, cmd_arg_name, tmp);
+            if (ret) {
+                gf_log("", GF_LOG_ERROR,
+                       "Unable to set "
+                       "%s in dict",
+                       cmd_arg_name);
                 goto out;
+            }
         }
+    } while (tmp);
 
-        for (i=1; i <= cmd_args_count; i++) {
-                in_cmd_args_count++;
-                memset (cmd_arg_name, '\0', sizeof(cmd_arg_name));
-                snprintf (cmd_arg_name, sizeof(cmd_arg_name),
-                          "cmd_arg_%d", in_cmd_args_count);
-                ret = dict_set_str (dict, cmd_arg_name,
-                                    (char *)words[2+i]);
-                if (ret) {
-                        gf_log ("", GF_LOG_ERROR, "Unable to set %s in dict",
-                               cmd_arg_name);
-                        goto out;
-                }
-        }
+    cmd_args_count = wordcount - 3;
 
-        ret = dict_set_int32 (dict, "cmd_args_count", in_cmd_args_count);
-        if (ret) {
-                gf_log ("", GF_LOG_ERROR,
-                        "Unable to set cmd_args_count in dict");
-                goto out;
-        }
+    ret = dict_set_str(dict, "command", command);
+    if (ret) {
+        gf_log("", GF_LOG_ERROR, "Unable to set command in dict");
+        goto out;
+    }
 
-        ret = dict_set_str (dict, "volname", "N/A");
+    for (i = 1; i <= cmd_args_count; i++) {
+        in_cmd_args_count++;
+        len = snprintf(cmd_arg_name, sizeof(cmd_arg_name), "cmd_arg_%d",
+                       in_cmd_args_count);
+        ret = dict_set_strn(dict, cmd_arg_name, len, (char *)words[2 + i]);
         if (ret) {
-                gf_log ("", GF_LOG_ERROR, "Unable to set volname in dict");
-                goto out;
-        }
-
-        proc = &cli_rpc_prog->proctable[GLUSTER_CLI_SYS_EXEC];
-        if (proc && proc->fn) {
-                frame = create_frame (THIS, THIS->ctx->pool);
-                if (!frame)
-                        goto out;
-                CLI_LOCAL_INIT (local, words, frame, dict);
-                ret = proc->fn (frame, THIS, (void*)dict);
-        }
+            gf_log("", GF_LOG_ERROR, "Unable to set %s in dict", cmd_arg_name);
+            goto out;
+        }
+    }
+
+    ret = dict_set_int32(dict, "cmd_args_count", in_cmd_args_count);
+    if (ret) {
+        gf_log("", GF_LOG_ERROR, "Unable to set cmd_args_count in dict");
+        goto out;
+    }
+
+    ret = dict_set_str(dict, "volname", "N/A");
+    if (ret) {
+        gf_log("", GF_LOG_ERROR, "Unable to set volname in dict");
+        goto out;
+    }
+
+    proc = &cli_rpc_prog->proctable[GLUSTER_CLI_SYS_EXEC];
+    if (proc->fn) {
+        frame = create_frame(THIS, THIS->ctx->pool);
+        if (!frame) {
+            gf_log(THIS->name, GF_LOG_ERROR, "failed to create frame");
+            ret = -1;
+            goto out;
+        }
+        CLI_LOCAL_INIT(local, words, frame, dict);
+        ret = proc->fn(frame, THIS, (void *)dict);
+
+        /* proc->fn is processed synchronously, which means that the
+         * execution flow won't return here until the operation is
+         * fully processed, including any related callback. For this
+         * reason, it's safe to destroy the stack here, since no one
+         * can still be using it. Additionally, it's not easy to move
+         * the stack destroy to the callback executed after completion
+         * of the operation because there are multiple things than can
+         * fail even before having queued the callback, so we would
+         * still need to destroy the stack if proc->fn returns an
+         * error. */
+        CLI_STACK_DESTROY(frame);
+        dict = NULL;
+    }
 out:
-        return ret;
+    if (dict != NULL) {
+        dict_unref(dict);
+    }
+
+    return ret;
 }
 
 int
-cli_cmd_copy_file_cbk (struct cli_state *state, struct cli_cmd_word *word,
-                       const char **words, int wordcount)
+cli_cmd_copy_file_cbk(struct cli_state *state, struct cli_cmd_word *word,
+                      const char **words, int wordcount)
 {
-        int                    ret      = -1;
-        rpc_clnt_procedure_t  *proc     = NULL;
-        call_frame_t          *frame    = NULL;
-        char                  *filename = "";
-        dict_t                *dict     = NULL;
-        cli_local_t           *local    = NULL;
-
-        if (wordcount != 4) {
-                cli_usage_out (word->pattern);
-                goto out;
-        }
-
-        dict = dict_new ();
-        if (!dict)
-                goto out;
-
-        filename = (char*)words[3];
-        ret = dict_set_str (dict, "source", filename);
-        if (ret)
-                 gf_log ("", GF_LOG_ERROR, "Unable to set filename in dict");
-
-        ret = dict_set_str (dict, "volname", "N/A");
-        if (ret)
-                 gf_log ("", GF_LOG_ERROR, "Unable to set volname in dict");
-
-        proc = &cli_rpc_prog->proctable[GLUSTER_CLI_COPY_FILE];
-        if (proc && proc->fn) {
-                frame = create_frame (THIS, THIS->ctx->pool);
-                if (!frame)
-                        goto out;
-                CLI_LOCAL_INIT (local, words, frame, dict);
-                ret = proc->fn (frame, THIS, (void*)dict);
-        }
+    int ret = -1;
+    rpc_clnt_procedure_t *proc = NULL;
+    call_frame_t *frame = NULL;
+    char *filename = "";
+    dict_t *dict = NULL;
+    cli_local_t *local = NULL;
+
+    if (wordcount != 4) {
+        cli_usage_out(word->pattern);
+        goto out;
+    }
+
+    dict = dict_new();
+    if (!dict)
+        goto out;
+
+    filename = (char *)words[3];
+    ret = dict_set_str(dict, "source", filename);
+    if (ret)
+        gf_log("", GF_LOG_ERROR, "Unable to set filename in dict");
+
+    ret = dict_set_str(dict, "volname", "N/A");
+    if (ret)
+        gf_log("", GF_LOG_ERROR, "Unable to set volname in dict");
+
+    proc = &cli_rpc_prog->proctable[GLUSTER_CLI_COPY_FILE];
+    if (proc && proc->fn) {
+        frame = create_frame(THIS, THIS->ctx->pool);
+        if (!frame) {
+            gf_log(THIS->name, GF_LOG_ERROR, "failed to create frame");
+            ret = -1;
+            goto out;
+        }
+        CLI_LOCAL_INIT(local, words, frame, dict);
+        ret = proc->fn(frame, THIS, (void *)dict);
+    }
 out:
-        return ret;
+    return ret;
 }
 
 int
-cli_cmd_system_help_cbk (struct cli_state *state, struct cli_cmd_word *in_word,
-                         const char **words, int wordcount)
+cli_cmd_system_help_cbk(struct cli_state *state, struct cli_cmd_word *in_word,
+                        const char **words, int wordcount)
 {
-        struct cli_cmd        *cmd = NULL;
-        struct cli_cmd        *system_cmd = NULL;
-        int                   count     = 0;
+    struct cli_cmd *cmd = NULL;
+    struct cli_cmd *system_cmd = NULL;
+    int count = 0;
 
-        cmd = GF_CALLOC (1, sizeof (cli_system_cmds), cli_mt_cli_cmd);
-        memcpy (cmd, cli_system_cmds, sizeof (cli_system_cmds));
-        count = (sizeof (cli_system_cmds) / sizeof (struct cli_cmd));
-        cli_cmd_sort (cmd, count);
+    cmd = GF_MALLOC(sizeof(cli_system_cmds), cli_mt_cli_cmd);
+    memcpy(cmd, cli_system_cmds, sizeof(cli_system_cmds));
+    count = (sizeof(cli_system_cmds) / sizeof(struct cli_cmd));
+    cli_cmd_sort(cmd, count);
 
-        for (system_cmd = cmd; system_cmd->pattern; system_cmd++)
-                cli_out ("%s - %s", system_cmd->pattern, system_cmd->desc);
+    for (system_cmd = cmd; system_cmd->pattern; system_cmd++)
+        cli_out("%s - %s", system_cmd->pattern, system_cmd->desc);
 
-        GF_FREE (cmd);
-        return 0;
+    GF_FREE(cmd);
+    return 0;
 }
 
 int
-cli_cmd_system_register (struct cli_state *state)
+cli_cmd_system_register(struct cli_state *state)
 {
-        int  ret = 0;
-        struct cli_cmd *cmd = NULL;
-
-        for (cmd = cli_system_cmds; cmd->pattern; cmd++) {
+    int ret = 0;
+    struct cli_cmd *cmd = NULL;
 
-                ret = cli_cmd_register (&state->tree, cmd);
-                if (ret)
-                        goto out;
-        }
+    for (cmd = cli_system_cmds; cmd->pattern; cmd++) {
+        ret = cli_cmd_register(&state->tree, cmd);
+        if (ret)
+            goto out;
+    }
 out:
-        return ret;
+    return ret;
 }
diff --git a/cli/src/cli-cmd-volume.c b/cli/src/cli-cmd-volume.c
index 19ae35a90b3..f238851586e 100644
--- a/cli/src/cli-cmd-volume.c
+++ b/cli/src/cli-cmd-volume.c
@@ -23,2818 +23,3256 @@
 #include "cli-cmd.h"
 #include "cli-mem-types.h"
 #include "cli1-xdr.h"
-#include "run.h"
-#include "syscall.h"
-#include "common-utils.h"
+#include <glusterfs/run.h>
+#include <glusterfs/syscall.h>
+#include <glusterfs/common-utils.h>
+#include <glusterfs/events.h>
 
-extern struct rpc_clnt *global_rpc;
-extern struct rpc_clnt *global_quotad_rpc;
-
-extern rpc_clnt_prog_t *cli_rpc_prog;
 extern rpc_clnt_prog_t cli_quotad_clnt;
 
-int
-cli_cmd_volume_help_cbk (struct cli_state *state, struct cli_cmd_word *in_word,
-                      const char **words, int wordcount);
+static int
+gf_asprintf_append(char **string_ptr, const char *format, ...);
 
 int
-cli_cmd_volume_info_cbk (struct cli_state *state, struct cli_cmd_word *word,
-                         const char **words, int wordcount)
-{
-        int                             ret = -1;
-        rpc_clnt_procedure_t            *proc = NULL;
-        call_frame_t                    *frame = NULL;
-        cli_cmd_volume_get_ctx_t        ctx = {0,};
-        cli_local_t                     *local = NULL;
-        int                             sent = 0;
-        int                             parse_error = 0;
+cli_cmd_volume_help_cbk(struct cli_state *state, struct cli_cmd_word *in_word,
+                        const char **words, int wordcount);
 
-        proc = &cli_rpc_prog->proctable[GLUSTER_CLI_GET_VOLUME];
+int
+cli_cmd_bitrot_help_cbk(struct cli_state *state, struct cli_cmd_word *in_word,
+                        const char **words, int wordcount);
 
-        frame = create_frame (THIS, THIS->ctx->pool);
-        if (!frame)
-                goto out;
+int
+cli_cmd_quota_help_cbk(struct cli_state *state, struct cli_cmd_word *in_word,
+                       const char **words, int wordcount);
 
-        if ((wordcount == 2)  || (wordcount == 3 &&
-                                  !strcmp (words[2], "all"))) {
-                ctx.flags = GF_CLI_GET_NEXT_VOLUME;
-                proc = &cli_rpc_prog->proctable[GLUSTER_CLI_GET_NEXT_VOLUME];
-        } else if (wordcount == 3) {
-                ctx.flags = GF_CLI_GET_VOLUME;
-                ctx.volname = (char *)words[2];
-                if (strlen (ctx.volname) > GD_VOLUME_NAME_MAX) {
-                        cli_out ("Invalid volume name");
-                        goto out;
-                }
-                proc = &cli_rpc_prog->proctable[GLUSTER_CLI_GET_VOLUME];
-        } else {
-                cli_usage_out (word->pattern);
-                parse_error = 1;
-                return -1;
+int
+cli_cmd_volume_info_cbk(struct cli_state *state, struct cli_cmd_word *word,
+                        const char **words, int wordcount)
+{
+    int ret = -1;
+    rpc_clnt_procedure_t *proc = NULL;
+    call_frame_t *frame = NULL;
+    cli_cmd_volume_get_ctx_t ctx = {
+        0,
+    };
+    cli_local_t *local = NULL;
+    int sent = 0;
+    int parse_error = 0;
+
+    proc = &cli_rpc_prog->proctable[GLUSTER_CLI_GET_VOLUME];
+
+    if ((wordcount == 2) || (wordcount == 3 && !strcmp(words[2], "all"))) {
+        ctx.flags = GF_CLI_GET_NEXT_VOLUME;
+        proc = &cli_rpc_prog->proctable[GLUSTER_CLI_GET_NEXT_VOLUME];
+    } else if (wordcount == 3) {
+        ctx.flags = GF_CLI_GET_VOLUME;
+        ctx.volname = (char *)words[2];
+        if (strlen(ctx.volname) > GD_VOLUME_NAME_MAX) {
+            cli_out("Invalid volume name");
+            goto out;
         }
+        proc = &cli_rpc_prog->proctable[GLUSTER_CLI_GET_VOLUME];
+    } else {
+        cli_usage_out(word->pattern);
+        parse_error = 1;
+        return -1;
+    }
 
-        local = cli_local_get ();
+    local = cli_local_get();
 
-        if (!local)
-                goto out;
+    if (!local)
+        goto out;
 
-        local->get_vol.flags = ctx.flags;
-        if (ctx.volname)
-                local->get_vol.volname = gf_strdup (ctx.volname);
+    frame = create_frame(THIS, THIS->ctx->pool);
+    if (!frame)
+        goto out;
 
-        frame->local = local;
+    local->get_vol.flags = ctx.flags;
+    if (ctx.volname)
+        local->get_vol.volname = gf_strdup(ctx.volname);
 
-        if (proc->fn) {
-                ret = proc->fn (frame, THIS, &ctx);
-        }
+    frame->local = local;
 
-out:
-        if (ret) {
-                cli_cmd_sent_status_get (&sent);
-                if ((sent == 0) && (parse_error == 0))
-                        cli_out ("Getting Volume information failed!");
-        }
+    if (proc->fn) {
+        ret = proc->fn(frame, THIS, &ctx);
+    }
 
-        CLI_STACK_DESTROY (frame);
+out:
+    if (ret) {
+        cli_cmd_sent_status_get(&sent);
+        if ((sent == 0) && (parse_error == 0))
+            cli_out("Getting Volume information failed!");
+    }
 
-        return ret;
+    CLI_STACK_DESTROY(frame);
 
+    return ret;
 }
 
 int
-cli_cmd_sync_volume_cbk (struct cli_state *state, struct cli_cmd_word *word,
-                         const char **words, int wordcount)
+cli_cmd_sync_volume_cbk(struct cli_state *state, struct cli_cmd_word *word,
+                        const char **words, int wordcount)
 {
-        int                     ret = -1;
-        rpc_clnt_procedure_t    *proc = NULL;
-        call_frame_t            *frame = NULL;
-        int                     sent = 0;
-        int                     parse_error = 0;
-        dict_t                  *dict = NULL;
-        cli_local_t             *local = NULL;
-        gf_answer_t             answer = GF_ANSWER_NO;
-        const char              *question = "Sync volume may make data "
-                                            "inaccessible while the sync "
-                                            "is in progress. Do you want "
-                                            "to continue?";
-
-        if ((wordcount < 3) || (wordcount > 4)) {
-                cli_usage_out (word->pattern);
-                parse_error = 1;
-                goto out;
-        }
+    int ret = -1;
+    rpc_clnt_procedure_t *proc = NULL;
+    call_frame_t *frame = NULL;
+    int sent = 0;
+    int parse_error = 0;
+    dict_t *dict = NULL;
+    cli_local_t *local = NULL;
+    gf_answer_t answer = GF_ANSWER_NO;
+    const char *question =
+        "Sync volume may make data "
+        "inaccessible while the sync "
+        "is in progress. Do you want "
+        "to continue?";
+
+    if ((wordcount < 3) || (wordcount > 4)) {
+        cli_usage_out(word->pattern);
+        parse_error = 1;
+        goto out;
+    }
 
-        dict = dict_new ();
-        if (!dict)
-                goto out;
+    dict = dict_new();
+    if (!dict)
+        goto out;
 
-        if ((wordcount == 3) || !strcmp(words[3], "all")) {
-                ret = dict_set_int32 (dict, "flags", (int32_t)
-                                      GF_CLI_SYNC_ALL);
-                if (ret) {
-                        gf_log (THIS->name, GF_LOG_ERROR, "failed to set"
-                                "flag");
-                        goto out;
-                }
-        } else {
-                ret = dict_set_str (dict, "volname", (char *) words[3]);
-                if (ret) {
-                        gf_log (THIS->name, GF_LOG_ERROR, "failed to set "
-                                "volume");
-                        goto out;
-                }
+    if ((wordcount == 3) || !strcmp(words[3], "all")) {
+        ret = dict_set_int32(dict, "flags", (int32_t)GF_CLI_SYNC_ALL);
+        if (ret) {
+            gf_log(THIS->name, GF_LOG_ERROR,
+                   "failed to set"
+                   "flag");
+            goto out;
         }
-
-        ret = dict_set_str (dict, "hostname", (char *) words[2]);
+    } else {
+        ret = dict_set_str(dict, "volname", (char *)words[3]);
         if (ret) {
-                gf_log (THIS->name, GF_LOG_ERROR, "failed to set hostname");
-                goto out;
+            gf_log(THIS->name, GF_LOG_ERROR,
+                   "failed to set "
+                   "volume");
+            goto out;
         }
+    }
 
-        if (!(state->mode & GLUSTER_MODE_SCRIPT)) {
-                answer = cli_cmd_get_confirmation (state, question);
-                if (GF_ANSWER_NO == answer) {
-                        ret = 0;
-                        goto out;
-                }
+    ret = dict_set_str(dict, "hostname", (char *)words[2]);
+    if (ret) {
+        gf_log(THIS->name, GF_LOG_ERROR, "failed to set hostname");
+        goto out;
+    }
+
+    if (!(state->mode & GLUSTER_MODE_SCRIPT)) {
+        answer = cli_cmd_get_confirmation(state, question);
+        if (GF_ANSWER_NO == answer) {
+            ret = 0;
+            goto out;
         }
+    }
 
-        proc = &cli_rpc_prog->proctable[GLUSTER_CLI_SYNC_VOLUME];
+    proc = &cli_rpc_prog->proctable[GLUSTER_CLI_SYNC_VOLUME];
 
-        frame = create_frame (THIS, THIS->ctx->pool);
-        if (!frame)
-                goto out;
+    frame = create_frame(THIS, THIS->ctx->pool);
+    if (!frame) {
+        gf_log(THIS->name, GF_LOG_ERROR, "failed to create frame");
+        ret = -1;
+        goto out;
+    }
 
-        CLI_LOCAL_INIT (local, words, frame, dict);
+    CLI_LOCAL_INIT(local, words, frame, dict);
 
-        if (proc->fn) {
-                ret = proc->fn (frame, THIS, dict);
-        }
+    if (proc->fn) {
+        ret = proc->fn(frame, THIS, dict);
+    }
 
 out:
-        if (ret) {
-                cli_cmd_sent_status_get (&sent);
-                if ((sent == 0) && (parse_error == 0))
-                        cli_out ("Volume sync failed");
-        }
+    if (ret) {
+        cli_cmd_sent_status_get(&sent);
+        if ((sent == 0) && (parse_error == 0))
+            cli_out("Volume sync failed");
+    }
 
-        CLI_STACK_DESTROY (frame);
+    CLI_STACK_DESTROY(frame);
 
-        return ret;
+    return ret;
 }
 
 int
-cli_cmd_volume_create_cbk (struct cli_state *state, struct cli_cmd_word *word,
-                           const char **words, int wordcount)
+cli_cmd_volume_create_cbk(struct cli_state *state, struct cli_cmd_word *word,
+                          const char **words, int wordcount)
 {
-        int                     ret = -1;
-        rpc_clnt_procedure_t    *proc = NULL;
-        call_frame_t            *frame = NULL;
-        dict_t                  *options = NULL;
-        int                     sent = 0;
-        int                     parse_error = 0;
-        char                    *brick_list = NULL;
-        int32_t                 brick_count = 0;
-        int32_t                 sub_count = 0;
-        int32_t                 type = GF_CLUSTER_TYPE_NONE;
-        cli_local_t             *local = NULL;
-        char                    *trans_type = NULL;
-
-        proc = &cli_rpc_prog->proctable[GLUSTER_CLI_CREATE_VOLUME];
-
-        frame = create_frame (THIS, THIS->ctx->pool);
-        if (!frame)
-                goto out;
-
-        ret = cli_cmd_volume_create_parse (state, words, wordcount, &options);
+    int ret = -1;
+    rpc_clnt_procedure_t *proc = NULL;
+    call_frame_t *frame = NULL;
+    dict_t *options = NULL;
+    int sent = 0;
+    int parse_error = 0;
+    cli_local_t *local = NULL;
+    char *trans_type = NULL;
+    char *bricks = NULL;
+
+    proc = &cli_rpc_prog->proctable[GLUSTER_CLI_CREATE_VOLUME];
+
+    ret = cli_cmd_volume_create_parse(state, words, wordcount, &options,
+                                      &bricks);
+
+    if (ret) {
+        cli_usage_out(word->pattern);
+        parse_error = 1;
+        goto out;
+    }
 
-        if (ret) {
-                cli_usage_out (word->pattern);
-                parse_error = 1;
-                goto out;
-        }
+    ret = dict_get_str(options, "transport", &trans_type);
+    if (ret) {
+        gf_log("cli", GF_LOG_ERROR, "Unable to get transport type");
+        goto out;
+    }
 
-        ret = dict_get_str (options, "transport", &trans_type);
+    if (state->mode & GLUSTER_MODE_WIGNORE) {
+        ret = dict_set_int32(options, "force", _gf_true);
         if (ret) {
-                gf_log("cli", GF_LOG_ERROR, "Unable to get transport type");
-                goto out;
+            gf_log("cli", GF_LOG_ERROR,
+                   "Failed to set force "
+                   "option");
+            goto out;
         }
+    }
 
-        if (state->mode & GLUSTER_MODE_WIGNORE) {
-                ret = dict_set_int32 (options, "force", _gf_true);
-                if (ret) {
-                        gf_log ("cli", GF_LOG_ERROR, "Failed to set force "
-                                "option");
-                        goto out;
-                }
-        }
+    frame = create_frame(THIS, THIS->ctx->pool);
+    if (!frame) {
+        ret = -1;
+        goto out;
+    }
 
-        CLI_LOCAL_INIT (local, words, frame, options);
+    CLI_LOCAL_INIT(local, words, frame, options);
 
-        if (proc->fn) {
-                ret = proc->fn (frame, THIS, options);
-        }
+    if (proc->fn) {
+        ret = proc->fn(frame, THIS, options);
+    }
 
 out:
-        if (ret) {
-                cli_cmd_sent_status_get (&sent);
-                if ((sent == 0) && (parse_error == 0))
-                        cli_out ("Volume create failed");
-        }
-
-        CLI_STACK_DESTROY (frame);
-
-        return ret;
+    if (ret) {
+        cli_cmd_sent_status_get(&sent);
+        if ((sent == 0) && (parse_error == 0))
+            cli_out("Volume create failed");
+    }
+
+    if (ret == 0) {
+        gf_event(EVENT_VOLUME_CREATE, "name=%s;bricks=%s", (char *)words[2],
+                 bricks);
+    }
+
+    CLI_STACK_DESTROY(frame);
+    return ret;
 }
 
-
 int
-cli_cmd_volume_delete_cbk (struct cli_state *state, struct cli_cmd_word *word,
-                           const char **words, int wordcount)
+cli_cmd_volume_delete_cbk(struct cli_state *state, struct cli_cmd_word *word,
+                          const char **words, int wordcount)
 {
-        int                     ret = -1;
-        rpc_clnt_procedure_t    *proc = NULL;
-        call_frame_t            *frame = NULL;
-        char                    *volname = NULL;
-        gf_answer_t             answer = GF_ANSWER_NO;
-        const char              *question = NULL;
-        int                     sent = 0;
-        int                     parse_error = 0;
-        cli_local_t             *local = NULL;
-        dict_t                  *dict = NULL;
-
-        question = "Deleting volume will erase all information about the volume. "
-                   "Do you want to continue?";
-        proc = &cli_rpc_prog->proctable[GLUSTER_CLI_DELETE_VOLUME];
-
-        frame = create_frame (THIS, THIS->ctx->pool);
-        if (!frame)
-                goto out;
-
-        dict = dict_new ();
-        if (!dict)
-                goto out;
-
-        if (wordcount != 3) {
-                cli_usage_out (word->pattern);
-                parse_error = 1;
-                goto out;
-        }
+    int ret = -1;
+    rpc_clnt_procedure_t *proc = NULL;
+    call_frame_t *frame = NULL;
+    char *volname = NULL;
+    gf_answer_t answer = GF_ANSWER_NO;
+    const char *question = NULL;
+    int sent = 0;
+    int parse_error = 0;
+    cli_local_t *local = NULL;
+    dict_t *dict = NULL;
+
+    question =
+        "Deleting volume will erase all information about the volume. "
+        "Do you want to continue?";
+    proc = &cli_rpc_prog->proctable[GLUSTER_CLI_DELETE_VOLUME];
+
+    if (wordcount != 3) {
+        cli_usage_out(word->pattern);
+        parse_error = 1;
+        goto out;
+    }
 
-        volname = (char *)words[2];
+    volname = (char *)words[2];
 
-        ret = dict_set_str (dict, "volname", volname);
-        if (ret) {
-                gf_log (THIS->name, GF_LOG_WARNING, "dict set failed");
-                goto out;
-        }
+    dict = dict_new();
+    if (!dict)
+        goto out;
 
-        if (!strcmp (volname, GLUSTER_SHARED_STORAGE)) {
-                question = "Deleting the shared storage volume"
-                           "(gluster_shared_storage), will affect features "
-                           "like snapshot scheduler, geo-replication "
-                           "and NFS-Ganesha. Do you still want to "
-                           "continue?";
-        }
+    ret = dict_set_str(dict, "volname", volname);
+    if (ret) {
+        gf_log(THIS->name, GF_LOG_WARNING, "dict set failed");
+        goto out;
+    }
+
+    if (!strcmp(volname, GLUSTER_SHARED_STORAGE)) {
+        question =
+            "Deleting the shared storage volume"
+            "(gluster_shared_storage), will affect features "
+            "like snapshot scheduler, geo-replication "
+            "and NFS-Ganesha. Do you still want to "
+            "continue?";
+    }
+
+    answer = cli_cmd_get_confirmation(state, question);
+    if (GF_ANSWER_NO == answer) {
+        ret = 0;
+        goto out;
+    }
 
-        answer = cli_cmd_get_confirmation (state, question);
-        if (GF_ANSWER_NO == answer) {
-                ret = 0;
-                goto out;
-        }
+    frame = create_frame(THIS, THIS->ctx->pool);
+    if (!frame) {
+        ret = -1;
+        goto out;
+    }
 
-        CLI_LOCAL_INIT (local, words, frame, dict);
+    CLI_LOCAL_INIT(local, words, frame, dict);
 
-        if (proc->fn) {
-                ret = proc->fn (frame, THIS, dict);
-        }
+    if (proc->fn) {
+        ret = proc->fn(frame, THIS, dict);
+    }
 
 out:
-        if (ret) {
-                cli_cmd_sent_status_get (&sent);
-                if ((sent == 0) && (parse_error == 0))
-                        cli_out ("Volume delete failed");
-        }
+    if (ret) {
+        cli_cmd_sent_status_get(&sent);
+        if ((sent == 0) && (parse_error == 0))
+            cli_out("Volume delete failed");
+    }
+
+    CLI_STACK_DESTROY(frame);
 
-        CLI_STACK_DESTROY (frame);
+    if (ret == 0 && GF_ANSWER_YES == answer) {
+        gf_event(EVENT_VOLUME_DELETE, "name=%s", (char *)words[2]);
+    }
 
-        return ret;
+    return ret;
 }
 
 int
-cli_cmd_volume_start_cbk (struct cli_state *state, struct cli_cmd_word *word,
-                          const char **words, int wordcount)
+cli_cmd_volume_start_cbk(struct cli_state *state, struct cli_cmd_word *word,
+                         const char **words, int wordcount)
 {
-        int                     ret = -1;
-        rpc_clnt_procedure_t    *proc = NULL;
-        call_frame_t            *frame = NULL;
-        int                     sent = 0;
-        int                     parse_error = 0;
-        dict_t                  *dict = NULL;
-        int                     flags = 0;
-        cli_local_t             *local = NULL;
-
-        frame = create_frame (THIS, THIS->ctx->pool);
-        if (!frame)
-                goto out;
+    int ret = -1;
+    rpc_clnt_procedure_t *proc = NULL;
+    call_frame_t *frame = NULL;
+    int sent = 0;
+    int parse_error = 0;
+    dict_t *dict = NULL;
+    int flags = 0;
+    cli_local_t *local = NULL;
+
+    if (wordcount < 3 || wordcount > 4) {
+        cli_usage_out(word->pattern);
+        parse_error = 1;
+        goto out;
+    }
 
-        if (wordcount < 3 || wordcount > 4) {
-               cli_usage_out (word->pattern);
-                parse_error = 1;
-               goto out;
-        }
+    if (!words[2])
+        goto out;
 
-        dict = dict_new ();
-        if (!dict) {
-                goto out;
+    if (wordcount == 4) {
+        if (!strcmp("force", words[3])) {
+            flags |= GF_CLI_FLAG_OP_FORCE;
+        } else {
+            ret = -1;
+            cli_usage_out(word->pattern);
+            parse_error = 1;
+            goto out;
         }
+    }
 
-        if (!words[2])
-                goto out;
+    dict = dict_new();
+    if (!dict) {
+        goto out;
+    }
 
-        ret = dict_set_str (dict, "volname", (char *)words[2]);
-        if (ret) {
-                gf_log (THIS->name, GF_LOG_ERROR, "dict set failed");
-                goto out;
-        }
+    ret = dict_set_str(dict, "volname", (char *)words[2]);
+    if (ret) {
+        gf_log(THIS->name, GF_LOG_ERROR, "dict set failed");
+        goto out;
+    }
 
-        if (wordcount == 4) {
-                if (!strcmp("force", words[3])) {
-                        flags |= GF_CLI_FLAG_OP_FORCE;
-                } else {
-                        ret = -1;
-                        cli_usage_out (word->pattern);
-                        parse_error = 1;
-                        goto out;
-                }
-        }
-        ret = dict_set_int32 (dict, "flags", flags);
-        if (ret) {
-                 gf_log (THIS->name, GF_LOG_ERROR,
-                         "dict set failed");
-                 goto out;
-        }
+    ret = dict_set_int32(dict, "flags", flags);
+    if (ret) {
+        gf_log(THIS->name, GF_LOG_ERROR, "dict set failed");
+        goto out;
+    }
 
-        proc = &cli_rpc_prog->proctable[GLUSTER_CLI_START_VOLUME];
+    proc = &cli_rpc_prog->proctable[GLUSTER_CLI_START_VOLUME];
 
-        CLI_LOCAL_INIT (local, words, frame, dict);
+    frame = create_frame(THIS, THIS->ctx->pool);
+    if (!frame) {
+        ret = -1;
+        goto out;
+    }
 
-        if (proc->fn) {
-                ret = proc->fn (frame, THIS, dict);
-        }
+    CLI_LOCAL_INIT(local, words, frame, dict);
+
+    if (proc->fn) {
+        ret = proc->fn(frame, THIS, dict);
+    }
 
 out:
-        if (ret) {
-                cli_cmd_sent_status_get (&sent);
-                if ((sent == 0) && (parse_error == 0))
-                        cli_out ("Volume start failed");
-        }
+    if (ret) {
+        cli_cmd_sent_status_get(&sent);
+        if ((sent == 0) && (parse_error == 0))
+            cli_out("Volume start failed");
+    }
+
+    CLI_STACK_DESTROY(frame);
 
-        CLI_STACK_DESTROY (frame);
+    if (ret == 0) {
+        gf_event(EVENT_VOLUME_START, "name=%s;force=%d", (char *)words[2],
+                 (flags & GF_CLI_FLAG_OP_FORCE));
+    }
 
-        return ret;
+    return ret;
 }
 
 gf_answer_t
-cli_cmd_get_confirmation (struct cli_state *state, const char *question)
+cli_cmd_get_confirmation(struct cli_state *state, const char *question)
 {
-        char                    answer[5] = {'\0', };
-        char                    flush = '\0';
-        size_t			len;
+    char answer[5] = {
+        '\0',
+    };
+    int flush = '\0';
+    size_t len;
 
-        if (state->mode & GLUSTER_MODE_SCRIPT)
-                return GF_ANSWER_YES;
+    if (state->mode & GLUSTER_MODE_SCRIPT)
+        return GF_ANSWER_YES;
 
-	printf ("%s (y/n) ", question);
+    printf("%s (y/n) ", question);
 
-        if (fgets (answer, 4, stdin) == NULL) {
-                cli_out("gluster cli read error");
-                goto out;
-        }
+    if (fgets(answer, 4, stdin) == NULL) {
+        cli_out("gluster cli read error");
+        goto out;
+    }
 
-	len = strlen (answer);
+    len = strlen(answer);
 
-	if (len && answer [len - 1] == '\n'){
-		answer [--len] = '\0';
-	} else {
-		do{
-			flush = getchar ();
-		}while (flush != '\n');
-	}
+    if (len && answer[len - 1] == '\n') {
+        answer[--len] = '\0';
+    } else {
+        do {
+            flush = getchar();
+        } while (flush != '\n');
+    }
 
-	if (len > 3)
-		goto out;
+    if (len > 3)
+        goto out;
 
-	if (!strcasecmp (answer, "y") || !strcasecmp (answer, "yes"))
-		return GF_ANSWER_YES;
+    if (!strcasecmp(answer, "y") || !strcasecmp(answer, "yes"))
+        return GF_ANSWER_YES;
 
-	else if (!strcasecmp (answer, "n") || !strcasecmp (answer, "no"))
-		return GF_ANSWER_NO;
+    else if (!strcasecmp(answer, "n") || !strcasecmp(answer, "no"))
+        return GF_ANSWER_NO;
 
 out:
-	cli_out ("Invalid input, please enter y/n");
+    cli_out("Invalid input, please enter y/n");
 
-	return GF_ANSWER_NO;
+    return GF_ANSWER_NO;
 }
 
 int
-cli_cmd_volume_stop_cbk (struct cli_state *state, struct cli_cmd_word *word,
-                           const char **words, int wordcount)
+cli_cmd_volume_stop_cbk(struct cli_state *state, struct cli_cmd_word *word,
+                        const char **words, int wordcount)
 {
-        int                     ret = -1;
-        rpc_clnt_procedure_t    *proc = NULL;
-        call_frame_t            *frame = NULL;
-        int                     flags   = 0;
-        gf_answer_t             answer = GF_ANSWER_NO;
-        int                     sent = 0;
-        int                     parse_error = 0;
-        dict_t                  *dict = NULL;
-        char                    *volname = NULL;
-        cli_local_t             *local = NULL;
-
-        const char *question = "Stopping volume will make its data inaccessible. "
-                               "Do you want to continue?";
-
-        frame = create_frame (THIS, THIS->ctx->pool);
-        if (!frame)
-                goto out;
-
-        if (wordcount < 3 || wordcount > 4) {
-                cli_usage_out (word->pattern);
-                parse_error = 1;
-                goto out;
-        }
+    int ret = -1;
+    rpc_clnt_procedure_t *proc = NULL;
+    call_frame_t *frame = NULL;
+    int flags = 0;
+    gf_answer_t answer = GF_ANSWER_NO;
+    int sent = 0;
+    int parse_error = 0;
+    dict_t *dict = NULL;
+    char *volname = NULL;
+    cli_local_t *local = NULL;
+
+    const char *question =
+        "Stopping volume will make its data inaccessible. "
+        "Do you want to continue?";
+
+    if (wordcount < 3 || wordcount > 4) {
+        cli_usage_out(word->pattern);
+        parse_error = 1;
+        goto out;
+    }
 
-        volname = (char*) words[2];
+    volname = (char *)words[2];
 
-        dict = dict_new ();
-        ret = dict_set_str (dict, "volname", volname);
-        if (ret) {
-                gf_log (THIS->name, GF_LOG_ERROR, "dict set failed");
-                goto out;
-        }
-
-        if (!strcmp (volname, GLUSTER_SHARED_STORAGE)) {
-                question = "Stopping the shared storage volume"
-                           "(gluster_shared_storage), will affect features "
-                           "like snapshot scheduler, geo-replication "
-                           "and NFS-Ganesha. Do you still want to "
-                           "continue?";
+    dict = dict_new();
+    ret = dict_set_str(dict, "volname", volname);
+    if (ret) {
+        gf_log(THIS->name, GF_LOG_ERROR, "dict set failed");
+        goto out;
+    }
+
+    if (!strcmp(volname, GLUSTER_SHARED_STORAGE)) {
+        question =
+            "Stopping the shared storage volume"
+            "(gluster_shared_storage), will affect features "
+            "like snapshot scheduler, geo-replication "
+            "and NFS-Ganesha. Do you still want to "
+            "continue?";
+    }
+
+    if (wordcount == 4) {
+        if (!strcmp("force", words[3])) {
+            flags |= GF_CLI_FLAG_OP_FORCE;
+        } else {
+            ret = -1;
+            cli_usage_out(word->pattern);
+            parse_error = 1;
+            goto out;
         }
+    }
 
-        if (wordcount == 4) {
-                if (!strcmp("force", words[3])) {
-                        flags |= GF_CLI_FLAG_OP_FORCE;
-                } else {
-                        ret = -1;
-                        cli_usage_out (word->pattern);
-                        parse_error = 1;
-                        goto out;
-                }
-        }
+    ret = dict_set_int32(dict, "flags", flags);
+    if (ret) {
+        gf_log(THIS->name, GF_LOG_ERROR, "dict set failed");
+        goto out;
+    }
 
-        ret = dict_set_int32 (dict, "flags", flags);
-        if (ret) {
-                gf_log (THIS->name, GF_LOG_ERROR,
-                        "dict set failed");
-                goto out;
-        }
+    answer = cli_cmd_get_confirmation(state, question);
 
-        answer = cli_cmd_get_confirmation (state, question);
+    if (GF_ANSWER_NO == answer) {
+        ret = 0;
+        goto out;
+    }
 
-        if (GF_ANSWER_NO == answer) {
-                ret = 0;
-                goto out;
-        }
+    proc = &cli_rpc_prog->proctable[GLUSTER_CLI_STOP_VOLUME];
 
-        proc = &cli_rpc_prog->proctable[GLUSTER_CLI_STOP_VOLUME];
+    frame = create_frame(THIS, THIS->ctx->pool);
+    if (!frame) {
+        ret = -1;
+        goto out;
+    }
 
-        CLI_LOCAL_INIT (local, words, frame, dict);
+    CLI_LOCAL_INIT(local, words, frame, dict);
 
-        if (proc->fn) {
-                ret = proc->fn (frame, THIS, dict);
-        }
+    if (proc->fn) {
+        ret = proc->fn(frame, THIS, dict);
+    }
 
 out:
-        if (ret) {
-                cli_cmd_sent_status_get (&sent);
-                if ((sent == 0) && (parse_error == 0))
-                        cli_out ("Volume stop on '%s' failed", volname);
-        }
-
-        CLI_STACK_DESTROY (frame);
-
-        return ret;
+    if (ret) {
+        cli_cmd_sent_status_get(&sent);
+        if ((sent == 0) && (parse_error == 0))
+            cli_out("Volume stop on '%s' failed", volname);
+    }
+
+    CLI_STACK_DESTROY(frame);
+    if (dict)
+        dict_unref(dict);
+
+    if (ret == 0 && GF_ANSWER_YES == answer) {
+        gf_event(EVENT_VOLUME_STOP, "name=%s;force=%d", (char *)words[2],
+                 (flags & GF_CLI_FLAG_OP_FORCE));
+    }
+
+    return ret;
 }
 
-
 int
-cli_cmd_volume_rename_cbk (struct cli_state *state, struct cli_cmd_word *word,
-                           const char **words, int wordcount)
+cli_cmd_volume_rename_cbk(struct cli_state *state, struct cli_cmd_word *word,
+                          const char **words, int wordcount)
 {
-        int                     ret = -1;
-        rpc_clnt_procedure_t    *proc = NULL;
-        call_frame_t            *frame = NULL;
-        dict_t                  *dict = NULL;
-        int                     sent = 0;
-        int                     parse_error = 0;
-
-
-        frame = create_frame (THIS, THIS->ctx->pool);
-        if (!frame)
-                goto out;
-
-        dict = dict_new ();
-        if (!dict)
-                goto out;
+    int ret = -1;
+    rpc_clnt_procedure_t *proc = NULL;
+    call_frame_t *frame = NULL;
+    dict_t *dict = NULL;
+    int sent = 0;
+    int parse_error = 0;
+
+    if (wordcount != 4) {
+        cli_usage_out(word->pattern);
+        parse_error = 1;
+        goto out;
+    }
 
-        if (wordcount != 4) {
-                cli_usage_out (word->pattern);
-                parse_error = 1;
-                goto out;
-        }
+    dict = dict_new();
+    if (!dict)
+        goto out;
 
-        ret = dict_set_str (dict, "old-volname", (char *)words[2]);
+    ret = dict_set_str(dict, "old-volname", (char *)words[2]);
 
-        if (ret)
-                goto out;
+    if (ret)
+        goto out;
 
-        ret = dict_set_str (dict, "new-volname", (char *)words[3]);
+    ret = dict_set_str(dict, "new-volname", (char *)words[3]);
 
-        if (ret)
-                goto out;
+    if (ret)
+        goto out;
 
-        proc = &cli_rpc_prog->proctable[GLUSTER_CLI_RENAME_VOLUME];
+    proc = &cli_rpc_prog->proctable[GLUSTER_CLI_RENAME_VOLUME];
 
-        if (proc->fn) {
-                ret = proc->fn (frame, THIS, dict);
+    if (proc->fn) {
+        frame = create_frame(THIS, THIS->ctx->pool);
+        if (!frame) {
+            ret = -1;
+            goto out;
         }
+        ret = proc->fn(frame, THIS, dict);
+    }
 
 out:
-        if (dict)
-                dict_destroy (dict);
+    if (dict)
+        dict_unref(dict);
 
-        if (ret) {
-                cli_cmd_sent_status_get (&sent);
-                if ((sent == 0) && (parse_error == 0))
-                        cli_out ("Volume rename on '%s' failed", (char *)words[2]);
-        }
+    if (ret) {
+        cli_cmd_sent_status_get(&sent);
+        if ((sent == 0) && (parse_error == 0))
+            cli_out("Volume rename on '%s' failed", (char *)words[2]);
+    }
 
-        CLI_STACK_DESTROY (frame);
+    CLI_STACK_DESTROY(frame);
 
-        return ret;
+    return ret;
 }
 
 int
-cli_cmd_volume_defrag_cbk (struct cli_state *state, struct cli_cmd_word *word,
-                           const char **words, int wordcount)
+cli_cmd_volume_defrag_cbk(struct cli_state *state, struct cli_cmd_word *word,
+                          const char **words, int wordcount)
 {
-        int                   ret     = -1;
-        rpc_clnt_procedure_t *proc    = NULL;
-        call_frame_t         *frame   = NULL;
-        dict_t               *dict = NULL;
-        int                     sent = 0;
-        int                     parse_error = 0;
-        cli_local_t          *local = NULL;
+    int ret = -1;
+    rpc_clnt_procedure_t *proc = NULL;
+    call_frame_t *frame = NULL;
+    dict_t *dict = NULL;
+    int sent = 0;
+    int parse_error = 0;
+    cli_local_t *local = NULL;
+#if (USE_EVENTS)
+    eventtypes_t event = EVENT_LAST;
+#endif
+
 #ifdef GF_SOLARIS_HOST_OS
-        cli_out ("Command not supported on Solaris");
-        goto out;
+    cli_out("Command not supported on Solaris");
+    goto out;
 #endif
 
-        frame = create_frame (THIS, THIS->ctx->pool);
-        if (!frame)
-                goto out;
+    ret = cli_cmd_volume_defrag_parse(words, wordcount, &dict);
 
-        ret = cli_cmd_volume_defrag_parse (words, wordcount, &dict);
+    if (ret) {
+        cli_usage_out(word->pattern);
+        parse_error = 1;
+    }
 
-        if (ret) {
-                cli_usage_out (word->pattern);
-                parse_error = 1;
-        }
+    proc = &cli_rpc_prog->proctable[GLUSTER_CLI_DEFRAG_VOLUME];
 
-        proc = &cli_rpc_prog->proctable[GLUSTER_CLI_DEFRAG_VOLUME];
+    frame = create_frame(THIS, THIS->ctx->pool);
+    if (!frame) {
+        ret = -1;
+        goto out;
+    }
 
-        CLI_LOCAL_INIT (local, words, frame, dict);
+    CLI_LOCAL_INIT(local, words, frame, dict);
 
-        if (proc->fn) {
-                ret = proc->fn (frame, THIS, dict);
-        }
+    if (proc->fn) {
+        ret = proc->fn(frame, THIS, dict);
+    }
 
 out:
-        if (ret) {
-                cli_cmd_sent_status_get (&sent);
-                if ((sent == 0) && (parse_error == 0))
-                        cli_out ("Volume rebalance failed");
-        }
+    if (ret) {
+        cli_cmd_sent_status_get(&sent);
+        if ((sent == 0) && (parse_error == 0))
+            cli_out("Volume rebalance failed");
+    } else {
+#if (USE_EVENTS)
+        if (!(strcmp(words[wordcount - 1], "start")) ||
+            !(strcmp(words[wordcount - 1], "force"))) {
+            event = EVENT_VOLUME_REBALANCE_START;
+        } else if (!strcmp(words[wordcount - 1], "stop")) {
+            event = EVENT_VOLUME_REBALANCE_STOP;
+        }
+
+        if (event != EVENT_LAST)
+            gf_event(event, "volume=%s", (char *)words[2]);
+#endif
+    }
 
-        CLI_STACK_DESTROY (frame);
+    CLI_STACK_DESTROY(frame);
 
-        return ret;
+    return ret;
 }
 
 int
-cli_cmd_volume_reset_cbk (struct cli_state *state, struct cli_cmd_word *word,
-                          const char **words, int wordcount)
+cli_cmd_volume_reset_cbk(struct cli_state *state, struct cli_cmd_word *word,
+                         const char **words, int wordcount)
 {
-        int                     sent = 0;
-        int                     parse_error = 0;
-        int                     ret = -1;
-        rpc_clnt_procedure_t    *proc = NULL;
-        call_frame_t            *frame = NULL;
-        dict_t                  *options = NULL;
-        cli_local_t             *local = NULL;
+    int sent = 0;
+    int parse_error = 0;
+    int ret = -1;
+    rpc_clnt_procedure_t *proc = NULL;
+    call_frame_t *frame = NULL;
+    dict_t *options = NULL;
+    cli_local_t *local = NULL;
+#if (USE_EVENTS)
+    int ret1 = -1;
+    char *tmp_opt = NULL;
+#endif
 
-        proc = &cli_rpc_prog->proctable[GLUSTER_CLI_RESET_VOLUME];
+    proc = &cli_rpc_prog->proctable[GLUSTER_CLI_RESET_VOLUME];
 
-        frame = create_frame (THIS, THIS->ctx->pool);
-        if (!frame)
-                goto out;
+    ret = cli_cmd_volume_reset_parse(words, wordcount, &options);
+    if (ret) {
+        cli_usage_out(word->pattern);
+        parse_error = 1;
+        goto out;
+    }
 
-        ret = cli_cmd_volume_reset_parse (words, wordcount, &options);
-        if (ret) {
-                cli_usage_out (word->pattern);
-                parse_error = 1;
-                goto out;
-        }
+    frame = create_frame(THIS, THIS->ctx->pool);
+    if (!frame) {
+        ret = -1;
+        goto out;
+    }
 
-        CLI_LOCAL_INIT (local, words, frame, options);
+    CLI_LOCAL_INIT(local, words, frame, options);
 
-        if (proc->fn) {
-                ret = proc->fn (frame, THIS, options);
-        }
+    if (proc->fn) {
+        ret = proc->fn(frame, THIS, options);
+    }
 
 out:
-        if (ret) {
-                cli_cmd_sent_status_get (&sent);
-                if ((sent == 0) && (parse_error == 0))
-                        cli_out ("Volume reset failed");
-        }
-
-        CLI_STACK_DESTROY (frame);
+    if (ret) {
+        cli_cmd_sent_status_get(&sent);
+        if ((sent == 0) && (parse_error == 0))
+            cli_out("Volume reset failed");
+    }
+
+#if (USE_EVENTS)
+    if (ret == 0) {
+        ret1 = dict_get_str(options, "key", &tmp_opt);
+        if (ret1)
+            tmp_opt = "";
+
+        gf_event(EVENT_VOLUME_RESET, "name=%s;option=%s", (char *)words[2],
+                 tmp_opt);
+    }
+#endif
 
-        return ret;
+    CLI_STACK_DESTROY(frame);
 
+    return ret;
 }
 
 int
-cli_cmd_volume_profile_cbk (struct cli_state *state, struct cli_cmd_word *word,
-                          const char **words, int wordcount)
+cli_cmd_volume_profile_cbk(struct cli_state *state, struct cli_cmd_word *word,
+                           const char **words, int wordcount)
 {
-        int                     sent = 0;
-        int                     parse_error = 0;
+    int sent = 0;
+    int parse_error = 0;
 
-        int                     ret      = -1;
-        rpc_clnt_procedure_t    *proc    = NULL;
-        call_frame_t            *frame   = NULL;
-        dict_t                  *options = NULL;
-        cli_local_t             *local = NULL;
+    int ret = -1;
+    rpc_clnt_procedure_t *proc = NULL;
+    call_frame_t *frame = NULL;
+    dict_t *options = NULL;
+    cli_local_t *local = NULL;
 
-        ret = cli_cmd_volume_profile_parse (words, wordcount, &options);
+    ret = cli_cmd_volume_profile_parse(words, wordcount, &options);
 
-        if (ret) {
-                cli_usage_out (word->pattern);
-                parse_error = 1;
-                goto out;
-        }
+    if (ret) {
+        cli_usage_out(word->pattern);
+        parse_error = 1;
+        goto out;
+    }
 
-        proc = &cli_rpc_prog->proctable[GLUSTER_CLI_PROFILE_VOLUME];
+    proc = &cli_rpc_prog->proctable[GLUSTER_CLI_PROFILE_VOLUME];
 
-        frame = create_frame (THIS, THIS->ctx->pool);
-        if (!frame)
-                goto out;
+    frame = create_frame(THIS, THIS->ctx->pool);
+    if (!frame) {
+        gf_log(THIS->name, GF_LOG_ERROR, "failed to create frame");
+        ret = -1;
+        goto out;
+    }
 
-        CLI_LOCAL_INIT (local, words, frame, options);
+    CLI_LOCAL_INIT(local, words, frame, options);
 
-        if (proc->fn) {
-                ret = proc->fn (frame, THIS, options);
-        }
+    if (proc->fn) {
+        ret = proc->fn(frame, THIS, options);
+    }
 
 out:
-        if (ret) {
-                cli_cmd_sent_status_get (&sent);
-                if ((sent == 0) && (parse_error == 0))
-                        cli_out ("Volume profile failed");
-        }
-
-        CLI_STACK_DESTROY (frame);
+    if (ret) {
+        cli_cmd_sent_status_get(&sent);
+        if ((sent == 0) && (parse_error == 0))
+            cli_out("Volume profile failed");
+    }
 
-        return ret;
+    CLI_STACK_DESTROY(frame);
 
+    return ret;
 }
 
 int
-cli_cmd_volume_set_cbk (struct cli_state *state, struct cli_cmd_word *word,
-                        const char **words, int wordcount)
+cli_cmd_volume_set_cbk(struct cli_state *state, struct cli_cmd_word *word,
+                       const char **words, int wordcount)
 {
-        int                     sent = 0;
-        int                     parse_error = 0;
-
-        int                     ret = -1;
-        rpc_clnt_procedure_t    *proc = NULL;
-        call_frame_t            *frame = NULL;
-        dict_t                  *options = NULL;
-        cli_local_t             *local = NULL;
-        char                    *op_errstr = NULL;
+    int sent = 0;
+    int parse_error = 0;
+
+    int ret = -1;
+    rpc_clnt_procedure_t *proc = NULL;
+    call_frame_t *frame = NULL;
+    dict_t *options = NULL;
+    cli_local_t *local = NULL;
+    char *op_errstr = NULL;
+
+#if (USE_EVENTS)
+    int ret1 = -1;
+    int i = 1;
+    char dict_key[50] = {
+        0,
+    };
+    char *tmp_opt = NULL;
+    char *opts_str = NULL;
+    int num_options = 0;
+#endif
 
-        proc = &cli_rpc_prog->proctable[GLUSTER_CLI_SET_VOLUME];
+    proc = &cli_rpc_prog->proctable[GLUSTER_CLI_SET_VOLUME];
 
-        frame = create_frame (THIS, THIS->ctx->pool);
-        if (!frame)
-                goto out;
+    ret = cli_cmd_volume_set_parse(state, words, wordcount, &options,
+                                   &op_errstr);
+    if (ret) {
+        if (op_errstr) {
+            cli_err("%s", op_errstr);
+            GF_FREE(op_errstr);
+        } else
+            cli_usage_out(word->pattern);
 
-        ret = cli_cmd_volume_set_parse (state, words, wordcount,
-                                        &options, &op_errstr);
-        if (ret) {
-                if (op_errstr) {
-                    cli_err ("%s", op_errstr);
-                    GF_FREE (op_errstr);
-                } else
-                    cli_usage_out (word->pattern);
+        parse_error = 1;
+        goto out;
+    }
 
-                parse_error = 1;
-                goto out;
-        }
+    frame = create_frame(THIS, THIS->ctx->pool);
+    if (!frame) {
+        ret = -1;
+        goto out;
+    }
 
-        CLI_LOCAL_INIT (local, words, frame, options);
+    CLI_LOCAL_INIT(local, words, frame, options);
 
-        if (proc->fn) {
-                ret = proc->fn (frame, THIS, options);
-        }
+    if (proc->fn) {
+        ret = proc->fn(frame, THIS, options);
+    }
 
 out:
-        if (ret) {
-                cli_cmd_sent_status_get (&sent);
-                if ((sent == 0) && (parse_error == 0))
-                        cli_out ("Volume set failed");
+    if (ret) {
+        cli_cmd_sent_status_get(&sent);
+        if ((sent == 0) && (parse_error == 0))
+            cli_out("Volume set failed");
+    }
+
+#if (USE_EVENTS)
+    if (ret == 0 && strcmp(words[2], "help") != 0) {
+        ret1 = dict_get_int32(options, "count", &num_options);
+        if (ret1) {
+            num_options = 0;
+            goto end;
+        } else {
+            num_options = num_options / 2;
         }
 
-        CLI_STACK_DESTROY (frame);
+        char *free_list_key[num_options];
+        char *free_list_val[num_options];
+        for (i = 0; i < num_options; i++) {
+            free_list_key[i] = NULL;
+            free_list_val[i] = NULL;
+        }
+        /* Initialize opts_str */
+        opts_str = "";
 
-        return ret;
+        /* Prepare String in format options=KEY1,VALUE1,KEY2,VALUE2 */
+        for (i = 1; i <= num_options; i++) {
+            sprintf(dict_key, "key%d", i);
+            ret1 = dict_get_str(options, dict_key, &tmp_opt);
+            if (ret1)
+                tmp_opt = "";
 
-}
+            gf_asprintf(&opts_str, "%s,%s", opts_str, tmp_opt);
+            free_list_key[i - 1] = opts_str;
 
-int
-cli_cmd_volume_add_brick_cbk (struct cli_state *state,
-                              struct cli_cmd_word *word, const char **words,
-                              int wordcount)
-{
-        int                     ret = -1;
-        rpc_clnt_procedure_t    *proc = NULL;
-        call_frame_t            *frame = NULL;
-        dict_t                  *options = NULL;
-        int                     sent = 0;
-        int                     parse_error = 0;
-        gf_answer_t             answer = GF_ANSWER_NO;
-        cli_local_t             *local = NULL;
-
-        const char *question = "Changing the 'stripe count' of the volume is "
-                "not a supported feature. In some cases it may result in data "
-                "loss on the volume. Also there may be issues with regular "
-                "filesystem operations on the volume after the change. Do you "
-                "really want to continue with 'stripe' count option ? ";
-
-        frame = create_frame (THIS, THIS->ctx->pool);
-        if (!frame)
-                goto out;
+            sprintf(dict_key, "value%d", i);
+            ret1 = dict_get_str(options, dict_key, &tmp_opt);
+            if (ret1)
+                tmp_opt = "";
 
-        ret = cli_cmd_volume_add_brick_parse (words, wordcount, &options, 0);
-        if (ret) {
-                cli_usage_out (word->pattern);
-                parse_error = 1;
-                goto out;
+            gf_asprintf(&opts_str, "%s,%s", opts_str, tmp_opt);
+            free_list_val[i - 1] = opts_str;
         }
 
-        /* TODO: there are challenges in supporting changing of
-           stripe-count, until it is properly supported give warning to user */
-        if (dict_get (options, "stripe-count")) {
-                answer = cli_cmd_get_confirmation (state, question);
+        gf_event(EVENT_VOLUME_SET, "name=%s;options=%s", (char *)words[2],
+                 opts_str);
 
-                if (GF_ANSWER_NO == answer) {
-                        ret = 0;
-                        goto out;
-                }
+        /* Allocated by gf_strdup and gf_asprintf */
+        for (i = 0; i < num_options; i++) {
+            GF_FREE(free_list_key[i]);
+            GF_FREE(free_list_val[i]);
         }
+    }
+#endif
 
-        if (state->mode & GLUSTER_MODE_WIGNORE) {
-                ret = dict_set_int32 (options, "force", _gf_true);
-                if (ret) {
-                        gf_log ("cli", GF_LOG_ERROR, "Failed to set force "
-                                "option");
-                        goto out;
-                }
-        }
+end:
+    CLI_STACK_DESTROY(frame);
 
-        proc = &cli_rpc_prog->proctable[GLUSTER_CLI_ADD_BRICK];
+    return ret;
+}
 
-        CLI_LOCAL_INIT (local, words, frame, options);
+static int
+cli_event_remove_brick_str(dict_t *options, char **event_str,
+                           eventtypes_t *event)
+{
+    int ret = -1;
+    char *bricklist = NULL;
+    char *brick = NULL;
+    char *volname = NULL;
+    char key[256] = {
+        0,
+    };
+    const char *eventstrformat = "volume=%s;bricks=%s";
+    int32_t command = 0;
+    int32_t i = 1;
+    int32_t count = 0;
+    int32_t eventstrlen = 1;
+    int bricklen = 0;
+    char *tmp_ptr = NULL;
+
+    if (!options || !event_str || !event)
+        goto out;
 
-        if (proc->fn) {
-                ret = proc->fn (frame, THIS, options);
-        }
+    ret = dict_get_str(options, "volname", &volname);
+    if (ret || !volname) {
+        gf_log("cli", GF_LOG_ERROR, "Failed to fetch volname");
+        ret = -1;
+        goto out;
+    }
+    /* Get the list of bricks for the event */
+    ret = dict_get_int32(options, "command", &command);
+    if (ret) {
+        gf_log("cli", GF_LOG_ERROR, "Failed to fetch command");
+        ret = -1;
+        goto out;
+    }
+
+    switch (command) {
+        case GF_OP_CMD_START:
+            *event = EVENT_VOLUME_REMOVE_BRICK_START;
+            break;
+        case GF_OP_CMD_COMMIT:
+            *event = EVENT_VOLUME_REMOVE_BRICK_COMMIT;
+            break;
+        case GF_OP_CMD_COMMIT_FORCE:
+            *event = EVENT_VOLUME_REMOVE_BRICK_FORCE;
+            break;
+        case GF_OP_CMD_STOP:
+            *event = EVENT_VOLUME_REMOVE_BRICK_STOP;
+            break;
+        default:
+            *event = EVENT_LAST;
+            break;
+    }
 
-out:
+    ret = -1;
+
+    if (*event == EVENT_LAST) {
+        goto out;
+    }
+
+    /* I could just get this from words[] but this is cleaner in case the
+     * format changes  */
+    while (i) {
+        snprintf(key, sizeof(key), "brick%d", i);
+        ret = dict_get_str(options, key, &brick);
         if (ret) {
-                cli_cmd_sent_status_get (&sent);
-                if ((sent == 0) && (parse_error == 0))
-                        cli_out ("Volume add-brick failed");
+            break;
         }
+        eventstrlen += strlen(brick) + 1;
+        i++;
+    }
 
-        CLI_STACK_DESTROY (frame);
+    count = --i;
 
-        return ret;
-}
+    eventstrlen += 1;
 
-int
-cli_tier_validate_replica_type (dict_t *dict, int type)
-{
+    bricklist = GF_CALLOC(eventstrlen, sizeof(char), gf_common_mt_char);
+    if (!bricklist) {
+        gf_log(THIS->name, GF_LOG_ERROR,
+               "memory allocation failed for"
+               "bricklist");
+        ret = -1;
+        goto out;
+    }
 
-        int             brick_count             = -1;
-        int             replica_count           = 1;
-        int             ret                     = -1;
+    tmp_ptr = bricklist;
 
-        ret = dict_get_int32 (dict, "count", &brick_count);
+    i = 1;
+    while (i <= count) {
+        snprintf(key, sizeof(key), "brick%d", i);
+        ret = dict_get_str(options, key, &brick);
         if (ret) {
-                gf_log ("cli", GF_LOG_ERROR, "Failed to get brick count");
-                goto out;
+            break;
         }
+        snprintf(tmp_ptr, eventstrlen, "%s ", brick);
+        bricklen = strlen(brick);
+        eventstrlen -= (bricklen + 1);
+        tmp_ptr += (bricklen + 1);
+        i++;
+    }
 
-        ret = dict_get_int32 (dict, "replica-count", &replica_count);
-        if (ret) {
-                gf_log ("cli", GF_LOG_DEBUG, "Failed to get replica count. "
-                        "Defaulting to one");
-                replica_count = 1;
-        }
+    if (!ret) {
+        gf_asprintf(event_str, eventstrformat, volname, bricklist);
+    } else {
+        gf_asprintf(event_str, eventstrformat, volname, "<unavailable>");
+    }
 
-        /*
-         * Change the calculation of sub_count once attach-tier support
-         * disperse volume.
-         * sub_count = disperse_count for disperse volume
-         * */
-
-
-        if (brick_count % replica_count) {
-             if (type == GF_CLUSTER_TYPE_REPLICATE)
-                     cli_err ("number of bricks is not a multiple of "
-                              "replica count");
-             else if (type == GF_CLUSTER_TYPE_DISPERSE)
-                     cli_err ("number of bricks is not a multiple of "
-                              "disperse count");
-             else
-                     cli_err ("number of bricks given doesn't match "
-                              "required count");
-
-             ret = -1;
-             goto out;
-     }
-        ret = 0;
+    ret = 0;
 out:
-        return ret;
+    GF_FREE(bricklist);
+    return ret;
 }
 
 int
-do_cli_cmd_volume_attach_tier (struct cli_state *state,
-                                struct cli_cmd_word *word, const char **words,
-                                int wordcount)
+cli_cmd_volume_add_brick_cbk(struct cli_state *state, struct cli_cmd_word *word,
+                             const char **words, int wordcount)
 {
-        int                     ret = -1;
-        rpc_clnt_procedure_t    *proc = NULL;
-        call_frame_t            *frame = NULL;
-        dict_t                  *options = NULL;
-        int                     sent = 0;
-        int                     parse_error = 0;
-        cli_local_t             *local = NULL;
-        int                     type = 0;
-
-        frame = create_frame (THIS, THIS->ctx->pool);
-        if (!frame)
-                goto out;
-
-        ret = cli_cmd_volume_add_brick_parse (words, wordcount, &options, &type);
-        if (ret) {
-                cli_usage_out (word->pattern);
-                parse_error = 1;
-                goto out;
-        }
-
-        /*
-         * Merge this check when attach-tier has it's own cli parse function.
-         */
-        ret = cli_tier_validate_replica_type (options, type);
-        if (ret) {
-                cli_usage_out (word->pattern);
-                parse_error = 1;
-                goto out;
-        }
-
-        if (state->mode & GLUSTER_MODE_WIGNORE) {
-                ret = dict_set_int32 (options, "force", _gf_true);
-                if (ret) {
-                        gf_log ("cli", GF_LOG_ERROR, "Failed to set force "
-                                "option");
-                        goto out;
-                }
-        }
-
-        ret = dict_set_int32 (options, "attach-tier", 1);
-        if (ret)
-                goto out;
-
-        ret = dict_set_int32 (options, "hot-type", type);
-        if (ret)
-                goto out;
-
-        proc = &cli_rpc_prog->proctable[GLUSTER_CLI_ATTACH_TIER];
+    int ret = -1;
+    rpc_clnt_procedure_t *proc = NULL;
+    call_frame_t *frame = NULL;
+    dict_t *options = NULL;
+    int sent = 0;
+    int parse_error = 0;
+    gf_answer_t answer = GF_ANSWER_NO;
+    cli_local_t *local = NULL;
+
+#if (USE_EVENTS)
+    char *event_str = NULL;
+    char *bricks = NULL;
+    const char *eventstrformat = "volume=%s;bricks=%s";
+#endif
 
-        CLI_LOCAL_INIT (local, words, frame, options);
+    const char *question =
+        "Changing the 'stripe count' of the volume is "
+        "not a supported feature. In some cases it may result in data "
+        "loss on the volume. Also there may be issues with regular "
+        "filesystem operations on the volume after the change. Do you "
+        "really want to continue with 'stripe' count option ? ";
+
+    ret = cli_cmd_volume_add_brick_parse(state, words, wordcount, &options, 0);
+    if (ret) {
+        cli_usage_out(word->pattern);
+        parse_error = 1;
+        goto out;
+    }
 
-        if (proc->fn) {
-                ret = proc->fn (frame, THIS, options);
-        }
+    /* TODO: there are challenges in supporting changing of
+       stripe-count, until it is properly supported give warning to user */
+    if (dict_get(options, "stripe-count")) {
+        answer = cli_cmd_get_confirmation(state, question);
 
-out:
-        if (ret) {
-                cli_cmd_sent_status_get (&sent);
-                if ((sent == 0) && (parse_error == 0))
-                        cli_out ("attach-tier failed");
+        if (GF_ANSWER_NO == answer) {
+            ret = 0;
+            goto out;
         }
+    }
 
-        CLI_STACK_DESTROY (frame);
+#if (USE_EVENTS)
+    /* Get the list of bricks for the event */
 
-        return ret;
-}
+    ret = dict_get_str(options, "bricks", &bricks);
 
-int
-do_cli_cmd_volume_detach_tier (struct cli_state *state,
-                              struct cli_cmd_word *word, const char **words,
-                              int wordcount)
-{
-        int                     ret = -1;
-        rpc_clnt_procedure_t    *proc = NULL;
-        call_frame_t            *frame = NULL;
-        dict_t                  *options = NULL;
-        int                     sent = 0;
-        int                     parse_error = 0;
-        gf_answer_t             answer = GF_ANSWER_NO;
-        cli_local_t             *local = NULL;
-        int                     need_question = 0;
-
-        const char *question = "Removing tier can result in data loss. "
-                               "Do you want to Continue?";
-
-        frame = create_frame (THIS, THIS->ctx->pool);
-        if (!frame)
-                goto out;
+    if (!ret) {
+        gf_asprintf(&event_str, eventstrformat, (char *)words[2],
+                    &bricks[1] /*Skip leading space*/);
+    } else {
+        gf_asprintf(&event_str, eventstrformat, (char *)words[2],
+                    "<unavailable>");
+    }
+#endif
 
-        ret = cli_cmd_volume_detach_tier_parse(words, wordcount, &options,
-                                               &need_question);
+    if (state->mode & GLUSTER_MODE_WIGNORE) {
+        ret = dict_set_int32(options, "force", _gf_true);
         if (ret) {
-                cli_usage_out (word->pattern);
-                parse_error = 1;
-                goto out;
+            gf_log("cli", GF_LOG_ERROR,
+                   "Failed to set force "
+                   "option");
+            goto out;
         }
+    }
 
-        ret = dict_set_int32 (options, "force", 1);
-        if (ret)
-                goto out;
-
-        ret = dict_set_int32 (options, "count", 0);
-        if (ret)
-                goto out;
-
-        if (!(state->mode & GLUSTER_MODE_SCRIPT) && need_question) {
-                /* we need to ask question only in case of 'commit or force' */
-                answer = cli_cmd_get_confirmation (state, question);
-                if (GF_ANSWER_NO == answer) {
-                        ret = 0;
-                        goto out;
-                }
-        }
+    proc = &cli_rpc_prog->proctable[GLUSTER_CLI_ADD_BRICK];
 
-        proc = &cli_rpc_prog->proctable[GLUSTER_CLI_DETACH_TIER];
+    frame = create_frame(THIS, THIS->ctx->pool);
+    if (!frame) {
+        ret = -1;
+        goto out;
+    }
 
-        CLI_LOCAL_INIT (local, words, frame, options);
+    CLI_LOCAL_INIT(local, words, frame, options);
 
-        if (proc->fn) {
-                ret = proc->fn (frame, THIS, options);
-        }
+    if (proc->fn) {
+        ret = proc->fn(frame, THIS, options);
+    }
 
 out:
-        if (ret) {
-                cli_cmd_sent_status_get (&sent);
-                if ((sent == 0) && (parse_error == 0))
-                        cli_out ("Volume detach-tier failed");
-        }
-
-        CLI_STACK_DESTROY (frame);
+    if (ret) {
+        cli_cmd_sent_status_get(&sent);
+        if ((sent == 0) && (parse_error == 0))
+            cli_out("Volume add-brick failed");
+    } else {
+#if (USE_EVENTS)
+        gf_event(EVENT_VOLUME_ADD_BRICK, "%s", event_str);
+#endif
+    }
+#if (USE_EVENTS)
+    GF_FREE(event_str);
+#endif
 
-        return ret;
+    CLI_STACK_DESTROY(frame);
+    return ret;
 }
 
 int
-cli_cmd_volume_tier_cbk (struct cli_state *state,
-                         struct cli_cmd_word *word, const char **words,
-                         int wordcount)
+cli_get_soft_limit(dict_t *options, const char **words, dict_t *xdata)
 {
-        int                      ret     = -1;
-        call_frame_t            *frame   = NULL;
-        dict_t                  *options = NULL;
-        char                    *volname = NULL;
-        rpc_clnt_procedure_t    *proc    = NULL;
-        cli_local_t             *local   = NULL;
-        int                      i       = 0;
-
-        if (wordcount < 4) {
-                cli_usage_out (word->pattern);
-                if (wordcount == 3 && !strcmp(words[2], "help"))
-                        ret = 0;
-                goto out;
-        }
-
-        if (!strcmp(words[1], "detach-tier")) {
-                ret = do_cli_cmd_volume_detach_tier (state, word,
-                                                     words, wordcount);
-                goto out;
-        } else if (!strcmp(words[3], "detach")) {
-                for (i = 3; i < wordcount; i++)
-                        words[i] = words[i+1];
+    call_frame_t *frame = NULL;
+    cli_local_t *local = NULL;
+    rpc_clnt_procedure_t *proc = NULL;
+    char *default_sl = NULL;
+    char *default_sl_dup = NULL;
+    int ret = -1;
+
+    frame = create_frame(THIS, THIS->ctx->pool);
+    if (!frame) {
+        ret = -1;
+        goto out;
+    }
+
+    // We need a ref on @options to prevent CLI_STACK_DESTROY
+    // from destroying it prematurely.
+    dict_ref(options);
+    CLI_LOCAL_INIT(local, words, frame, options);
+    proc = &cli_rpc_prog->proctable[GLUSTER_CLI_QUOTA];
+    ret = proc->fn(frame, THIS, options);
+
+    ret = dict_get_str(options, "default-soft-limit", &default_sl);
+    if (ret) {
+        gf_log("cli", GF_LOG_ERROR, "Failed to get default soft limit");
+        goto out;
+    }
 
-                ret = do_cli_cmd_volume_detach_tier (state, word,
-                                                     words, wordcount-1);
-                goto out;
+    default_sl_dup = gf_strdup(default_sl);
+    if (!default_sl_dup) {
+        ret = -1;
+        goto out;
+    }
 
-        } else if (!strcmp(words[1], "attach-tier")) {
-                ret = do_cli_cmd_volume_attach_tier (state, word,
-                                                     words, wordcount);
-                goto out;
-        } else if (!strcmp(words[3], "attach")) {
-                for (i = 3; i < wordcount; i++)
-                        words[i] = words[i+1];
+    ret = dict_set_dynstr(xdata, "default-soft-limit", default_sl_dup);
+    if (ret) {
+        gf_log("cli", GF_LOG_ERROR, "Failed to set default soft limit");
+        GF_FREE(default_sl_dup);
+        goto out;
+    }
 
-                ret = do_cli_cmd_volume_attach_tier (state, word,
-                                                     words, wordcount-1);
-                goto out;
-        }
+out:
+    CLI_STACK_DESTROY(frame);
+    return ret;
+}
 
-        ret = cli_cmd_volume_tier_parse (words, wordcount, &options);
-        if (ret) {
-                cli_usage_out (word->pattern);
-                goto out;
-        }
+/* Checks if at least one limit has been set on the volume
+ *
+ * Returns true if at least one limit is set. Returns false otherwise.
+ */
+gf_boolean_t
+_limits_set_on_volume(char *volname, int type)
+{
+    gf_boolean_t limits_set = _gf_false;
+    int ret = -1;
+    char quota_conf_file[PATH_MAX] = {
+        0,
+    };
+    int fd = -1;
+    char buf[16] = {
+        0,
+    };
+    float version = 0.0f;
+    char gfid_type_stored = 0;
+    char gfid_type = 0;
+
+    /* TODO: fix hardcoding; Need to perform an RPC call to glusterd
+     * to fetch working directory
+     */
+    snprintf(quota_conf_file, sizeof quota_conf_file, "%s/vols/%s/quota.conf",
+             GLUSTERD_DEFAULT_WORKDIR, volname);
+    fd = open(quota_conf_file, O_RDONLY);
+    if (fd == -1)
+        goto out;
 
-        proc = &cli_rpc_prog->proctable[GLUSTER_CLI_TIER];
+    ret = quota_conf_read_version(fd, &version);
+    if (ret)
+        goto out;
 
-        frame = create_frame (THIS, THIS->ctx->pool);
-        if (!frame)
-                goto out;
+    if (type == GF_QUOTA_OPTION_TYPE_LIST)
+        gfid_type = GF_QUOTA_CONF_TYPE_USAGE;
+    else
+        gfid_type = GF_QUOTA_CONF_TYPE_OBJECTS;
 
-        CLI_LOCAL_INIT (local, words, frame, options);
+    /* Try to read at least one gfid  of type 'gfid_type' */
+    while (1) {
+        ret = quota_conf_read_gfid(fd, buf, &gfid_type_stored, version);
+        if (ret <= 0)
+            break;
 
-        if (proc->fn) {
-                ret = proc->fn (frame, THIS, options);
+        if (gfid_type_stored == gfid_type) {
+            limits_set = _gf_true;
+            break;
         }
-
+    }
 out:
-        if (ret) {
-                cli_out ("Tier command failed");
-        }
-        if (options)
-                dict_unref (options);
+    if (fd != -1)
+        sys_close(fd);
 
-        return ret;
+    return limits_set;
 }
 
-static int
-gf_cli_create_auxiliary_mount (char *volname)
+int
+cli_cmd_quota_handle_list_all(const char **words, dict_t *options)
 {
-        int      ret                     = -1;
-        char     mountdir[PATH_MAX]      = {0,};
-        char     pidfile_path[PATH_MAX]  = {0,};
-        char     logfile[PATH_MAX]       = {0,};
-        char     qpid [16]               = {0,};
-        char     *sockpath               = NULL;
-
-        GLUSTERFS_GET_AUX_MOUNT_PIDFILE (pidfile_path, volname);
-
-        if (gf_is_service_running (pidfile_path, NULL)) {
-                gf_log ("cli", GF_LOG_DEBUG, "Aux mount of volume %s is running"
-                        " already", volname);
-                ret = 0;
-                goto out;
-        }
+    int all_failed = 1;
+    int count = 0;
+    int ret = -1;
+    rpc_clnt_procedure_t *proc = NULL;
+    cli_local_t *local = NULL;
+    call_frame_t *frame = NULL;
+    dict_t *xdata = NULL;
+    char gfid_str[UUID_CANONICAL_FORM_LEN + 1];
+    char *volname = NULL;
+    char *volname_dup = NULL;
+    unsigned char buf[16] = {0};
+    int fd = -1;
+    char quota_conf_file[PATH_MAX] = {0};
+    gf_boolean_t xml_err_flag = _gf_false;
+    char err_str[NAME_MAX] = {
+        0,
+    };
+    int32_t type = 0;
+    char gfid_type = 0;
+    float version = 0.0f;
+    int32_t max_count = 0;
+
+    xdata = dict_new();
+    if (!xdata) {
+        ret = -1;
+        goto out;
+    }
 
-        GLUSTERD_GET_QUOTA_AUX_MOUNT_PATH (mountdir, volname, "/");
-        ret = sys_mkdir (mountdir, 0777);
-        if (ret && errno != EEXIST) {
-                gf_log ("cli", GF_LOG_ERROR, "Failed to create auxiliary mount "
-                        "directory %s. Reason : %s", mountdir,
-                        strerror (errno));
-                goto out;
-        }
+    ret = dict_get_str(options, "volname", &volname);
+    if (ret) {
+        gf_log("cli", GF_LOG_ERROR, "Failed to get volume name");
+        goto out;
+    }
 
-        snprintf (logfile, PATH_MAX-1, "%s/quota-mount-%s.log",
-                  DEFAULT_LOG_FILE_DIRECTORY, volname);
-        snprintf(qpid, 15, "%d", GF_CLIENT_PID_QUOTA_MOUNT);
+    ret = dict_get_int32(options, "type", &type);
+    if (ret) {
+        gf_log("cli", GF_LOG_ERROR, "Failed to get quota option type");
+        goto out;
+    }
 
-        if (global_state->glusterd_sock) {
-                sockpath = global_state->glusterd_sock;
+    ret = dict_set_int32(xdata, "type", type);
+    if (ret) {
+        gf_log("cli", GF_LOG_ERROR, "Failed to set type in xdata");
+        goto out;
+    }
+
+    ret = cli_get_soft_limit(options, words, xdata);
+    if (ret) {
+        gf_log("cli", GF_LOG_ERROR,
+               "Failed to fetch default "
+               "soft-limit");
+        goto out;
+    }
+
+    /* Check if at least one limit is set on volume. No need to check for
+     * quota enabled as cli_get_soft_limit() handles that
+     */
+    if (!_limits_set_on_volume(volname, type)) {
+        snprintf(err_str, sizeof(err_str),
+                 "No%s quota configured on"
+                 " volume %s",
+                 (type == GF_QUOTA_OPTION_TYPE_LIST) ? "" : " inode", volname);
+        if (global_state->mode & GLUSTER_MODE_XML) {
+            xml_err_flag = _gf_true;
         } else {
-                sockpath = DEFAULT_GLUSTERD_SOCKFILE;
+            cli_out("quota: %s", err_str);
         }
+        ret = 0;
+        goto out;
+    }
 
-        ret = runcmd (SBIN_DIR"/glusterfs",
-                      "--volfile-server", sockpath,
-                      "--volfile-server-transport", "unix",
-                      "--volfile-id", volname,
-                      "-l", logfile,
-                      "-p", pidfile_path,
-                      "--client-pid", qpid,
-                      mountdir,
-                      NULL);
-
-        if (ret) {
-                gf_log ("cli", GF_LOG_WARNING, "failed to mount glusterfs "
-                        "client. Please check the log file %s for more details",
-                        logfile);
-                ret = -1;
-                goto out;
-        }
+    volname_dup = gf_strdup(volname);
+    if (!volname_dup) {
+        ret = -1;
+        goto out;
+    }
 
+    ret = dict_set_dynstr(xdata, "volume-uuid", volname_dup);
+    if (ret) {
+        gf_log("cli", GF_LOG_ERROR, "Failed to set volume-uuid");
+        GF_FREE(volname_dup);
+        goto out;
+    }
+
+    // TODO: fix hardcoding; Need to perform an RPC call to glusterd
+    // to fetch working directory
+    snprintf(quota_conf_file, sizeof quota_conf_file, "%s/vols/%s/quota.conf",
+             GLUSTERD_DEFAULT_WORKDIR, volname);
+    fd = open(quota_conf_file, O_RDONLY);
+    if (fd == -1) {
+        // This may because no limits were yet set on the volume
+        gf_log("cli", GF_LOG_TRACE,
+               "Unable to open "
+               "quota.conf");
         ret = 0;
+        goto out;
+    }
 
-out:
-        return ret;
-}
+    ret = quota_conf_read_version(fd, &version);
+    if (ret)
+        goto out;
 
-static int
-cli_stage_quota_op (char *volname, int op_code)
-{
-        int ret = -1;
-
-        switch (op_code) {
-        case GF_QUOTA_OPTION_TYPE_ENABLE:
-        case GF_QUOTA_OPTION_TYPE_LIMIT_USAGE:
-        case GF_QUOTA_OPTION_TYPE_LIMIT_OBJECTS:
-        case GF_QUOTA_OPTION_TYPE_REMOVE:
-        case GF_QUOTA_OPTION_TYPE_REMOVE_OBJECTS:
-        case GF_QUOTA_OPTION_TYPE_LIST:
-                ret = gf_cli_create_auxiliary_mount (volname);
-                if (ret) {
-                        cli_err ("quota: Could not start quota "
-                                 "auxiliary mount");
-                        goto out;
-                }
-                ret = 0;
-                break;
+    frame = create_frame(THIS, THIS->ctx->pool);
+    if (!frame) {
+        ret = -1;
+        goto out;
+    }
+
+    CLI_LOCAL_INIT(local, words, frame, xdata);
+    proc = &cli_quotad_clnt.proctable[GF_AGGREGATOR_GETLIMIT];
+
+    for (count = 0;; count++) {
+        ret = quota_conf_read_gfid(fd, buf, &gfid_type, version);
+        if (ret == 0) {
+            break;
+        } else if (ret < 0) {
+            gf_log(THIS->name, GF_LOG_CRITICAL,
+                   "Quota "
+                   "configuration store may be corrupt.");
+            goto out;
+        }
+
+        if ((type == GF_QUOTA_OPTION_TYPE_LIST &&
+             gfid_type == GF_QUOTA_CONF_TYPE_OBJECTS) ||
+            (type == GF_QUOTA_OPTION_TYPE_LIST_OBJECTS &&
+             gfid_type == GF_QUOTA_CONF_TYPE_USAGE))
+            continue;
+
+        max_count++;
+    }
+    ret = dict_set_int32(xdata, "max_count", max_count);
+    if (ret) {
+        gf_log("cli", GF_LOG_ERROR, "Failed to set max_count");
+        goto out;
+    }
+
+    ret = sys_lseek(fd, 0L, SEEK_SET);
+    if (ret < 0) {
+        gf_log(THIS->name, GF_LOG_ERROR,
+               "failed to move offset to "
+               "the beginning: %s",
+               strerror(errno));
+        goto out;
+    }
+    ret = quota_conf_read_version(fd, &version);
+    if (ret)
+        goto out;
 
-        default:
-                ret = 0;
-                break;
+    for (count = 0;; count++) {
+        ret = quota_conf_read_gfid(fd, buf, &gfid_type, version);
+        if (ret == 0) {
+            break;
+        } else if (ret < 0) {
+            gf_log(THIS->name, GF_LOG_CRITICAL,
+                   "Quota "
+                   "configuration store may be corrupt.");
+            goto out;
         }
 
-out:
-        return ret;
-}
+        if ((type == GF_QUOTA_OPTION_TYPE_LIST &&
+             gfid_type == GF_QUOTA_CONF_TYPE_OBJECTS) ||
+            (type == GF_QUOTA_OPTION_TYPE_LIST_OBJECTS &&
+             gfid_type == GF_QUOTA_CONF_TYPE_USAGE))
+            continue;
 
-int
-cli_get_soft_limit (dict_t *options, const char **words, dict_t *xdata)
-{
-        call_frame_t            *frame          = NULL;
-        cli_local_t             *local          = NULL;
-        rpc_clnt_procedure_t    *proc           = NULL;
-        char                    *default_sl     = NULL;
-        char                    *default_sl_dup = NULL;
-        int                      ret  = -1;
-
-        frame = create_frame (THIS, THIS->ctx->pool);
-        if (!frame) {
-                ret = -1;
-                goto out;
+        uuid_utoa_r(buf, gfid_str);
+        ret = dict_set_str(xdata, "gfid", gfid_str);
+        if (ret) {
+            gf_log("cli", GF_LOG_ERROR, "Failed to set gfid");
+            goto out;
         }
 
-        //We need a ref on @options to prevent CLI_STACK_DESTROY
-        //from destroying it prematurely.
-        dict_ref (options);
-        CLI_LOCAL_INIT (local, words, frame, options);
-        proc = &cli_rpc_prog->proctable[GLUSTER_CLI_QUOTA];
-        ret = proc->fn (frame, THIS, options);
-
-        ret = dict_get_str (options, "default-soft-limit", &default_sl);
+        ret = proc->fn(frame, THIS, xdata);
         if (ret) {
-                gf_log ("cli", GF_LOG_ERROR, "Failed to get default soft limit");
-                goto out;
+            gf_log("cli", GF_LOG_ERROR,
+                   "Failed to get quota "
+                   "limits for %s",
+                   uuid_utoa((unsigned char *)buf));
         }
 
-        default_sl_dup = gf_strdup (default_sl);
-        if (!default_sl_dup) {
-                ret = -1;
-                goto out;
-        }
+        dict_del(xdata, "gfid");
+        all_failed = all_failed && ret;
+    }
 
-        ret = dict_set_dynstr (xdata, "default-soft-limit", default_sl_dup);
+    if (global_state->mode & GLUSTER_MODE_XML) {
+        ret = cli_xml_output_vol_quota_limit_list_end(local);
         if (ret) {
-                gf_log ("cli", GF_LOG_ERROR, "Failed to set default soft limit");
-                GF_FREE (default_sl_dup);
-                goto out;
+            gf_log("cli", GF_LOG_ERROR,
+                   "Error in printing "
+                   "xml output");
+            goto out;
         }
+    }
 
-out:
-        CLI_STACK_DESTROY (frame);
-        return ret;
-}
-
-/* Checks if at least one limit has been set on the volume
- *
- * Returns true if at least one limit is set. Returns false otherwise.
- */
-gf_boolean_t
-_limits_set_on_volume (char *volname, int type) {
-        gf_boolean_t    limits_set                = _gf_false;
-        int             ret                       = -1;
-        char            quota_conf_file[PATH_MAX] = {0,};
-        int             fd                        = -1;
-        char            buf[16]                   = {0,};
-        float           version                   = 0.0f;
-        char            gfid_type_stored          = 0;
-        char            gfid_type                 = 0;
-
-        /* TODO: fix hardcoding; Need to perform an RPC call to glusterd
-         * to fetch working directory
-         */
-        snprintf (quota_conf_file, sizeof quota_conf_file,
-                  "%s/vols/%s/quota.conf",
-                  GLUSTERD_DEFAULT_WORKDIR,
-                  volname);
-        fd = open (quota_conf_file, O_RDONLY);
-        if (fd == -1)
-                goto out;
-
-        ret = quota_conf_read_version (fd, &version);
-        if (ret)
-                goto out;
+    if (count > 0) {
+        ret = all_failed ? -1 : 0;
+    } else {
+        ret = 0;
+    }
 
-        if (type == GF_QUOTA_OPTION_TYPE_LIST)
-                gfid_type = GF_QUOTA_CONF_TYPE_USAGE;
-        else
-                gfid_type = GF_QUOTA_CONF_TYPE_OBJECTS;
-
-        /* Try to read atleast one gfid  of type 'gfid_type' */
-        while (1) {
-                ret = quota_conf_read_gfid (fd, buf, &gfid_type_stored,
-                                            version);
-                if (ret <= 0)
-                        break;
-
-                if (gfid_type_stored == gfid_type) {
-                        limits_set = _gf_true;
-                        break;
-                }
-        }
 out:
-        if (fd != -1)
-                sys_close (fd);
-
-        return limits_set;
+    if (xml_err_flag) {
+        ret = cli_xml_output_str("volQuota", NULL, -1, 0, err_str);
+        if (ret) {
+            gf_log("cli", GF_LOG_ERROR,
+                   "Error outputting in "
+                   "xml format");
+        }
+    }
+    if (xdata)
+        dict_unref(xdata);
+
+    if (fd != -1) {
+        sys_close(fd);
+    }
+
+    if (ret) {
+        gf_log("cli", GF_LOG_ERROR,
+               "Could not fetch and display quota"
+               " limits");
+    }
+    CLI_STACK_DESTROY(frame);
+    return ret;
 }
 
-/* Checks if the mount is connected to the bricks
- *
- * Returns true if connected and false if not
- */
-gf_boolean_t
-_quota_aux_mount_online (char *volname)
+int
+cli_cmd_bitrot_cbk(struct cli_state *state, struct cli_cmd_word *word,
+                   const char **words, int wordcount)
 {
-        int         ret = 0;
-        char        mount_path[PATH_MAX + 1] = {0,};
-        struct stat buf = {0,};
+    int ret = -1;
+    int parse_err = 0;
+    call_frame_t *frame = NULL;
+    dict_t *options = NULL;
+    cli_local_t *local = NULL;
+    rpc_clnt_procedure_t *proc = NULL;
+    int sent = 0;
+#if (USE_EVENTS)
+    int cmd_type = -1;
+    int ret1 = -1;
+    int event_type = -1;
+    char *tmp = NULL;
+    char *events_str = NULL;
+    char *volname = NULL;
+#endif
 
-        GF_ASSERT (volname);
+    ret = cli_cmd_bitrot_parse(words, wordcount, &options);
+    if (ret < 0) {
+        cli_usage_out(word->pattern);
+        parse_err = 1;
+        goto out;
+    }
 
-        /* Try to create the aux mount before checking if bricks are online */
-        ret = gf_cli_create_auxiliary_mount (volname);
-        if (ret) {
-                cli_err ("quota: Could not start quota auxiliary mount");
-                return _gf_false;
-        }
+    if (ret == 1) {
+        /* this is 'volume bitrot help' */
+        cli_cmd_bitrot_help_cbk(state, word, words, wordcount);
+        ret = 0;
+        goto out2;
+    }
 
-        GLUSTERD_GET_QUOTA_AUX_MOUNT_PATH (mount_path, volname, "/");
+    frame = create_frame(THIS, THIS->ctx->pool);
+    if (!frame) {
+        ret = -1;
+        goto out;
+    }
 
-        ret = sys_stat (mount_path, &buf);
-        if (ret) {
-                if (ENOTCONN == errno) {
-                        cli_err ("quota: Cannot connect to bricks. Check if "
-                                 "bricks are online.");
-                } else {
-                        cli_err ("quota: Error on quota auxiliary mount (%s).",
-                                 strerror (errno));
-                }
-                return _gf_false;
-        }
-        return _gf_true;
-}
+    proc = &cli_rpc_prog->proctable[GLUSTER_CLI_BITROT];
 
-int
-cli_cmd_quota_handle_list_all (const char **words, dict_t *options)
-{
-        int                      all_failed = 1;
-        int                      count      = 0;
-        int                      ret        = -1;
-        rpc_clnt_procedure_t    *proc       = NULL;
-        cli_local_t             *local      = NULL;
-        call_frame_t            *frame      = NULL;
-        dict_t                  *xdata      = NULL;
-        char                    *gfid_str   = NULL;
-        char                    *volname    = NULL;
-        char                    *volname_dup = NULL;
-        unsigned char            buf[16]   = {0};
-        int                      fd        = -1;
-        char                     quota_conf_file[PATH_MAX] = {0};
-        gf_boolean_t             xml_err_flag   = _gf_false;
-        char                     err_str[NAME_MAX] = {0,};
-        int32_t                  type       = 0;
-        char                     gfid_type  = 0;
-        float                    version    = 0.0f;
-
-        xdata = dict_new ();
-        if (!xdata) {
-                ret = -1;
-                goto out;
-        }
+    CLI_LOCAL_INIT(local, words, frame, options);
 
-        ret = dict_get_str (options, "volname", &volname);
-        if (ret) {
-                gf_log ("cli", GF_LOG_ERROR, "Failed to get volume name");
-                goto out;
-        }
+    if (proc->fn) {
+        ret = proc->fn(frame, THIS, options);
+    }
 
-        ret = dict_get_int32 (options, "type", &type);
-        if (ret) {
-                gf_log ("cli", GF_LOG_ERROR, "Failed to get quota option type");
-                goto out;
+out:
+    if (ret) {
+        cli_cmd_sent_status_get(&sent);
+        if ((sent == 0) && (parse_err == 0))
+            cli_err(
+                "Bit rot command failed. Please check the cli "
+                "logs for more details");
+    }
+
+#if (USE_EVENTS)
+    if (ret == 0) {
+        ret1 = dict_get_int32(options, "type", &cmd_type);
+        if (ret1)
+            cmd_type = -1;
+        else {
+            ret1 = dict_get_str(options, "volname", &volname);
+            if (ret1)
+                volname = "";
         }
 
-        ret = dict_set_int32 (xdata, "type", type);
-        if (ret) {
-                gf_log ("cli", GF_LOG_ERROR, "Failed to set type in xdata");
-                goto out;
+        switch (cmd_type) {
+            case GF_BITROT_OPTION_TYPE_ENABLE:
+                event_type = EVENT_BITROT_ENABLE;
+                break;
+            case GF_BITROT_OPTION_TYPE_DISABLE:
+                event_type = EVENT_BITROT_DISABLE;
+                break;
+            case GF_BITROT_CMD_SCRUB_ONDEMAND:
+                event_type = EVENT_BITROT_SCRUB_ONDEMAND;
+                break;
+            case GF_BITROT_OPTION_TYPE_SCRUB_THROTTLE:
+                event_type = EVENT_BITROT_SCRUB_THROTTLE;
+                ret1 = dict_get_str(options, "scrub-throttle-value", &tmp);
+                if (ret1)
+                    tmp = "";
+                gf_asprintf(&events_str, "name=%s;value=%s", volname, tmp);
+                break;
+            case GF_BITROT_OPTION_TYPE_SCRUB_FREQ:
+                event_type = EVENT_BITROT_SCRUB_FREQ;
+                ret1 = dict_get_str(options, "scrub-frequency-value", &tmp);
+                if (ret1)
+                    tmp = "";
+                gf_asprintf(&events_str, "name=%s;value=%s", volname, tmp);
+                break;
+            case GF_BITROT_OPTION_TYPE_SCRUB:
+                event_type = EVENT_BITROT_SCRUB_OPTION;
+                ret1 = dict_get_str(options, "scrub-value", &tmp);
+                if (ret1)
+                    tmp = "";
+                gf_asprintf(&events_str, "name=%s;value=%s", volname, tmp);
+                break;
+            default:
+                break;
         }
 
-        ret = cli_get_soft_limit (options, words, xdata);
-        if (ret) {
-                gf_log ("cli", GF_LOG_ERROR, "Failed to fetch default "
-                        "soft-limit");
-                goto out;
-        }
+        if (event_type > -1)
+            gf_event(event_type, "%s", events_str);
 
-        /* Check if at least one limit is set on volume. No need to check for
-         * quota enabled as cli_get_soft_limit() handles that
-         */
-        if (!_limits_set_on_volume (volname, type)) {
-                snprintf (err_str, sizeof (err_str), "No%s quota configured on"
-                          " volume %s",
-                          (type == GF_QUOTA_OPTION_TYPE_LIST) ? "" : " inode",
-                          volname);
-                if (global_state->mode & GLUSTER_MODE_XML) {
-                        xml_err_flag = _gf_true;
-                } else {
-                        cli_out ("quota: %s", err_str);
-                }
-                ret = 0;
-                goto out;
-        }
+        if (events_str)
+            GF_FREE(events_str);
+    }
+#endif
 
-        /* Check if the mount is online before doing any listing */
-        if (!_quota_aux_mount_online (volname)) {
-                ret = -1;
-                goto out;
-        }
+    CLI_STACK_DESTROY(frame);
+out2:
+    return ret;
+}
 
-        frame = create_frame (THIS, THIS->ctx->pool);
-        if (!frame) {
-                ret = -1;
-                goto out;
+int
+cli_cmd_quota_cbk(struct cli_state *state, struct cli_cmd_word *word,
+                  const char **words, int wordcount)
+{
+    int ret = 0;
+    int parse_err = 0;
+    int32_t type = 0;
+    rpc_clnt_procedure_t *proc = NULL;
+    call_frame_t *frame = NULL;
+    dict_t *options = NULL;
+    gf_answer_t answer = GF_ANSWER_NO;
+    cli_local_t *local = NULL;
+    int sent = 0;
+    char *volname = NULL;
+    const char *question =
+        "Disabling quota will delete all the quota "
+        "configuration. Do you want to continue?";
+
+    // parse **words into options dictionary
+    if (strcmp(words[1], "inode-quota") == 0) {
+        ret = cli_cmd_inode_quota_parse(words, wordcount, &options);
+        if (ret < 0) {
+            cli_usage_out(word->pattern);
+            parse_err = 1;
+            goto out;
         }
+    } else {
+        ret = cli_cmd_quota_parse(words, wordcount, &options);
 
-        volname_dup = gf_strdup (volname);
-        if (!volname_dup) {
-                ret = -1;
-                goto out;
+        if (ret == 1) {
+            cli_cmd_quota_help_cbk(state, word, words, wordcount);
+            ret = 0;
+            goto out;
         }
-
-        ret = dict_set_dynstr (xdata, "volume-uuid", volname_dup);
-        if (ret) {
-                gf_log ("cli", GF_LOG_ERROR, "Failed to set volume-uuid");
-                GF_FREE (volname_dup);
-                goto out;
+        if (ret < 0) {
+            cli_usage_out(word->pattern);
+            parse_err = 1;
+            goto out;
         }
+    }
 
-        //TODO: fix hardcoding; Need to perform an RPC call to glusterd
-        //to fetch working directory
-        snprintf (quota_conf_file, sizeof quota_conf_file,
-                  "%s/vols/%s/quota.conf",
-                  GLUSTERD_DEFAULT_WORKDIR,
-                  volname);
-        fd = open (quota_conf_file, O_RDONLY);
-        if (fd == -1) {
-                //This may because no limits were yet set on the volume
-                gf_log ("cli", GF_LOG_TRACE, "Unable to open "
-                        "quota.conf");
-                ret = 0;
-                goto out;
-         }
-
-        ret = quota_conf_read_version (fd, &version);
-        if (ret)
-                goto out;
-
-        CLI_LOCAL_INIT (local, words, frame, xdata);
-        proc = &cli_quotad_clnt.proctable[GF_AGGREGATOR_GETLIMIT];
+    ret = dict_get_int32(options, "type", &type);
+    if (ret) {
+        gf_log("cli", GF_LOG_ERROR, "Failed to get opcode");
+        goto out;
+    }
 
-        gfid_str = GF_CALLOC (1, gf_common_mt_char, 64);
-        if (!gfid_str) {
-                ret = -1;
+    // handle quota-disable and quota-list-all different from others
+    switch (type) {
+        case GF_QUOTA_OPTION_TYPE_DISABLE:
+            answer = cli_cmd_get_confirmation(state, question);
+            if (answer == GF_ANSWER_NO)
                 goto out;
-        }
-        for (count = 0;; count++) {
-                ret = quota_conf_read_gfid (fd, buf, &gfid_type, version);
-                if (ret == 0) {
-                        break;
-                } else if (ret < 0) {
-                        gf_log (THIS->name, GF_LOG_CRITICAL, "Quota "
-                                "configuration store may be corrupt.");
-                        goto out;
-                }
-
-                if ((type == GF_QUOTA_OPTION_TYPE_LIST &&
-                     gfid_type == GF_QUOTA_CONF_TYPE_OBJECTS) ||
-                    (type == GF_QUOTA_OPTION_TYPE_LIST_OBJECTS &&
-                     gfid_type == GF_QUOTA_CONF_TYPE_USAGE))
-                        continue;
-
-                uuid_utoa_r (buf, gfid_str);
-                ret = dict_set_str (xdata, "gfid", gfid_str);
-                if (ret) {
-                        gf_log ("cli", GF_LOG_ERROR, "Failed to set gfid");
-                        goto out;
-                }
-
-                ret = proc->fn (frame, THIS, xdata);
-                if (ret) {
-                        gf_log ("cli", GF_LOG_ERROR, "Failed to get quota "
-                                "limits for %s", uuid_utoa ((unsigned char*)buf));
-                }
+            break;
+        case GF_QUOTA_OPTION_TYPE_LIST:
+        case GF_QUOTA_OPTION_TYPE_LIST_OBJECTS:
+            if (wordcount != 4)
+                break;
+            ret = cli_cmd_quota_handle_list_all(words, options);
+            goto out;
+        default:
+            break;
+    }
 
-                dict_del (xdata, "gfid");
-                all_failed = all_failed && ret;
-        }
+    ret = dict_get_str(options, "volname", &volname);
+    if (ret) {
+        gf_log("cli", GF_LOG_ERROR, "Failed to get volume name");
+        goto out;
+    }
 
-        if (global_state->mode & GLUSTER_MODE_XML) {
-                ret = cli_xml_output_vol_quota_limit_list_end (local);
-                if (ret) {
-                        gf_log ("cli", GF_LOG_ERROR, "Error in printing "
-                                "xml output");
-                        goto out;
-                }
-        }
+    frame = create_frame(THIS, THIS->ctx->pool);
+    if (!frame) {
+        ret = -1;
+        goto out;
+    }
 
-        if (count > 0) {
-                ret = all_failed? -1: 0;
-        } else {
-                ret = 0;
-        }
+    CLI_LOCAL_INIT(local, words, frame, options);
+    proc = &cli_rpc_prog->proctable[GLUSTER_CLI_QUOTA];
 
+    if (proc->fn)
+        ret = proc->fn(frame, THIS, options);
 
 out:
-        if (xml_err_flag) {
-                ret = cli_xml_output_str ("volQuota", NULL, -1, 0, err_str);
-                if (ret) {
-                        gf_log ("cli", GF_LOG_ERROR, "Error outputting in "
-                                "xml format");
-                }
-        }
-
-        if (fd != -1) {
-                sys_close (fd);
+    if (ret) {
+        cli_cmd_sent_status_get(&sent);
+        if (sent == 0 && parse_err == 0)
+            cli_out(
+                "Quota command failed. Please check the cli "
+                "logs for more details");
+    }
+    if (options)
+        dict_unref(options);
+
+    /* Events for Quota */
+    if (ret == 0) {
+        switch (type) {
+            case GF_QUOTA_OPTION_TYPE_ENABLE:
+                gf_event(EVENT_QUOTA_ENABLE, "volume=%s", volname);
+                break;
+            case GF_QUOTA_OPTION_TYPE_DISABLE:
+                gf_event(EVENT_QUOTA_DISABLE, "volume=%s", volname);
+                break;
+            case GF_QUOTA_OPTION_TYPE_LIMIT_USAGE:
+                gf_event(EVENT_QUOTA_SET_USAGE_LIMIT,
+                         "volume=%s;"
+                         "path=%s;limit=%s",
+                         volname, words[4], words[5]);
+                break;
+            case GF_QUOTA_OPTION_TYPE_LIMIT_OBJECTS:
+                gf_event(EVENT_QUOTA_SET_OBJECTS_LIMIT,
+                         "volume=%s;"
+                         "path=%s;limit=%s",
+                         volname, words[4], words[5]);
+                break;
+            case GF_QUOTA_OPTION_TYPE_REMOVE:
+                gf_event(EVENT_QUOTA_REMOVE_USAGE_LIMIT,
+                         "volume=%s;"
+                         "path=%s",
+                         volname, words[4]);
+                break;
+            case GF_QUOTA_OPTION_TYPE_REMOVE_OBJECTS:
+                gf_event(EVENT_QUOTA_REMOVE_OBJECTS_LIMIT,
+                         "volume=%s;"
+                         "path=%s",
+                         volname, words[4]);
+                break;
+            case GF_QUOTA_OPTION_TYPE_ALERT_TIME:
+                gf_event(EVENT_QUOTA_ALERT_TIME, "volume=%s;time=%s", volname,
+                         words[4]);
+                break;
+            case GF_QUOTA_OPTION_TYPE_SOFT_TIMEOUT:
+                gf_event(EVENT_QUOTA_SOFT_TIMEOUT,
+                         "volume=%s;"
+                         "soft-timeout=%s",
+                         volname, words[4]);
+                break;
+            case GF_QUOTA_OPTION_TYPE_HARD_TIMEOUT:
+                gf_event(EVENT_QUOTA_HARD_TIMEOUT,
+                         "volume=%s;"
+                         "hard-timeout=%s",
+                         volname, words[4]);
+                break;
+            case GF_QUOTA_OPTION_TYPE_DEFAULT_SOFT_LIMIT:
+                gf_event(EVENT_QUOTA_DEFAULT_SOFT_LIMIT,
+                         "volume=%s;"
+                         "default-soft-limit=%s",
+                         volname, words[4]);
+                break;
         }
+    }
 
-        GF_FREE (gfid_str);
-        if (ret) {
-                gf_log ("cli", GF_LOG_ERROR, "Could not fetch and display quota"
-                        " limits");
-        }
-        CLI_STACK_DESTROY (frame);
-        return ret;
+    CLI_STACK_DESTROY(frame);
+    return ret;
 }
 
 int
-cli_cmd_bitrot_cbk (struct cli_state *state, struct cli_cmd_word *word,
-                    const char **words, int wordcount)
+cli_cmd_volume_remove_brick_cbk(struct cli_state *state,
+                                struct cli_cmd_word *word, const char **words,
+                                int wordcount)
 {
+    int ret = -1;
+    rpc_clnt_procedure_t *proc = NULL;
+    call_frame_t *frame = NULL;
+    dict_t *options = NULL;
+    gf_answer_t answer = GF_ANSWER_NO;
+    int brick_count = 0;
+    int sent = 0;
+    int parse_error = 0;
+    int need_question = 0;
+    cli_local_t *local = NULL;
+    char *volname = NULL;
+#if (USE_EVENTS)
+    eventtypes_t event = EVENT_LAST;
+    char *event_str = NULL;
+    int event_ret = -1;
+#endif
+    int32_t command = GF_OP_CMD_NONE;
+    char *question = NULL;
+
+    ret = cli_cmd_volume_remove_brick_parse(state, words, wordcount, &options,
+                                            &need_question, &brick_count,
+                                            &command);
+    if (ret) {
+        cli_usage_out(word->pattern);
+        parse_error = 1;
+        goto out;
+    }
+
+    if (command == GF_OP_CMD_COMMIT_FORCE) {
+        question =
+            "Remove-brick force will not migrate files from the "
+            "removed bricks, so they will no longer be available"
+            " on the volume.\nDo you want to continue?";
+    } else if (command == GF_OP_CMD_START) {
+        question =
+            "It is recommended that remove-brick be run with"
+            " cluster.force-migration option disabled to prevent"
+            " possible data corruption. Doing so will ensure that"
+            " files that receive writes during migration will not"
+            " be migrated and will need to be manually copied"
+            " after the remove-brick commit operation. Please"
+            " check the value of the option and update accordingly."
+            " \nDo you want to continue with your current"
+            " cluster.force-migration settings?";
+    }
+
+    if (!brick_count) {
+        cli_err("No bricks specified");
+        cli_usage_out(word->pattern);
+        parse_error = 1;
+        ret = -1;
+        goto out;
+    }
+    ret = dict_get_str(options, "volname", &volname);
+    if (ret || !volname) {
+        gf_log("cli", GF_LOG_ERROR, "Failed to fetch volname");
+        ret = -1;
+        goto out;
+    }
 
-        int                     ret        = -1;
-        int                     parse_err  = 0;
-        call_frame_t            *frame     = NULL;
-        dict_t                  *options   = NULL;
-        cli_local_t             *local     = NULL;
-        rpc_clnt_procedure_t    *proc      = NULL;
-        int                     sent       = 0;
+#if (USE_EVENTS)
+    event_ret = cli_event_remove_brick_str(options, &event_str, &event);
+#endif
 
-        ret = cli_cmd_bitrot_parse (words, wordcount, &options);
-        if (ret < 0) {
-                cli_usage_out (word->pattern);
-                parse_err = 1;
-                goto out;
+    if (!strcmp(volname, GLUSTER_SHARED_STORAGE)) {
+        question =
+            "Removing brick from the shared storage volume"
+            "(gluster_shared_storage), will affect features "
+            "like snapshot scheduler, geo-replication "
+            "and NFS-Ganesha. Do you still want to "
+            "continue?";
+        need_question = _gf_true;
+    }
+
+    if (!(state->mode & GLUSTER_MODE_SCRIPT) && need_question) {
+        answer = cli_cmd_get_confirmation(state, question);
+        if (GF_ANSWER_NO == answer) {
+            ret = 0;
+            goto out;
         }
+    }
 
-        frame = create_frame (THIS, THIS->ctx->pool);
-        if (!frame) {
-                ret = -1;
-                goto out;
-        }
+    proc = &cli_rpc_prog->proctable[GLUSTER_CLI_REMOVE_BRICK];
 
-        proc = &cli_rpc_prog->proctable[GLUSTER_CLI_BITROT];
-        if (proc == NULL) {
-                ret = -1;
-                goto out;
-        }
+    frame = create_frame(THIS, THIS->ctx->pool);
+    if (!frame) {
+        ret = -1;
+        goto out;
+    }
 
-        CLI_LOCAL_INIT (local, words, frame, options);
+    CLI_LOCAL_INIT(local, words, frame, options);
 
-        if (proc->fn) {
-                ret = proc->fn (frame, THIS, options);
-        }
+    if (proc->fn) {
+        ret = proc->fn(frame, THIS, options);
+    }
 
 out:
-        if (ret) {
-                cli_cmd_sent_status_get (&sent);
-                if ((sent == 0) && (parse_err == 0))
-                    cli_err ("Bit rot command failed. Please check the cli "
-                             "logs for more details");
+    if (ret) {
+        cli_cmd_sent_status_get(&sent);
+        if ((sent == 0) && (parse_error == 0))
+            cli_out("Volume remove-brick failed");
+    }
+#if (USE_EVENTS)
+    if (!ret && !event_ret)
+        gf_event(event, "%s", event_str);
+    if (event_str)
+        GF_FREE(event_str);
 
-        }
+#endif
 
-        CLI_STACK_DESTROY (frame);
+    CLI_STACK_DESTROY(frame);
+    if (options)
+        dict_unref(options);
 
-        return ret;
+    return ret;
 }
 
 int
-cli_cmd_quota_cbk (struct cli_state *state, struct cli_cmd_word *word,
-                   const char **words, int wordcount)
+cli_cmd_volume_reset_brick_cbk(struct cli_state *state,
+                               struct cli_cmd_word *word, const char **words,
+                               int wordcount)
 {
+    int ret = -1;
+    rpc_clnt_procedure_t *proc = NULL;
+    call_frame_t *frame = NULL;
+    dict_t *options = NULL;
+    int sent = 0;
+    int parse_error = 0;
+    cli_local_t *local = NULL;
 
-        int                      ret       = 0;
-        int                      parse_err = 0;
-        int32_t                  type      = 0;
-        rpc_clnt_procedure_t    *proc      = NULL;
-        call_frame_t            *frame     = NULL;
-        dict_t                  *options   = NULL;
-        gf_answer_t              answer    = GF_ANSWER_NO;
-        cli_local_t             *local     = NULL;
-        int                      sent      = 0;
-        char                    *volname   = NULL;
-        const char *question = "Disabling quota will delete all the quota "
-                               "configuration. Do you want to continue?";
-
-        //parse **words into options dictionary
-        if (strcmp (words[1], "inode-quota") == 0) {
-                ret = cli_cmd_inode_quota_parse (words, wordcount, &options);
-                if (ret < 0) {
-                        cli_usage_out (word->pattern);
-                        parse_err = 1;
-                        goto out;
-                }
-        } else {
-                ret = cli_cmd_quota_parse (words, wordcount, &options);
-                if (ret < 0) {
-                        cli_usage_out (word->pattern);
-                        parse_err = 1;
-                        goto out;
-                }
-        }
+#ifdef GF_SOLARIS_HOST_OS
+    cli_out("Command not supported on Solaris");
+    goto out;
+#endif
+    proc = &cli_rpc_prog->proctable[GLUSTER_CLI_RESET_BRICK];
 
-        ret = dict_get_int32 (options, "type", &type);
-        if (ret) {
-                gf_log ("cli", GF_LOG_ERROR, "Failed to get opcode");
-                goto out;
-        }
+    ret = cli_cmd_volume_reset_brick_parse(words, wordcount, &options);
 
-        //handle quota-disable and quota-list-all different from others
-        switch (type) {
-        case GF_QUOTA_OPTION_TYPE_DISABLE:
-                answer = cli_cmd_get_confirmation (state, question);
-                if (answer == GF_ANSWER_NO)
-                        goto out;
-                break;
-        case GF_QUOTA_OPTION_TYPE_LIST:
-        case GF_QUOTA_OPTION_TYPE_LIST_OBJECTS:
-                if (wordcount != 4)
-                        break;
-                ret = cli_cmd_quota_handle_list_all (words, options);
-                goto out;
-        default:
-                break;
-        }
+    if (ret) {
+        cli_usage_out(word->pattern);
+        parse_error = 1;
+        goto out;
+    }
 
-        ret = dict_get_str (options, "volname", &volname);
+    if (state->mode & GLUSTER_MODE_WIGNORE_PARTITION) {
+        ret = dict_set_int32(options, "ignore-partition", _gf_true);
         if (ret) {
-                gf_log ("cli", GF_LOG_ERROR, "Failed to get volume name");
-                goto out;
+            gf_log("cli", GF_LOG_ERROR,
+                   "Failed to set ignore-"
+                   "partition option");
+            goto out;
         }
+    }
 
-        //create auxiliary mount need for quota commands that operate on path
-        ret = cli_stage_quota_op (volname, type);
-        if (ret)
-                goto out;
-
-        frame = create_frame (THIS, THIS->ctx->pool);
-        if (!frame) {
-                ret = -1;
-                goto out;
-        }
+    frame = create_frame(THIS, THIS->ctx->pool);
+    if (!frame) {
+        ret = -1;
+        goto out;
+    }
 
-        CLI_LOCAL_INIT (local, words, frame, options);
-        proc = &cli_rpc_prog->proctable[GLUSTER_CLI_QUOTA];
+    CLI_LOCAL_INIT(local, words, frame, options);
 
-        if (proc->fn)
-                ret = proc->fn (frame, THIS, options);
+    if (proc->fn) {
+        ret = proc->fn(frame, THIS, options);
+    }
 
 out:
-        if (ret) {
-                cli_cmd_sent_status_get (&sent);
-                if (sent == 0 && parse_err == 0)
-                        cli_out ("Quota command failed. Please check the cli "
-                                 "logs for more details");
+    if (ret) {
+        cli_cmd_sent_status_get(&sent);
+        if ((sent == 0) && (parse_error == 0))
+            cli_out("Volume reset-brick failed");
+    } else {
+        if (wordcount > 5) {
+            gf_event(EVENT_BRICK_RESET_COMMIT,
+                     "Volume=%s;source-brick=%s;"
+                     "destination-brick=%s",
+                     (char *)words[2], (char *)words[3], (char *)words[4]);
+        } else {
+            gf_event(EVENT_BRICK_RESET_START, "Volume=%s;source-brick=%s",
+                     (char *)words[2], (char *)words[3]);
         }
+    }
+    CLI_STACK_DESTROY(frame);
 
-        CLI_STACK_DESTROY (frame);
-        return ret;
+    return ret;
 }
 
 int
-cli_cmd_volume_remove_brick_cbk (struct cli_state *state,
+cli_cmd_volume_replace_brick_cbk(struct cli_state *state,
                                  struct cli_cmd_word *word, const char **words,
                                  int wordcount)
 {
-        int                     ret = -1;
-        rpc_clnt_procedure_t    *proc = NULL;
-        call_frame_t            *frame = NULL;
-        dict_t                  *options = NULL;
-        gf_answer_t             answer = GF_ANSWER_NO;
-        int                     sent = 0;
-        int                     parse_error = 0;
-        int                     need_question = 0;
-        cli_local_t             *local = NULL;
-        char                    *volname = NULL;
-
-        const char *question = "Removing brick(s) can result in data loss. "
-                               "Do you want to Continue?";
-
-        frame = create_frame (THIS, THIS->ctx->pool);
-        if (!frame)
-                goto out;
-
-        ret = cli_cmd_volume_remove_brick_parse (words, wordcount, &options,
-                                                 &need_question);
-        if (ret) {
-                cli_usage_out (word->pattern);
-                parse_error = 1;
-                goto out;
-        }
-
-        ret = dict_get_str (options, "volname", &volname);
-        if (ret || !volname) {
-                gf_log ("cli", GF_LOG_ERROR, "Failed to fetch volname");
-                ret = -1;
-                goto out;
-        }
-
-        if (!strcmp (volname, GLUSTER_SHARED_STORAGE)) {
-                question = "Removing brick from the shared storage volume"
-                           "(gluster_shared_storage), will affect features "
-                           "like snapshot scheduler, geo-replication "
-                           "and NFS-Ganesha. Do you still want to "
-                           "continue?";
-                need_question = _gf_true;
-        }
-
-        if (!(state->mode & GLUSTER_MODE_SCRIPT) && need_question) {
-                /* we need to ask question only in case of 'commit or force' */
-                answer = cli_cmd_get_confirmation (state, question);
-                if (GF_ANSWER_NO == answer) {
-                        ret = 0;
-                        goto out;
-                }
-        }
-
-        proc = &cli_rpc_prog->proctable[GLUSTER_CLI_REMOVE_BRICK];
-
-        CLI_LOCAL_INIT (local, words, frame, options);
-
-        if (proc->fn) {
-                ret = proc->fn (frame, THIS, options);
-        }
-
-out:
-        if (ret) {
-                cli_cmd_sent_status_get (&sent);
-                if ((sent == 0) && (parse_error == 0))
-                        cli_out ("Volume remove-brick failed");
-        }
-
-        CLI_STACK_DESTROY (frame);
-
-        return ret;
-
-}
-
-int
-cli_cmd_volume_replace_brick_cbk (struct cli_state *state,
-                                  struct cli_cmd_word *word,
-                                  const char **words,
-                                  int wordcount)
-{
-        int                      ret          = -1;
-        rpc_clnt_procedure_t    *proc         = NULL;
-        call_frame_t            *frame        = NULL;
-        dict_t                  *options = NULL;
-        int                      sent = 0;
-        int                      parse_error = 0;
-        cli_local_t             *local = NULL;
+    int ret = -1;
+    rpc_clnt_procedure_t *proc = NULL;
+    call_frame_t *frame = NULL;
+    dict_t *options = NULL;
+    int sent = 0;
+    int parse_error = 0;
+    cli_local_t *local = NULL;
 
 #ifdef GF_SOLARIS_HOST_OS
-        cli_out ("Command not supported on Solaris");
-        goto out;
+    cli_out("Command not supported on Solaris");
+    goto out;
 #endif
-        proc = &cli_rpc_prog->proctable[GLUSTER_CLI_REPLACE_BRICK];
+    proc = &cli_rpc_prog->proctable[GLUSTER_CLI_REPLACE_BRICK];
 
-        frame = create_frame (THIS, THIS->ctx->pool);
-        if (!frame)
-                goto out;
+    ret = cli_cmd_volume_replace_brick_parse(words, wordcount, &options);
 
-        ret = cli_cmd_volume_replace_brick_parse (words, wordcount, &options);
+    if (ret) {
+        cli_usage_out(word->pattern);
+        parse_error = 1;
+        goto out;
+    }
 
-        if (ret) {
-                cli_usage_out (word->pattern);
-                parse_error = 1;
-                goto out;
-        }
+    frame = create_frame(THIS, THIS->ctx->pool);
+    if (!frame) {
+        ret = -1;
+        goto out;
+    }
 
-        CLI_LOCAL_INIT (local, words, frame, options);
+    CLI_LOCAL_INIT(local, words, frame, options);
 
-        if (proc->fn) {
-                ret = proc->fn (frame, THIS, options);
-        }
+    if (proc->fn) {
+        ret = proc->fn(frame, THIS, options);
+    }
 
 out:
-        if (ret) {
-                cli_cmd_sent_status_get (&sent);
-                if ((sent == 0) && (parse_error == 0))
-                        cli_out ("Volume replace-brick failed");
-        }
-
-        CLI_STACK_DESTROY (frame);
-
-        return ret;
+    if (ret) {
+        cli_cmd_sent_status_get(&sent);
+        if ((sent == 0) && (parse_error == 0))
+            cli_out("Volume replace-brick failed");
+    } else {
+        gf_event(EVENT_BRICK_REPLACE,
+                 "Volume=%s;source-brick=%s;destination-brick=%s",
+                 (char *)words[2], (char *)words[3], (char *)words[4]);
+    }
+    CLI_STACK_DESTROY(frame);
+
+    return ret;
 }
 
-
 int
-cli_cmd_volume_set_transport_cbk (struct cli_state *state,
-                                  struct cli_cmd_word *word,
-                                  const char **words, int wordcount)
+cli_cmd_volume_set_transport_cbk(struct cli_state *state,
+                                 struct cli_cmd_word *word, const char **words,
+                                 int wordcount)
 {
-        cli_cmd_broadcast_response (0);
-        return 0;
+    cli_cmd_broadcast_response(0);
+    return 0;
 }
 
 int
-cli_cmd_volume_top_cbk (struct cli_state *state, struct cli_cmd_word *word,
-                          const char **words, int wordcount)
+cli_cmd_volume_top_cbk(struct cli_state *state, struct cli_cmd_word *word,
+                       const char **words, int wordcount)
 {
+    int ret = -1;
+    rpc_clnt_procedure_t *proc = NULL;
+    call_frame_t *frame = NULL;
+    dict_t *options = NULL;
+    int sent = 0;
+    int parse_error = 0;
+    cli_local_t *local = NULL;
+
+    ret = cli_cmd_volume_top_parse(words, wordcount, &options);
+
+    if (ret) {
+        parse_error = 1;
+        cli_usage_out(word->pattern);
+        goto out;
+    }
 
-        int                     ret      = -1;
-        rpc_clnt_procedure_t    *proc    = NULL;
-        call_frame_t            *frame   = NULL;
-        dict_t                  *options = NULL;
-        int                     sent     = 0;
-        int                     parse_error = 0;
-        cli_local_t             *local = NULL;
-
-        ret = cli_cmd_volume_top_parse (words, wordcount, &options);
-
-        if (ret) {
-                parse_error = 1;
-                cli_usage_out (word->pattern);
-                goto out;
-        }
-
-        proc = &cli_rpc_prog->proctable[GLUSTER_CLI_TOP_VOLUME];
+    proc = &cli_rpc_prog->proctable[GLUSTER_CLI_TOP_VOLUME];
 
-        frame = create_frame (THIS, THIS->ctx->pool);
-        if (!frame)
-                goto out;
+    frame = create_frame(THIS, THIS->ctx->pool);
+    if (!frame) {
+        gf_log(THIS->name, GF_LOG_ERROR, "failed to create frame");
+        ret = -1;
+        goto out;
+    }
 
-        CLI_LOCAL_INIT (local, words, frame, options);
+    CLI_LOCAL_INIT(local, words, frame, options);
 
-        if (proc->fn) {
-                ret = proc->fn (frame, THIS, options);
-        }
+    if (proc->fn) {
+        ret = proc->fn(frame, THIS, options);
+    }
 
 out:
-        if (ret) {
-                cli_cmd_sent_status_get (&sent);
-                if ((sent == 0) && (parse_error == 0))
-                        cli_out ("Volume top failed");
-        }
-
-        CLI_STACK_DESTROY (frame);
+    if (ret) {
+        cli_cmd_sent_status_get(&sent);
+        if ((sent == 0) && (parse_error == 0))
+            cli_out("Volume top failed");
+    }
 
-        return ret;
+    CLI_STACK_DESTROY(frame);
 
+    return ret;
 }
 
-
 int
-cli_cmd_log_rotate_cbk (struct cli_state *state, struct cli_cmd_word *word,
-                        const char **words, int wordcount)
+cli_cmd_log_rotate_cbk(struct cli_state *state, struct cli_cmd_word *word,
+                       const char **words, int wordcount)
 {
-        int                     ret = -1;
-        rpc_clnt_procedure_t    *proc = NULL;
-        call_frame_t            *frame = NULL;
-        dict_t                  *options = NULL;
-        int                     sent = 0;
-        int                     parse_error = 0;
-        cli_local_t             *local = NULL;
-
-        if (!((wordcount == 4) || (wordcount == 5))) {
-                cli_usage_out (word->pattern);
-                parse_error = 1;
-                goto out;
-        }
+    int ret = -1;
+    rpc_clnt_procedure_t *proc = NULL;
+    call_frame_t *frame = NULL;
+    dict_t *options = NULL;
+    int sent = 0;
+    int parse_error = 0;
+    cli_local_t *local = NULL;
+
+    if (!((wordcount == 4) || (wordcount == 5))) {
+        cli_usage_out(word->pattern);
+        parse_error = 1;
+        goto out;
+    }
 
-        if (!((strcmp ("rotate", words[2]) == 0) ||
-              (strcmp ("rotate", words[3]) == 0))) {
-                cli_usage_out (word->pattern);
-                parse_error = 1;
-                goto out;
-        }
+    if (!(strcmp("rotate", words[3]) == 0)) {
+        cli_usage_out(word->pattern);
+        parse_error = 1;
+        goto out;
+    }
 
-        proc = &cli_rpc_prog->proctable[GLUSTER_CLI_LOG_ROTATE];
+    proc = &cli_rpc_prog->proctable[GLUSTER_CLI_LOG_ROTATE];
 
-        frame = create_frame (THIS, THIS->ctx->pool);
-        if (!frame)
-                goto out;
+    ret = cli_cmd_log_rotate_parse(words, wordcount, &options);
+    if (ret)
+        goto out;
 
-        ret = cli_cmd_log_rotate_parse (words, wordcount, &options);
-        if (ret)
-                goto out;
+    frame = create_frame(THIS, THIS->ctx->pool);
+    if (!frame) {
+        gf_log(THIS->name, GF_LOG_ERROR, "failed to create frame");
+        ret = -1;
+        goto out;
+    }
 
-        CLI_LOCAL_INIT (local, words, frame, options);
+    CLI_LOCAL_INIT(local, words, frame, options);
 
-        if (proc->fn) {
-                ret = proc->fn (frame, THIS, options);
-        }
+    if (proc->fn) {
+        ret = proc->fn(frame, THIS, options);
+    }
 
 out:
-        if (ret) {
-                cli_cmd_sent_status_get (&sent);
-                if ((sent == 0) && (parse_error == 0))
-                        cli_out ("Volume log rotate failed");
-        }
-        CLI_STACK_DESTROY (frame);
-
-        return ret;
+    if (ret) {
+        cli_cmd_sent_status_get(&sent);
+        if ((sent == 0) && (parse_error == 0))
+            cli_out("Volume log rotate failed");
+    }
+    CLI_STACK_DESTROY(frame);
+
+    return ret;
 }
 
 #if (SYNCDAEMON_COMPILE)
 static int
-cli_check_gsync_present ()
+cli_check_gsync_present()
 {
-        char                buff[PATH_MAX] = {0, };
-        runner_t            runner = {0,};
-        char                *ptr = NULL;
-        int                 ret = 0;
-
-        ret = setenv ("_GLUSTERD_CALLED_", "1", 1);
-        if (-1 == ret) {
-                gf_log ("", GF_LOG_WARNING, "setenv syscall failed, hence could"
-                        "not assert if geo-replication is installed");
-                goto out;
-        }
-
-        runinit (&runner);
-        runner_add_args (&runner, GSYNCD_PREFIX"/gsyncd", "--version", NULL);
-        runner_redir (&runner, STDOUT_FILENO, RUN_PIPE);
-        ret = runner_start (&runner);
-        if (ret == -1) {
-                gf_log ("", GF_LOG_INFO, "geo-replication not installed");
-                goto out;
-        }
+    char buff[PATH_MAX] = {
+        0,
+    };
+    runner_t runner = {
+        0,
+    };
+    char *ptr = NULL;
+    int ret = 0;
+
+    ret = setenv("_GLUSTERD_CALLED_", "1", 1);
+    if (-1 == ret) {
+        gf_log("", GF_LOG_WARNING,
+               "setenv syscall failed, hence could"
+               "not assert if geo-replication is installed");
+        goto out;
+    }
+
+    runinit(&runner);
+    runner_add_args(&runner, GSYNCD_PREFIX "/gsyncd", "--version", NULL);
+    runner_redir(&runner, STDOUT_FILENO, RUN_PIPE);
+    ret = runner_start(&runner);
+    if (ret == -1) {
+        gf_log("", GF_LOG_INFO, "geo-replication not installed");
+        goto out;
+    }
 
-        ptr = fgets(buff, sizeof(buff), runner_chio (&runner, STDOUT_FILENO));
-        if (ptr) {
-                if (!strstr (buff, "gsyncd")) {
-                        ret  = -1;
-                        goto out;
-                }
-        } else {
-                ret = -1;
-                goto out;
+    ptr = fgets(buff, sizeof(buff), runner_chio(&runner, STDOUT_FILENO));
+    if (ptr) {
+        if (!strstr(buff, "gsyncd")) {
+            ret = -1;
+            goto out;
         }
+    } else {
+        ret = -1;
+        goto out;
+    }
 
-        ret = runner_end (&runner);
+    ret = runner_end(&runner);
 
-        if (ret)
-                gf_log ("", GF_LOG_ERROR, "geo-replication not installed");
+    if (ret)
+        gf_log("", GF_LOG_ERROR, "geo-replication not installed");
 
 out:
-        gf_log ("cli", GF_LOG_DEBUG, "Returning %d", ret);
-        return ret ? -1 : 0;
-
+    gf_log("cli", GF_LOG_DEBUG, "Returning %d", ret);
+    return ret ? -1 : 0;
 }
 
 void
-cli_cmd_check_gsync_exists_cbk (struct cli_cmd *this)
+cli_cmd_check_gsync_exists_cbk(struct cli_cmd *this)
 {
+    int ret = 0;
 
-        int             ret = 0;
-
-        ret = cli_check_gsync_present ();
-        if (ret)
-                this->disable = _gf_true;
-
+    ret = cli_check_gsync_present();
+    if (ret)
+        this->disable = _gf_true;
 }
 #endif
 
 int
-cli_cmd_volume_gsync_set_cbk (struct cli_state *state, struct cli_cmd_word *word,
-                              const char **words, int wordcount)
+cli_cmd_volume_gsync_set_cbk(struct cli_state *state, struct cli_cmd_word *word,
+                             const char **words, int wordcount)
 {
-        int                      ret     = 0;
-        int                      parse_err = 0;
-        dict_t                  *options = NULL;
-        rpc_clnt_procedure_t    *proc    = NULL;
-        call_frame_t            *frame   = NULL;
-        cli_local_t             *local   = NULL;
-
-        proc = &cli_rpc_prog->proctable [GLUSTER_CLI_GSYNC_SET];
-
-        frame = create_frame (THIS, THIS->ctx->pool);
-        if (frame == NULL) {
-                ret = -1;
-                goto out;
-        }
+    int ret = 0;
+    int parse_err = 0;
+    dict_t *options = NULL;
+    rpc_clnt_procedure_t *proc = NULL;
+    call_frame_t *frame = NULL;
+    cli_local_t *local = NULL;
+    char *errstr = NULL;
+#if (USE_EVENTS)
+    int ret1 = -1;
+    int cmd_type = -1;
+    int tmpi = 0;
+    char *tmp = NULL;
+    char *events_str = NULL;
+    int event_type = -1;
+#endif
 
-        ret = cli_cmd_gsync_set_parse (words, wordcount, &options);
-        if (ret) {
-                cli_usage_out (word->pattern);
-                parse_err = 1;
-                goto out;
+    proc = &cli_rpc_prog->proctable[GLUSTER_CLI_GSYNC_SET];
+
+    ret = cli_cmd_gsync_set_parse(state, words, wordcount, &options, &errstr);
+    if (ret) {
+        if (errstr) {
+            cli_err("%s", errstr);
+            GF_FREE(errstr);
+        } else {
+            cli_usage_out(word->pattern);
         }
+        parse_err = 1;
+        goto out;
+    }
+
+    frame = create_frame(THIS, THIS->ctx->pool);
+    if (frame == NULL) {
+        ret = -1;
+        goto out;
+    }
 
-        CLI_LOCAL_INIT (local, words, frame, options);
+    CLI_LOCAL_INIT(local, words, frame, options);
 
-        if (proc->fn)
-                ret = proc->fn (frame, THIS, options);
+    if (proc->fn)
+        ret = proc->fn(frame, THIS, options);
 
 out:
-        if (ret && parse_err == 0)
-                cli_out (GEOREP" command failed");
+    if (ret && parse_err == 0)
+        cli_out(GEOREP " command failed");
+
+#if (USE_EVENTS)
+    if (ret == 0) {
+        events_str = gf_strdup("");
+
+        /* Type of Geo-rep Action - Create, Start etc */
+        ret1 = dict_get_int32(options, "type", &cmd_type);
+        if (ret1)
+            cmd_type = -1;
+
+        /* Only capture Events for modification commands */
+        switch (cmd_type) {
+            case GF_GSYNC_OPTION_TYPE_CREATE:
+                event_type = EVENT_GEOREP_CREATE;
+                break;
+            case GF_GSYNC_OPTION_TYPE_START:
+                event_type = EVENT_GEOREP_START;
+                break;
+            case GF_GSYNC_OPTION_TYPE_STOP:
+                event_type = EVENT_GEOREP_STOP;
+                break;
+            case GF_GSYNC_OPTION_TYPE_PAUSE:
+                event_type = EVENT_GEOREP_PAUSE;
+                break;
+            case GF_GSYNC_OPTION_TYPE_RESUME:
+                event_type = EVENT_GEOREP_RESUME;
+                break;
+            case GF_GSYNC_OPTION_TYPE_DELETE:
+                event_type = EVENT_GEOREP_DELETE;
+                break;
+            case GF_GSYNC_OPTION_TYPE_CONFIG:
+                ret1 = dict_get_str(options, "subop", &tmp);
+                if (ret1)
+                    tmp = "";
+
+                /* For Config Set additionally capture key and value */
+                /* For Config Reset capture key */
+                if (strcmp(tmp, "set") == 0) {
+                    event_type = EVENT_GEOREP_CONFIG_SET;
+
+                    ret1 = dict_get_str(options, "op_name", &tmp);
+                    if (ret1)
+                        tmp = "";
+
+                    gf_asprintf_append(&events_str, "%soption=%s;", events_str,
+                                       tmp);
+
+                    ret1 = dict_get_str(options, "op_value", &tmp);
+                    if (ret1)
+                        tmp = "";
+
+                    gf_asprintf_append(&events_str, "%svalue=%s;", events_str,
+                                       tmp);
+                } else if (strcmp(tmp, "del") == 0) {
+                    event_type = EVENT_GEOREP_CONFIG_RESET;
+
+                    ret1 = dict_get_str(options, "op_name", &tmp);
+                    if (ret1)
+                        tmp = "";
+
+                    gf_asprintf_append(&events_str, "%soption=%s;", events_str,
+                                       tmp);
+                }
+                break;
+            default:
+                break;
+        }
 
-        CLI_STACK_DESTROY (frame);
+        if (event_type > -1) {
+            /* Capture all optional arguments used */
+            ret1 = dict_get_int32(options, "force", &tmpi);
+            if (ret1 == 0) {
+                gf_asprintf_append(&events_str, "%sforce=%d;", events_str,
+                                   tmpi);
+            }
+            ret1 = dict_get_int32(options, "push_pem", &tmpi);
+            if (ret1 == 0) {
+                gf_asprintf_append(&events_str, "%spush_pem=%d;", events_str,
+                                   tmpi);
+            }
+            ret1 = dict_get_int32(options, "no_verify", &tmpi);
+            if (ret1 == 0) {
+                gf_asprintf_append(&events_str, "%sno_verify=%d;", events_str,
+                                   tmpi);
+            }
+
+            ret1 = dict_get_int32(options, "ssh_port", &tmpi);
+            if (ret1 == 0) {
+                gf_asprintf_append(&events_str, "%sssh_port=%d;", events_str,
+                                   tmpi);
+            }
+
+            ret1 = dict_get_int32(options, "reset-sync-time", &tmpi);
+            if (ret1 == 0) {
+                gf_asprintf_append(&events_str, "%sreset_sync_time=%d;",
+                                   events_str, tmpi);
+            }
+            /* Capture Master and Slave Info */
+            ret1 = dict_get_str(options, "master", &tmp);
+            if (ret1)
+                tmp = "";
+            gf_asprintf_append(&events_str, "%smaster=%s;", events_str, tmp);
+
+            ret1 = dict_get_str(options, "slave", &tmp);
+            if (ret1)
+                tmp = "";
+            gf_asprintf_append(&events_str, "%sslave=%s", events_str, tmp);
+
+            gf_event(event_type, "%s", events_str);
+        }
+
+        /* Allocated by gf_strdup and gf_asprintf */
+        if (events_str)
+            GF_FREE(events_str);
+    }
+#endif
 
-        return ret;
+    CLI_STACK_DESTROY(frame);
+
+    return ret;
 }
 
 int
-cli_cmd_volume_status_cbk (struct cli_state *state,
-                           struct cli_cmd_word *word,
-                           const char **words, int wordcount)
+cli_cmd_volume_status_cbk(struct cli_state *state, struct cli_cmd_word *word,
+                          const char **words, int wordcount)
 {
-        int                   ret         = -1;
-        rpc_clnt_procedure_t *proc        = NULL;
-        call_frame_t         *frame       = NULL;
-        dict_t               *dict        = NULL;
-        uint32_t              cmd         = 0;
-        cli_local_t          *local       = NULL;
+    int ret = -1;
+    rpc_clnt_procedure_t *proc = NULL;
+    call_frame_t *frame = NULL;
+    dict_t *dict = NULL;
+    uint32_t cmd = 0;
+    cli_local_t *local = NULL;
 
-        ret = cli_cmd_volume_status_parse (words, wordcount, &dict);
+    ret = cli_cmd_volume_status_parse(words, wordcount, &dict);
 
-        if (ret) {
-                cli_usage_out (word->pattern);
-                goto out;
-        }
+    if (ret) {
+        cli_usage_out(word->pattern);
+        goto out;
+    }
 
-        ret = dict_get_uint32 (dict, "cmd", &cmd);
-        if (ret)
-                goto out;
+    ret = dict_get_uint32(dict, "cmd", &cmd);
+    if (ret)
+        goto out;
 
-        if (!(cmd & GF_CLI_STATUS_ALL)) {
-                /* for one volume or brick */
-                proc = &cli_rpc_prog->proctable[GLUSTER_CLI_STATUS_VOLUME];
-        } else {
-                /* volume status all or all detail */
-                proc = &cli_rpc_prog->proctable[GLUSTER_CLI_STATUS_ALL];
-        }
+    if (!(cmd & GF_CLI_STATUS_ALL)) {
+        /* for one volume or brick */
+        proc = &cli_rpc_prog->proctable[GLUSTER_CLI_STATUS_VOLUME];
+    } else {
+        /* volume status all or all detail */
+        proc = &cli_rpc_prog->proctable[GLUSTER_CLI_STATUS_ALL];
+    }
 
-        if (!proc->fn)
-                goto out;
+    if (!proc->fn) {
+        ret = -1;
+        goto out;
+    }
 
-        frame = create_frame (THIS, THIS->ctx->pool);
-        if (!frame)
-                goto out;
+    frame = create_frame(THIS, THIS->ctx->pool);
+    if (!frame) {
+        gf_log(THIS->name, GF_LOG_ERROR, "failed to create frame");
+        ret = -1;
+        goto out;
+    }
 
-        CLI_LOCAL_INIT (local, words, frame, dict);
+    CLI_LOCAL_INIT(local, words, frame, dict);
 
-        ret = proc->fn (frame, THIS, dict);
+    ret = proc->fn(frame, THIS, dict);
 
 out:
-        CLI_STACK_DESTROY (frame);
+    CLI_STACK_DESTROY(frame);
 
-        return ret;
+    return ret;
 }
 
-
 int
-cli_get_detail_status (dict_t *dict, int i, cli_volume_status_t *status)
+cli_get_detail_status(dict_t *dict, int i, cli_volume_status_t *status)
 {
-        uint64_t                   free            = 0;
-        uint64_t                   total           = 0;
-        char                       key[1024]       = {0};
-        int                        ret             = 0;
-
-        memset (key, 0, sizeof (key));
-        snprintf (key, sizeof (key), "brick%d.free", i);
-        ret = dict_get_uint64 (dict, key, &free);
-
-        status->free = gf_uint64_2human_readable (free);
-        if (!status->free)
-                goto out;
+    uint64_t free = 0;
+    uint64_t total = 0;
+    char key[1024] = {0};
+    int ret = 0;
 
-        memset (key, 0, sizeof (key));
-        snprintf (key, sizeof (key), "brick%d.total", i);
-        ret = dict_get_uint64 (dict, key, &total);
+    snprintf(key, sizeof(key), "brick%d.free", i);
+    ret = dict_get_uint64(dict, key, &free);
 
-        status->total = gf_uint64_2human_readable (total);
-        if (!status->total)
-                goto out;
-
-#ifdef GF_LINUX_HOST_OS
-        memset (key, 0, sizeof (key));
-        snprintf (key, sizeof (key), "brick%d.device", i);
-        ret = dict_get_str (dict, key, &(status->device));
-        if (ret)
-                status->device = NULL;
-#endif
+    status->free = gf_uint64_2human_readable(free);
+    if (!status->free)
+        goto out;
 
-        memset (key, 0, sizeof (key));
-        snprintf (key, sizeof (key), "brick%d.block_size", i);
-        ret = dict_get_uint64 (dict, key, &(status->block_size));
-        if (ret) {
-                ret = 0;
-                status->block_size = 0;
-        }
+    snprintf(key, sizeof(key), "brick%d.total", i);
+    ret = dict_get_uint64(dict, key, &total);
 
-#ifdef GF_LINUX_HOST_OS
-        memset (key, 0, sizeof (key));
-        snprintf (key, sizeof (key), "brick%d.mnt_options", i);
-        ret = dict_get_str (dict, key, &(status->mount_options));
-        if (ret)
-                status->mount_options = NULL;
-
-        memset (key, 0, sizeof (key));
-        snprintf (key, sizeof (key), "brick%d.fs_name", i);
-        ret = dict_get_str (dict, key, &(status->fs_name));
-        if (ret) {
-                ret = 0;
-                status->fs_name = NULL;
-        }
+    status->total = gf_uint64_2human_readable(total);
+    if (!status->total)
+        goto out;
 
-        memset (key, 0, sizeof (key));
-        snprintf (key, sizeof (key), "brick%d.inode_size", i);
-        ret = dict_get_str (dict, key, &(status->inode_size));
-        if (ret)
-                status->inode_size = NULL;
-#endif /* GF_LINUX_HOST_OS */
+    snprintf(key, sizeof(key), "brick%d.device", i);
+    ret = dict_get_str(dict, key, &(status->device));
+    if (ret)
+        status->device = NULL;
 
-        memset (key, 0, sizeof (key));
-        snprintf (key, sizeof (key), "brick%d.total_inodes", i);
-        ret = dict_get_uint64 (dict, key,
-                        &(status->total_inodes));
-        if (ret)
-                status->total_inodes = 0;
+    snprintf(key, sizeof(key), "brick%d.block_size", i);
+    ret = dict_get_uint64(dict, key, &(status->block_size));
+    if (ret) {
+        ret = 0;
+        status->block_size = 0;
+    }
 
-        memset (key, 0, sizeof (key));
-        snprintf (key, sizeof (key), "brick%d.free_inodes", i);
-        ret = dict_get_uint64 (dict, key, &(status->free_inodes));
-        if (ret) {
-                ret = 0;
-                status->free_inodes = 0;
-        }
+    snprintf(key, sizeof(key), "brick%d.mnt_options", i);
+    ret = dict_get_str(dict, key, &(status->mount_options));
+    if (ret)
+        status->mount_options = NULL;
 
+    snprintf(key, sizeof(key), "brick%d.fs_name", i);
+    ret = dict_get_str(dict, key, &(status->fs_name));
+    if (ret) {
+        ret = 0;
+        status->fs_name = NULL;
+    }
+
+    snprintf(key, sizeof(key), "brick%d.inode_size", i);
+    ret = dict_get_str(dict, key, &(status->inode_size));
+    if (ret)
+        status->inode_size = NULL;
+
+    snprintf(key, sizeof(key), "brick%d.total_inodes", i);
+    ret = dict_get_uint64(dict, key, &(status->total_inodes));
+    if (ret)
+        status->total_inodes = 0;
+
+    snprintf(key, sizeof(key), "brick%d.free_inodes", i);
+    ret = dict_get_uint64(dict, key, &(status->free_inodes));
+    if (ret) {
+        ret = 0;
+        status->free_inodes = 0;
+    }
 
- out:
-        return ret;
+out:
+    return ret;
 }
 
 void
-cli_print_detailed_status (cli_volume_status_t *status)
+cli_print_detailed_status(cli_volume_status_t *status)
 {
-        cli_out ("%-20s : %-20s", "Brick", status->brick);
-
-        if (status->online) {
-                cli_out ("%-20s : %-20d", "TCP Port", status->port);
-                cli_out ("%-20s : %-20d", "RDMA Port", status->rdma_port);
-        } else {
-                cli_out ("%-20s : %-20s", "TCP Port", "N/A");
-                cli_out ("%-20s : %-20s", "RDMA Port", "N/A");
-        }
-
-        cli_out ("%-20s : %-20c", "Online", (status->online) ? 'Y' : 'N');
-        cli_out ("%-20s : %-20s", "Pid", status->pid_str);
-
-#ifdef GF_LINUX_HOST_OS
-        if (status->fs_name)
-                cli_out ("%-20s : %-20s", "File System", status->fs_name);
-        else
-                cli_out ("%-20s : %-20s", "File System", "N/A");
-
-        if (status->device)
-                cli_out ("%-20s : %-20s", "Device", status->device);
-        else
-                cli_out ("%-20s : %-20s", "Device", "N/A");
-
-        if (status->mount_options) {
-                cli_out ("%-20s : %-20s", "Mount Options",
-                         status->mount_options);
-        } else {
-                cli_out ("%-20s : %-20s", "Mount Options", "N/A");
-        }
-
-        if (status->inode_size) {
-                cli_out ("%-20s : %-20s", "Inode Size",
-                         status->inode_size);
-        } else {
-                cli_out ("%-20s : %-20s", "Inode Size", "N/A");
-        }
-#endif
-        if (status->free)
-                cli_out ("%-20s : %-20s", "Disk Space Free", status->free);
-        else
-                cli_out ("%-20s : %-20s", "Disk Space Free", "N/A");
-
-        if (status->total)
-                cli_out ("%-20s : %-20s", "Total Disk Space", status->total);
-        else
-                cli_out ("%-20s : %-20s", "Total Disk Space", "N/A");
-
-
-        if (status->total_inodes) {
-                cli_out ("%-20s : %-20"GF_PRI_INODE, "Inode Count",
-                         status->total_inodes);
-        } else {
-                cli_out ("%-20s : %-20s", "Inode Count", "N/A");
-        }
-
-        if (status->free_inodes) {
-                cli_out ("%-20s : %-20"GF_PRI_INODE, "Free Inodes",
-                         status->free_inodes);
-        } else {
-                cli_out ("%-20s : %-20s", "Free Inodes", "N/A");
-        }
+    cli_out("%-20s : %-20s", "Brick", status->brick);
+
+    if (status->online) {
+        cli_out("%-20s : %-20d", "TCP Port", status->port);
+        cli_out("%-20s : %-20d", "RDMA Port", status->rdma_port);
+    } else {
+        cli_out("%-20s : %-20s", "TCP Port", "N/A");
+        cli_out("%-20s : %-20s", "RDMA Port", "N/A");
+    }
+
+    cli_out("%-20s : %-20c", "Online", (status->online) ? 'Y' : 'N');
+    cli_out("%-20s : %-20s", "Pid", status->pid_str);
+
+    if (status->fs_name)
+        cli_out("%-20s : %-20s", "File System", status->fs_name);
+    else
+        cli_out("%-20s : %-20s", "File System", "N/A");
+
+    if (status->device)
+        cli_out("%-20s : %-20s", "Device", status->device);
+    else
+        cli_out("%-20s : %-20s", "Device", "N/A");
+
+    if (status->mount_options) {
+        cli_out("%-20s : %-20s", "Mount Options", status->mount_options);
+    } else {
+        cli_out("%-20s : %-20s", "Mount Options", "N/A");
+    }
+
+    if (status->inode_size) {
+        cli_out("%-20s : %-20s", "Inode Size", status->inode_size);
+    } else {
+        cli_out("%-20s : %-20s", "Inode Size", "N/A");
+    }
+    if (status->free)
+        cli_out("%-20s : %-20s", "Disk Space Free", status->free);
+    else
+        cli_out("%-20s : %-20s", "Disk Space Free", "N/A");
+
+    if (status->total)
+        cli_out("%-20s : %-20s", "Total Disk Space", status->total);
+    else
+        cli_out("%-20s : %-20s", "Total Disk Space", "N/A");
+
+    if (status->total_inodes) {
+        cli_out("%-20s : %-20" GF_PRI_INODE, "Inode Count",
+                status->total_inodes);
+    } else {
+        cli_out("%-20s : %-20s", "Inode Count", "N/A");
+    }
+
+    if (status->free_inodes) {
+        cli_out("%-20s : %-20" GF_PRI_INODE, "Free Inodes",
+                status->free_inodes);
+    } else {
+        cli_out("%-20s : %-20s", "Free Inodes", "N/A");
+    }
 }
 
 int
-cli_print_brick_status (cli_volume_status_t *status)
+cli_print_brick_status(cli_volume_status_t *status)
 {
-        int  fieldlen = CLI_VOL_STATUS_BRICK_LEN;
-        int  bricklen = 0;
-        char *p = NULL;
-        int  num_spaces = 0;
-
-        p = status->brick;
-        bricklen = strlen (p);
-        while (bricklen > 0) {
-                if (bricklen > fieldlen) {
-                        cli_out ("%.*s", fieldlen, p);
-                        p += fieldlen;
-                        bricklen -= fieldlen;
-                } else {
-                        num_spaces = (fieldlen - bricklen) + 1;
-                        printf ("%s", p);
-                        while (num_spaces-- != 0)
-                                printf (" ");
-                        if (status->port || status->rdma_port) {
-                                if (status->online)
-                                        cli_out ("%-10d%-11d%-8c%-5s",
-                                                 status->port,
-                                                 status->rdma_port,
-                                                 status->online?'Y':'N',
-                                                 status->pid_str);
-                                else
-                                        cli_out ("%-10s%-11s%-8c%-5s",
-                                                 "N/A",
-                                                 "N/A",
-                                                 status->online?'Y':'N',
-                                                 status->pid_str);
-                        }
-                        else
-                                cli_out ("%-10s%-11s%-8c%-5s",
-                                         "N/A", "N/A", status->online?'Y':'N',
-                                         status->pid_str);
-                        bricklen = 0;
-                }
-        }
-
-        return 0;
+    int fieldlen = CLI_VOL_STATUS_BRICK_LEN;
+    int bricklen = 0;
+    char *p = NULL;
+    int num_spaces = 0;
+
+    p = status->brick;
+    bricklen = strlen(p);
+    while (bricklen > 0) {
+        if (bricklen > fieldlen) {
+            cli_out("%.*s", fieldlen, p);
+            p += fieldlen;
+            bricklen -= fieldlen;
+        } else {
+            num_spaces = (fieldlen - bricklen) + 1;
+            printf("%s", p);
+            while (num_spaces-- != 0)
+                printf(" ");
+            if (status->port || status->rdma_port) {
+                if (status->online)
+                    cli_out("%-10d%-11d%-8c%-5s", status->port,
+                            status->rdma_port, status->online ? 'Y' : 'N',
+                            status->pid_str);
+                else
+                    cli_out("%-10s%-11s%-8c%-5s", "N/A", "N/A",
+                            status->online ? 'Y' : 'N', status->pid_str);
+            } else
+                cli_out("%-10s%-11s%-8c%-5s", "N/A", "N/A",
+                        status->online ? 'Y' : 'N', status->pid_str);
+            bricklen = 0;
+        }
+    }
+
+    return 0;
 }
 
-#define NEEDS_GLFS_HEAL(op) ((op == GF_SHD_OP_SBRAIN_HEAL_FROM_BIGGER_FILE) || \
-                             (op == GF_SHD_OP_SBRAIN_HEAL_FROM_BRICK) ||      \
-                             (op == GF_SHD_OP_INDEX_SUMMARY) ||               \
-                             (op == GF_SHD_OP_SPLIT_BRAIN_FILES))
+#define NEEDS_GLFS_HEAL(op)                                                    \
+    ((op == GF_SHD_OP_SBRAIN_HEAL_FROM_BIGGER_FILE) ||                         \
+     (op == GF_SHD_OP_SBRAIN_HEAL_FROM_LATEST_MTIME) ||                        \
+     (op == GF_SHD_OP_SBRAIN_HEAL_FROM_BRICK) ||                               \
+     (op == GF_SHD_OP_INDEX_SUMMARY) || (op == GF_SHD_OP_SPLIT_BRAIN_FILES) || \
+     (op == GF_SHD_OP_GRANULAR_ENTRY_HEAL_ENABLE) ||                           \
+     (op == GF_SHD_OP_HEAL_SUMMARY))
 
 int
-cli_launch_glfs_heal (int heal_op, dict_t *options)
+cli_launch_glfs_heal(int heal_op, dict_t *options)
 {
-        char      buff[PATH_MAX] = {0};
-        runner_t  runner         = {0};
-        char      *filename      = NULL;
-        char      *hostname      = NULL;
-        char      *path          = NULL;
-        char      *volname       = NULL;
-        char      *out           = NULL;
-        int        ret           = 0;
-
-        runinit (&runner);
-        ret = dict_get_str (options, "volname", &volname);
-        runner_add_args (&runner, SBIN_DIR"/glfsheal", volname, NULL);
-        runner_redir (&runner, STDOUT_FILENO, RUN_PIPE);
-
-        switch (heal_op) {
+    char buff[PATH_MAX] = {0};
+    runner_t runner = {0};
+    char *filename = NULL;
+    char *hostname = NULL;
+    char *path = NULL;
+    char *volname = NULL;
+    char *out = NULL;
+    int ret = 0;
+
+    runinit(&runner);
+    ret = dict_get_str(options, "volname", &volname);
+    runner_add_args(&runner, GLFSHEAL_PREFIX "/glfsheal", volname, NULL);
+    runner_redir(&runner, STDOUT_FILENO, RUN_PIPE);
+
+    switch (heal_op) {
         case GF_SHD_OP_INDEX_SUMMARY:
-                break;
+            if (global_state->mode & GLUSTER_MODE_XML) {
+                runner_add_args(&runner, "--xml", NULL);
+            }
+            break;
         case GF_SHD_OP_SBRAIN_HEAL_FROM_BIGGER_FILE:
-                ret = dict_get_str (options, "file", &filename);
-                runner_add_args (&runner, "bigger-file", filename, NULL);
-                break;
+            ret = dict_get_str(options, "file", &filename);
+            runner_add_args(&runner, "bigger-file", filename, NULL);
+            break;
+        case GF_SHD_OP_SBRAIN_HEAL_FROM_LATEST_MTIME:
+            ret = dict_get_str(options, "file", &filename);
+            runner_add_args(&runner, "latest-mtime", filename, NULL);
+            break;
         case GF_SHD_OP_SBRAIN_HEAL_FROM_BRICK:
-                ret = dict_get_str (options, "heal-source-hostname",
-                                    &hostname);
-                ret = dict_get_str (options, "heal-source-brickpath",
-                                    &path);
-                runner_add_args (&runner, "source-brick", NULL);
-                runner_argprintf (&runner, "%s:%s", hostname, path);
-                if (dict_get_str (options, "file", &filename) == 0)
-                        runner_argprintf (&runner, filename);
-                break;
+            ret = dict_get_str(options, "heal-source-hostname", &hostname);
+            ret = dict_get_str(options, "heal-source-brickpath", &path);
+            runner_add_args(&runner, "source-brick", NULL);
+            runner_argprintf(&runner, "%s:%s", hostname, path);
+            if (dict_get_str(options, "file", &filename) == 0)
+                runner_argprintf(&runner, "%s", filename);
+            break;
         case GF_SHD_OP_SPLIT_BRAIN_FILES:
-                runner_add_args (&runner, "split-brain-info", NULL);
-                break;
+            runner_add_args(&runner, "split-brain-info", NULL);
+            if (global_state->mode & GLUSTER_MODE_XML) {
+                runner_add_args(&runner, "--xml", NULL);
+            }
+            break;
+        case GF_SHD_OP_GRANULAR_ENTRY_HEAL_ENABLE:
+        case GF_SHD_OP_GRANULAR_ENTRY_HEAL_DISABLE:
+            runner_add_args(&runner, "granular-entry-heal-op", NULL);
+            break;
+        case GF_SHD_OP_HEAL_SUMMARY:
+            runner_add_args(&runner, "info-summary", NULL);
+            if (global_state->mode & GLUSTER_MODE_XML) {
+                runner_add_args(&runner, "--xml", NULL);
+            }
+            break;
         default:
-                ret = -1;
-        }
-        ret = runner_start (&runner);
-        if (ret == -1)
-                goto out;
-        while ((out = fgets (buff, sizeof(buff),
-                             runner_chio (&runner, STDOUT_FILENO)))) {
-                printf ("%s", out);
-        }
-        ret = runner_end (&runner);
-        ret = WEXITSTATUS (ret);
+            ret = -1;
+            goto out;
+    }
+    if (global_state->mode & GLUSTER_MODE_GLFSHEAL_NOLOG)
+        runner_add_args(&runner, "--nolog", NULL);
+    ret = runner_start(&runner);
+    if (ret == -1)
+        goto out;
+    while ((
+        out = fgets(buff, sizeof(buff), runner_chio(&runner, STDOUT_FILENO)))) {
+        printf("%s", out);
+    }
+    ret = runner_end(&runner);
 
 out:
-        return ret;
+    return ret;
 }
+
 int
-cli_cmd_volume_heal_cbk (struct cli_state *state, struct cli_cmd_word *word,
-                          const char **words, int wordcount)
+cli_cmd_volume_heal_cbk(struct cli_state *state, struct cli_cmd_word *word,
+                        const char **words, int wordcount)
 {
-        int                     ret = -1;
-        rpc_clnt_procedure_t    *proc = NULL;
-        call_frame_t            *frame = NULL;
-        int                     sent = 0;
-        int                     parse_error = 0;
-        dict_t                  *options = NULL;
-        xlator_t                *this = NULL;
-        cli_local_t             *local = NULL;
-        int                     heal_op = 0;
-
-        this = THIS;
-        frame = create_frame (this, this->ctx->pool);
-        if (!frame)
-                goto out;
-
-        if (wordcount < 3) {
-               cli_usage_out (word->pattern);
-               parse_error = 1;
-               goto out;
-        }
+    int ret = -1;
+    rpc_clnt_procedure_t *proc = NULL;
+    call_frame_t *frame = NULL;
+    int sent = 0;
+    int parse_error = 0;
+    dict_t *options = NULL;
+    xlator_t *this = NULL;
+    cli_local_t *local = NULL;
+    int heal_op = 0;
+
+    this = THIS;
+
+    if (wordcount < 3) {
+        cli_usage_out(word->pattern);
+        parse_error = 1;
+        goto out;
+    }
 
-        ret = cli_cmd_volume_heal_options_parse (words, wordcount, &options);
-        if (ret) {
-                cli_usage_out (word->pattern);
-                parse_error = 1;
-                goto out;
-        }
-        ret = dict_get_int32 (options, "heal-op", &heal_op);
+    ret = cli_cmd_volume_heal_options_parse(words, wordcount, &options);
+    if (ret) {
+        cli_usage_out(word->pattern);
+        parse_error = 1;
+        goto out;
+    }
+    ret = dict_get_int32(options, "heal-op", &heal_op);
+    if (ret < 0)
+        goto out;
+    if (NEEDS_GLFS_HEAL(heal_op)) {
+        ret = cli_launch_glfs_heal(heal_op, options);
         if (ret < 0)
-                goto out;
-        if (NEEDS_GLFS_HEAL (heal_op)) {
-                ret = cli_launch_glfs_heal (heal_op, options);
-                if (ret == -1)
-                        goto out;
-        }
-        else {
-                proc = &cli_rpc_prog->proctable[GLUSTER_CLI_HEAL_VOLUME];
+            goto out;
+        if (heal_op != GF_SHD_OP_GRANULAR_ENTRY_HEAL_ENABLE)
+            goto out;
+    }
 
-                CLI_LOCAL_INIT (local, words, frame, options);
+    proc = &cli_rpc_prog->proctable[GLUSTER_CLI_HEAL_VOLUME];
 
-                if (proc->fn) {
-                        ret = proc->fn (frame, THIS, options);
-                }
-        }
+    frame = create_frame(this, this->ctx->pool);
+    if (!frame) {
+        ret = -1;
+        goto out;
+    }
+
+    CLI_LOCAL_INIT(local, words, frame, options);
 
+    if (proc->fn) {
+        ret = proc->fn(frame, THIS, options);
+    }
 out:
-        if (ret) {
-                cli_cmd_sent_status_get (&sent);
-                if ((sent == 0) && (parse_error == 0))
-                        cli_out ("Volume heal failed.");
+    if (ret) {
+        cli_cmd_sent_status_get(&sent);
+        if ((sent == 0) && (parse_error == 0) &&
+            !(global_state->mode & GLUSTER_MODE_XML)) {
+            cli_out("Volume heal failed.");
         }
+    }
+
+    if (options)
+        dict_unref(options);
 
-        CLI_STACK_DESTROY (frame);
+    CLI_STACK_DESTROY(frame);
 
-        return ret;
+    return ret;
 }
 
 int
-cli_cmd_volume_statedump_cbk (struct cli_state *state, struct cli_cmd_word *word,
-                              const char **words, int wordcount)
+cli_cmd_volume_statedump_cbk(struct cli_state *state, struct cli_cmd_word *word,
+                             const char **words, int wordcount)
 {
-        int                             ret = -1;
-        rpc_clnt_procedure_t            *proc = NULL;
-        call_frame_t                    *frame = NULL;
-        dict_t                          *options = NULL;
-        int                             sent = 0;
-        int                             parse_error = 0;
-        cli_local_t                     *local = NULL;
-
-        frame = create_frame (THIS, THIS->ctx->pool);
-        if (!frame)
-                goto out;
+    int ret = -1;
+    rpc_clnt_procedure_t *proc = NULL;
+    call_frame_t *frame = NULL;
+    dict_t *options = NULL;
+    int sent = 0;
+    int parse_error = 0;
+    cli_local_t *local = NULL;
+
+    if (wordcount < 3) {
+        cli_usage_out(word->pattern);
+        parse_error = 1;
+        goto out;
+    }
 
-        if (wordcount < 3) {
-                cli_usage_out (word->pattern);
-                parse_error = 1;
-                goto out;
+    if (wordcount >= 3) {
+        ret = cli_cmd_volume_statedump_options_parse(words, wordcount,
+                                                     &options);
+        if (ret) {
+            parse_error = 1;
+            gf_log("cli", GF_LOG_ERROR,
+                   "Error parsing "
+                   "statedump options");
+            cli_out("Error parsing options");
+            cli_usage_out(word->pattern);
         }
+    }
 
-        if (wordcount >= 3) {
-               ret = cli_cmd_volume_statedump_options_parse (words, wordcount,
-                                                              &options);
-               if (ret) {
-                       parse_error = 1;
-                       gf_log ("cli", GF_LOG_ERROR, "Error parsing "
-                               "statedump options");
-                       cli_out ("Error parsing options");
-                       cli_usage_out (word->pattern);
-               }
-        }
+    ret = dict_set_str(options, "volname", (char *)words[2]);
+    if (ret)
+        goto out;
 
-        ret = dict_set_str (options, "volname", (char *)words[2]);
-        if (ret)
-                goto out;
+    proc = &cli_rpc_prog->proctable[GLUSTER_CLI_STATEDUMP_VOLUME];
 
-        proc = &cli_rpc_prog->proctable[GLUSTER_CLI_STATEDUMP_VOLUME];
+    frame = create_frame(THIS, THIS->ctx->pool);
+    if (!frame) {
+        ret = -1;
+        goto out;
+    }
 
-        CLI_LOCAL_INIT (local, words, frame, options);
+    CLI_LOCAL_INIT(local, words, frame, options);
 
-        if (proc->fn) {
-                ret = proc->fn (frame, THIS, options);
-        }
+    if (proc->fn) {
+        ret = proc->fn(frame, THIS, options);
+    }
 
 out:
-        if (ret) {
-                cli_cmd_sent_status_get (&sent);
-                if ((sent == 0) && (parse_error == 0))
-                        cli_out ("Volume statedump failed");
-        }
+    if (ret) {
+        cli_cmd_sent_status_get(&sent);
+        if ((sent == 0) && (parse_error == 0))
+            cli_out("Volume statedump failed");
+    }
 
-        CLI_STACK_DESTROY (frame);
+    CLI_STACK_DESTROY(frame);
 
-        return ret;
+    return ret;
 }
 
 int
-cli_cmd_volume_list_cbk (struct cli_state *state, struct cli_cmd_word *word,
-                         const char **words, int wordcount)
+cli_cmd_volume_list_cbk(struct cli_state *state, struct cli_cmd_word *word,
+                        const char **words, int wordcount)
 {
-        int                     ret = -1;
-        call_frame_t            *frame = NULL;
-        rpc_clnt_procedure_t    *proc = NULL;
-        int                     sent = 0;
-
-        frame = create_frame (THIS, THIS->ctx->pool);
+    int ret = -1;
+    call_frame_t *frame = NULL;
+    rpc_clnt_procedure_t *proc = NULL;
+    int sent = 0;
+
+    proc = &cli_rpc_prog->proctable[GLUSTER_CLI_LIST_VOLUME];
+    if (proc->fn) {
+        frame = create_frame(THIS, THIS->ctx->pool);
         if (!frame)
-                goto out;
-
-        proc = &cli_rpc_prog->proctable[GLUSTER_CLI_LIST_VOLUME];
-        if (proc->fn) {
-                ret = proc->fn (frame, THIS, NULL);
-        }
+            goto out;
+        ret = proc->fn(frame, THIS, NULL);
+    }
 
 out:
-        if (ret) {
-                cli_cmd_sent_status_get (&sent);
-                if (sent == 0)
-                        cli_out ("Volume list failed");
-        }
+    if (ret) {
+        cli_cmd_sent_status_get(&sent);
+        if (sent == 0)
+            cli_out("Volume list failed");
+    }
 
-        CLI_STACK_DESTROY (frame);
+    CLI_STACK_DESTROY(frame);
 
-        return ret;
+    return ret;
 }
 
 int
-cli_cmd_volume_clearlocks_cbk (struct cli_state *state,
-                               struct cli_cmd_word *word,
-                               const char **words, int wordcount)
+cli_cmd_volume_clearlocks_cbk(struct cli_state *state,
+                              struct cli_cmd_word *word, const char **words,
+                              int wordcount)
 {
-        int                             ret = -1;
-        rpc_clnt_procedure_t            *proc = NULL;
-        call_frame_t                    *frame = NULL;
-        dict_t                          *options = NULL;
-        int                             sent = 0;
-        int                             parse_error = 0;
-        cli_local_t                     *local = NULL;
-
-        frame = create_frame (THIS, THIS->ctx->pool);
-        if (!frame)
-                goto out;
-
-        if (wordcount < 7 || wordcount > 8) {
-                cli_usage_out (word->pattern);
-                parse_error = 1;
-                goto out;
-        }
-
-       ret = cli_cmd_volume_clrlks_opts_parse (words, wordcount, &options);
-       if (ret) {
-               parse_error = 1;
-               gf_log ("cli", GF_LOG_ERROR, "Error parsing "
-                       "clear-locks options");
-               cli_out ("Error parsing options");
-               cli_usage_out (word->pattern);
-       }
+    int ret = -1;
+    rpc_clnt_procedure_t *proc = NULL;
+    call_frame_t *frame = NULL;
+    dict_t *options = NULL;
+    int sent = 0;
+    int parse_error = 0;
+    cli_local_t *local = NULL;
+
+    if (wordcount < 7 || wordcount > 8) {
+        cli_usage_out(word->pattern);
+        parse_error = 1;
+        goto out;
+    }
+
+    ret = cli_cmd_volume_clrlks_opts_parse(words, wordcount, &options);
+    if (ret) {
+        parse_error = 1;
+        gf_log("cli", GF_LOG_ERROR,
+               "Error parsing "
+               "clear-locks options");
+        cli_out("Error parsing options");
+        cli_usage_out(word->pattern);
+    }
+
+    ret = dict_set_str(options, "volname", (char *)words[2]);
+    if (ret)
+        goto out;
 
-        ret = dict_set_str (options, "volname", (char *)words[2]);
-        if (ret)
-                goto out;
+    ret = dict_set_str(options, "path", (char *)words[3]);
+    if (ret)
+        goto out;
 
-        ret = dict_set_str (options, "path", (char *)words[3]);
-        if (ret)
-                goto out;
+    proc = &cli_rpc_prog->proctable[GLUSTER_CLI_CLRLOCKS_VOLUME];
 
-        proc = &cli_rpc_prog->proctable[GLUSTER_CLI_CLRLOCKS_VOLUME];
+    frame = create_frame(THIS, THIS->ctx->pool);
+    if (!frame) {
+        ret = -1;
+        goto out;
+    }
 
-        CLI_LOCAL_INIT (local, words, frame, options);
+    CLI_LOCAL_INIT(local, words, frame, options);
 
-        if (proc->fn) {
-                ret = proc->fn (frame, THIS, options);
-        }
+    if (proc->fn) {
+        ret = proc->fn(frame, THIS, options);
+    }
 
 out:
-        if (ret) {
-                cli_cmd_sent_status_get (&sent);
-                if ((sent == 0) && (parse_error == 0))
-                        cli_out ("Volume clear-locks failed");
-        }
+    if (ret) {
+        cli_cmd_sent_status_get(&sent);
+        if ((sent == 0) && (parse_error == 0))
+            cli_out("Volume clear-locks failed");
+    }
 
-        CLI_STACK_DESTROY (frame);
+    CLI_STACK_DESTROY(frame);
 
-        return ret;
+    return ret;
 }
 
 int
-cli_cmd_volume_barrier_cbk (struct cli_state *state, struct cli_cmd_word *word,
-                            const char **words, int wordcount)
+cli_cmd_volume_barrier_cbk(struct cli_state *state, struct cli_cmd_word *word,
+                           const char **words, int wordcount)
 {
-        int ret = -1;
-        rpc_clnt_procedure_t *proc = NULL;
-        call_frame_t *frame = NULL;
-        dict_t *options = NULL;
-        int sent = 0;
-        int parse_error = 0;
-        cli_local_t *local = NULL;
-
-        frame = create_frame (THIS, THIS->ctx->pool);
-        if (!frame)
-                goto out;
+    int ret = -1;
+    rpc_clnt_procedure_t *proc = NULL;
+    call_frame_t *frame = NULL;
+    dict_t *options = NULL;
+    int sent = 0;
+    int parse_error = 0;
+    cli_local_t *local = NULL;
+
+    if (wordcount != 4) {
+        cli_usage_out(word->pattern);
+        parse_error = 1;
+        goto out;
+    }
 
-        if (wordcount != 4) {
-                cli_usage_out (word->pattern);
-                parse_error = 1;
-                goto out;
-        }
+    options = dict_new();
+    if (!options) {
+        ret = -1;
+        goto out;
+    }
+    ret = dict_set_str(options, "volname", (char *)words[2]);
+    if (ret)
+        goto out;
 
-        options = dict_new();
-        if (!options) {
-                ret = -1;
-                goto out;
-        }
-        ret = dict_set_str(options, "volname", (char *)words[2]);
-        if (ret)
-                goto out;
+    ret = dict_set_str(options, "barrier", (char *)words[3]);
+    if (ret)
+        goto out;
 
-        ret = dict_set_str (options, "barrier", (char *)words[3]);
-        if (ret)
-                goto out;
+    proc = &cli_rpc_prog->proctable[GLUSTER_CLI_BARRIER_VOLUME];
 
-        proc = &cli_rpc_prog->proctable[GLUSTER_CLI_BARRIER_VOLUME];
+    frame = create_frame(THIS, THIS->ctx->pool);
+    if (!frame) {
+        ret = -1;
+        goto out;
+    }
 
-        CLI_LOCAL_INIT (local, words, frame, options);
+    CLI_LOCAL_INIT(local, words, frame, options);
 
-        if (proc->fn)
-                ret = proc->fn (frame, THIS, options);
+    if (proc->fn)
+        ret = proc->fn(frame, THIS, options);
 
 out:
-        if (ret) {
-                cli_cmd_sent_status_get (&sent);
-                if ((sent == 0) && (parse_error == 0))
-                        cli_err ("Volume barrier failed");
-        }
-        CLI_STACK_DESTROY (frame);
-        if (options)
-                dict_unref (options);
-
-        return ret;
+    if (ret) {
+        cli_cmd_sent_status_get(&sent);
+        if ((sent == 0) && (parse_error == 0))
+            cli_err("Volume barrier failed");
+    }
+    CLI_STACK_DESTROY(frame);
+
+    return ret;
 }
 
 int
-cli_cmd_volume_getopt_cbk (struct cli_state *state, struct cli_cmd_word *word,
-                        const char **words, int wordcount)
+cli_cmd_volume_getopt_cbk(struct cli_state *state, struct cli_cmd_word *word,
+                          const char **words, int wordcount)
 {
-        int                   ret       = -1;
-        rpc_clnt_procedure_t *proc      = NULL;
-        call_frame_t         *frame     = NULL;
-        dict_t               *options   = NULL;
-        int                   sent      = 0;
-        int                   parse_err = 0;
-        cli_local_t          *local     = NULL;
-
-        if (wordcount != 4) {
-                cli_usage_out (word->pattern);
-                parse_err = 1;
-                goto out;
-        }
+    int ret = -1;
+    rpc_clnt_procedure_t *proc = NULL;
+    call_frame_t *frame = NULL;
+    dict_t *options = NULL;
+    int sent = 0;
+    int parse_err = 0;
+    cli_local_t *local = NULL;
+
+    if (wordcount != 4) {
+        cli_usage_out(word->pattern);
+        parse_err = 1;
+        goto out;
+    }
 
-        frame = create_frame (THIS, THIS->ctx->pool);
-        if (!frame)
-                goto out;
+    options = dict_new();
+    if (!options)
+        goto out;
 
-        options = dict_new ();
-        if (!options)
-                goto out;
+    ret = dict_set_str(options, "volname", (char *)words[2]);
+    if (ret)
+        goto out;
 
-        ret = dict_set_str (options, "volname", (char *)words[2]);
-        if (ret)
-                goto out;
+    ret = dict_set_str(options, "key", (char *)words[3]);
+    if (ret)
+        goto out;
 
-        ret = dict_set_str (options, "key", (char *)words[3]);
-        if (ret)
-                goto out;
+    proc = &cli_rpc_prog->proctable[GLUSTER_CLI_GET_VOL_OPT];
 
-        proc = &cli_rpc_prog->proctable[GLUSTER_CLI_GET_VOL_OPT];
+    frame = create_frame(THIS, THIS->ctx->pool);
+    if (!frame) {
+        ret = -1;
+        goto out;
+    }
 
-        CLI_LOCAL_INIT (local, words, frame, options);
+    CLI_LOCAL_INIT(local, words, frame, options);
 
-        if (proc->fn)
-                ret = proc->fn (frame, THIS, options);
+    if (proc->fn)
+        ret = proc->fn(frame, THIS, options);
 
 out:
-        if (ret) {
-                cli_cmd_sent_status_get (&sent);
-                if ((sent == 0) && (parse_err == 0))
-                        cli_err ("Volume get option failed");
-        }
-        CLI_STACK_DESTROY (frame);
-        if (options)
-                dict_unref (options);
-        return ret;
+    if (ret) {
+        cli_cmd_sent_status_get(&sent);
+        if ((sent == 0) && (parse_err == 0))
+            cli_err("Volume get option failed");
+    }
+    CLI_STACK_DESTROY(frame);
+    return ret;
 }
 
+/* This is a bit of a hack to display the help. The current bitrot cmd
+ * format does not work well when registering the cmds.
+ * Ideally the should have been of the form
+ * gluster volume bitrot <subcommand> <volumename> ...
+ */
+
+struct cli_cmd bitrot_cmds[] = {
+
+    {"volume bitrot help", cli_cmd_bitrot_help_cbk,
+     "display help for volume bitrot commands"},
+
+    {"volume bitrot <VOLNAME> {enable|disable}", NULL, /*cli_cmd_bitrot_cbk,*/
+     "Enable/disable bitrot for volume <VOLNAME>"},
+
+    {"volume bitrot <VOLNAME> signing-time <time-in-secs>",
+     NULL, /*cli_cmd_bitrot_cbk,*/
+     "Waiting time for an object after last fd is closed to start signing "
+     "process"},
+
+    {"volume bitrot <VOLNAME> signer-threads <count>",
+     NULL, /*cli_cmd_bitrot_cbk,*/
+     "Number of signing process threads. Usually set to number of available "
+     "cores"},
+
+    {"volume bitrot <VOLNAME> scrub-throttle {lazy|normal|aggressive}",
+     NULL, /*cli_cmd_bitrot_cbk,*/
+     "Set the speed of the scrubber for volume <VOLNAME>"},
+
+    {"volume bitrot <VOLNAME> scrub-frequency {hourly|daily|weekly|biweekly"
+     "|monthly}",
+     NULL, /*cli_cmd_bitrot_cbk,*/
+     "Set the frequency of the scrubber for volume <VOLNAME>"},
+
+    {"volume bitrot <VOLNAME> scrub {pause|resume|status|ondemand}",
+     NULL, /*cli_cmd_bitrot_cbk,*/
+     "Pause/resume the scrubber for <VOLNAME>. Status displays the status of "
+     "the scrubber. ondemand starts the scrubber immediately."},
+
+    {"volume bitrot <VOLNAME> {enable|disable}\n"
+     "volume bitrot <VOLNAME> signing-time <time-in-secs>\n"
+     "volume bitrot <VOLNAME> signer-threads <count>\n"
+     "volume bitrot <volname> scrub-throttle {lazy|normal|aggressive}\n"
+     "volume bitrot <volname> scrub-frequency {hourly|daily|weekly|biweekly"
+     "|monthly}\n"
+     "volume bitrot <volname> scrub {pause|resume|status|ondemand}",
+     cli_cmd_bitrot_cbk, NULL},
+
+    {NULL, NULL, NULL}};
+
+struct cli_cmd quota_cmds[] = {
+
+    /* Quota commands */
+    {"volume quota help", cli_cmd_quota_help_cbk,
+     "display help for volume quota commands"},
+
+    {"volume quota <VOLNAME> {enable|disable|list [<path> ...]| "
+     "list-objects [<path> ...] | remove <path>| remove-objects <path> | "
+     "default-soft-limit <percent>}",
+     cli_cmd_quota_cbk, "Enable/disable and configure quota for <VOLNAME>"},
+
+    {"volume quota <VOLNAME> {limit-usage <path> <size> [<percent>]}",
+     cli_cmd_quota_cbk, "Set maximum size for <path> for <VOLNAME>"},
+
+    {"volume quota <VOLNAME> {limit-objects <path> <number> [<percent>]}",
+     cli_cmd_quota_cbk,
+     "Set the maximum number of entries allowed in <path> for <VOLNAME>"},
+
+    {"volume quota <VOLNAME> {alert-time|soft-timeout|hard-timeout} {<time>}",
+     cli_cmd_quota_cbk, "Set quota timeout for <VOLNAME>"},
+
+    {"volume inode-quota <VOLNAME> enable", cli_cmd_quota_cbk,
+     "Enable/disable inode-quota for <VOLNAME>"},
+
+    {"volume quota <VOLNAME> {enable|disable|list [<path> ...]| "
+     "list-objects [<path> ...] | remove <path>| remove-objects <path> | "
+     "default-soft-limit <percent>}\n"
+     "volume quota <VOLNAME> {limit-usage <path> <size> [<percent>]}\n"
+     "volume quota <VOLNAME> {limit-objects <path> <number> [<percent>]}\n"
+     "volume quota <VOLNAME> {alert-time|soft-timeout|hard-timeout} {<time>}",
+     cli_cmd_quota_cbk, NULL},
+
+    {NULL, NULL, NULL}};
+
 struct cli_cmd volume_cmds[] = {
-        { "volume info [all|<VOLNAME>]",
-          cli_cmd_volume_info_cbk,
-          "list information of all volumes"},
-
-        { "volume create <NEW-VOLNAME> [stripe <COUNT>] "
-          "[replica <COUNT> [arbiter <COUNT>]] "
-          "[disperse [<COUNT>]] [disperse-data <COUNT>] [redundancy <COUNT>] "
-          "[transport <tcp|rdma|tcp,rdma>] <NEW-BRICK>"
-#ifdef HAVE_BD_XLATOR
-          "?<vg_name>"
-#endif
-          "... [force]",
-
-          cli_cmd_volume_create_cbk,
-          "create a new volume of specified type with mentioned bricks"},
-
-        { "volume delete <VOLNAME>",
-          cli_cmd_volume_delete_cbk,
-          "delete volume specified by <VOLNAME>"},
-
-        { "volume start <VOLNAME> [force]",
-          cli_cmd_volume_start_cbk,
-          "start volume specified by <VOLNAME>"},
-
-        { "volume stop <VOLNAME> [force]",
-          cli_cmd_volume_stop_cbk,
-          "stop volume specified by <VOLNAME>"},
-
-        /*{ "volume rename <VOLNAME> <NEW-VOLNAME>",
-          cli_cmd_volume_rename_cbk,
-          "rename volume <VOLNAME> to <NEW-VOLNAME>"},*/
-
-        { "volume tier <VOLNAME> status\n"
-        "volume tier <VOLNAME> start [force]\n"
-        "volume tier <VOLNAME> attach [<replica COUNT>] <NEW-BRICK>...\n"
-        "volume tier <VOLNAME> detach <start|stop|status|commit|[force]>\n",
-        cli_cmd_volume_tier_cbk,
-        "Tier translator specific operations."},
-
-        { "volume attach-tier <VOLNAME> [<replica COUNT>] <NEW-BRICK>...",
-        cli_cmd_volume_tier_cbk,
-          "NOTE: this is old syntax, will be depreciated in next release. "
-          "Please use gluster volume tier <vol> attach "
-          "[<replica COUNT>] <NEW-BRICK>..."},
-
-        { "volume detach-tier <VOLNAME> "
-          " <start|stop|status|commit|force>",
-        cli_cmd_volume_tier_cbk,
-          "NOTE: this is old syntax, will be depreciated in next release. "
-          "Please use gluster volume tier <vol> detach "
-          "{start|stop|commit} [force]"},
-
-        { "volume add-brick <VOLNAME> [<stripe|replica> <COUNT>] <NEW-BRICK> ... [force]",
-          cli_cmd_volume_add_brick_cbk,
-          "add brick to volume <VOLNAME>"},
-
-        { "volume remove-brick <VOLNAME> [replica <COUNT>] <BRICK> ..."
-          " <start|stop|status|commit|force>",
-          cli_cmd_volume_remove_brick_cbk,
-          "remove brick from volume <VOLNAME>"},
-
-        { "volume rebalance <VOLNAME> {{fix-layout start} | {start [force]|stop|status}}",
-          cli_cmd_volume_defrag_cbk,
-          "rebalance operations"},
-
-        { "volume replace-brick <VOLNAME> <SOURCE-BRICK> <NEW-BRICK> "
-          "{commit force}",
-          cli_cmd_volume_replace_brick_cbk,
-          "replace-brick operations"},
-
-        /*{ "volume set-transport <VOLNAME> <TRANSPORT-TYPE> [<TRANSPORT-TYPE>] ...",
-          cli_cmd_volume_set_transport_cbk,
-          "set transport type for volume <VOLNAME>"},*/
-
-        { "volume set <VOLNAME> <KEY> <VALUE>",
-          cli_cmd_volume_set_cbk,
-         "set options for volume <VOLNAME>"},
-
-        { "volume help",
-          cli_cmd_volume_help_cbk,
-          "display help for the volume command"},
-
-        { "volume log <VOLNAME> rotate [BRICK]",
-          cli_cmd_log_rotate_cbk,
-         "rotate the log file for corresponding volume/brick"},
-
-        { "volume log rotate <VOLNAME> [BRICK]",
-          cli_cmd_log_rotate_cbk,
-         "rotate the log file for corresponding volume/brick"
-         " NOTE: This is an old syntax, will be deprecated from next release."},
-
-        { "volume sync <HOSTNAME> [all|<VOLNAME>]",
-          cli_cmd_sync_volume_cbk,
-         "sync the volume information from a peer"},
-
-         { "volume reset <VOLNAME> [option] [force]",
-         cli_cmd_volume_reset_cbk,
-         "reset all the reconfigured options"},
+    {"volume help", cli_cmd_volume_help_cbk,
+     "display help for volume commands"},
+
+    {"volume info [all|<VOLNAME>]", cli_cmd_volume_info_cbk,
+     "list information of all volumes"},
+
+    {"volume create <NEW-VOLNAME> [stripe <COUNT>] "
+     "[[replica <COUNT> [arbiter <COUNT>]]|[replica 2 thin-arbiter 1]] "
+     "[disperse [<COUNT>]] [disperse-data <COUNT>] [redundancy <COUNT>] "
+     "[transport <tcp|rdma|tcp,rdma>] <NEW-BRICK> <TA-BRICK>"
+     "... [force]",
+
+     cli_cmd_volume_create_cbk,
+     "create a new volume of specified type with mentioned bricks"},
+
+    {"volume delete <VOLNAME>", cli_cmd_volume_delete_cbk,
+     "delete volume specified by <VOLNAME>"},
+
+    {"volume start <VOLNAME> [force]", cli_cmd_volume_start_cbk,
+     "start volume specified by <VOLNAME>"},
+
+    {"volume stop <VOLNAME> [force]", cli_cmd_volume_stop_cbk,
+     "stop volume specified by <VOLNAME>"},
+
+    /*{ "volume rename <VOLNAME> <NEW-VOLNAME>",
+      cli_cmd_volume_rename_cbk,
+      "rename volume <VOLNAME> to <NEW-VOLNAME>"},*/
+
+    {"volume add-brick <VOLNAME> [<stripe|replica> <COUNT> "
+     "[arbiter <COUNT>]] <NEW-BRICK> ... [force]",
+     cli_cmd_volume_add_brick_cbk, "add brick to volume <VOLNAME>"},
+
+    {"volume remove-brick <VOLNAME> [replica <COUNT>] <BRICK> ..."
+     " <start|stop|status|commit|force>",
+     cli_cmd_volume_remove_brick_cbk, "remove brick from volume <VOLNAME>"},
+
+    {"volume rebalance <VOLNAME> {{fix-layout start} | {start "
+     "[force]|stop|status}}",
+     cli_cmd_volume_defrag_cbk, "rebalance operations"},
+
+    {"volume replace-brick <VOLNAME> <SOURCE-BRICK> <NEW-BRICK> "
+     "{commit force}",
+     cli_cmd_volume_replace_brick_cbk, "replace-brick operations"},
+
+    /*{ "volume set-transport <VOLNAME> <TRANSPORT-TYPE> [<TRANSPORT-TYPE>]
+      ...", cli_cmd_volume_set_transport_cbk, "set transport type for volume
+      <VOLNAME>"},*/
+
+    {"volume set <VOLNAME> <KEY> <VALUE>", cli_cmd_volume_set_cbk,
+     "set options for volume <VOLNAME>"},
+
+    {"volume set <VOLNAME> group <GROUP>", cli_cmd_volume_set_cbk,
+     "This option can be used for setting multiple pre-defined volume options "
+     "where group_name is a file under /var/lib/glusterd/groups containing one "
+     "key value pair per line"},
+
+    {"volume log <VOLNAME> rotate [BRICK]", cli_cmd_log_rotate_cbk,
+     "rotate the log file for corresponding volume/brick"},
+
+    {"volume sync <HOSTNAME> [all|<VOLNAME>]", cli_cmd_sync_volume_cbk,
+     "sync the volume information from a peer"},
+
+    {"volume reset <VOLNAME> [option] [force]", cli_cmd_volume_reset_cbk,
+     "reset all the reconfigured options"},
 
 #if (SYNCDAEMON_COMPILE)
-        {"volume "GEOREP" [<VOLNAME>] [<SLAVE-URL>] {create [[ssh-port n] [[no-verify]|[push-pem]]] [force]"
-         "|start [force]|stop [force]|pause [force]|resume [force]|config|status [detail]|delete} [options...]",
-         cli_cmd_volume_gsync_set_cbk,
-         "Geo-sync operations",
-         cli_cmd_check_gsync_exists_cbk},
+    {"volume " GEOREP " [<MASTER-VOLNAME>] [<SLAVE-IP>]::[<SLAVE-VOLNAME>] {"
+     "\\\n create [[ssh-port n] [[no-verify] \\\n | [push-pem]]] [force] \\\n"
+     " | start [force] \\\n | stop [force] \\\n | pause [force] \\\n | resume "
+     "[force] \\\n"
+     " | config [[[\\!]<option>] [<value>]] \\\n | status "
+     "[detail] \\\n | delete [reset-sync-time]} ",
+     cli_cmd_volume_gsync_set_cbk, "Geo-sync operations",
+     cli_cmd_check_gsync_exists_cbk},
 #endif
 
-         { "volume profile <VOLNAME> {start|info [peek|incremental [peek]|cumulative|clear]|stop} [nfs]",
-           cli_cmd_volume_profile_cbk,
-           "volume profile operations"},
-
-        { "volume quota <VOLNAME> {enable|disable|list [<path> ...]| "
-          "list-objects [<path> ...] | remove <path>| remove-objects <path> | "
-          "default-soft-limit <percent>} |\n"
-          "volume quota <VOLNAME> {limit-usage <path> <size> [<percent>]} |\n"
-          "volume quota <VOLNAME> {limit-objects <path> <number> [<percent>]} |\n"
-          "volume quota <VOLNAME> {alert-time|soft-timeout|hard-timeout} {<time>}",
-          cli_cmd_quota_cbk,
-          "quota translator specific operations"},
-
-        { "volume inode-quota <VOLNAME> enable",
-          cli_cmd_quota_cbk,
-          "quota translator specific operations"},
-
-         { "volume top <VOLNAME> {open|read|write|opendir|readdir|clear} [nfs|brick <brick>] [list-cnt <value>] |\n"
-           "volume top <VOLNAME> {read-perf|write-perf} [bs <size> count <count>] [brick <brick>] [list-cnt <value>]",
-           cli_cmd_volume_top_cbk,
-           "volume top operations"},
-
-        { "volume status [all | <VOLNAME> [nfs|shd|<BRICK>|quotad]]"
-          " [detail|clients|mem|inode|fd|callpool|tasks]",
-          cli_cmd_volume_status_cbk,
-          "display status of all or specified volume(s)/brick"},
-
-        { "volume heal <VOLNAME> [enable | disable | full |"
-          "statistics [heal-count [replica <HOSTNAME:BRICKNAME>]] |"
-          "info [healed | heal-failed | split-brain] |"
-          "split-brain {bigger-file <FILE> |"
-                       "source-brick <HOSTNAME:BRICKNAME> [<FILE>]}]",
-          cli_cmd_volume_heal_cbk,
-          "self-heal commands on volume specified by <VOLNAME>"},
-
-        {"volume statedump <VOLNAME> [nfs|quotad] [all|mem|iobuf|callpool|priv|fd|"
-         "inode|history]...",
-         cli_cmd_volume_statedump_cbk,
-         "perform statedump on bricks"},
-
-        {"volume list",
-         cli_cmd_volume_list_cbk,
-         "list all volumes in cluster"},
-
-        {"volume clear-locks <VOLNAME> <path> kind {blocked|granted|all}"
-          "{inode [range]|entry [basename]|posix [range]}",
-          cli_cmd_volume_clearlocks_cbk,
-          "Clear locks held on path"
-        },
-        {"volume barrier <VOLNAME> {enable|disable}",
-         cli_cmd_volume_barrier_cbk,
-         "Barrier/unbarrier file operations on a volume"
-        },
-        {"volume get <VOLNAME> <key|all>",
-         cli_cmd_volume_getopt_cbk,
-         "Get the value of the all options or given option for volume <VOLNAME>"
-        },
-        {"volume bitrot <VOLNAME> {enable|disable} |\n"
-         "volume bitrot <volname> scrub-throttle {lazy|normal|aggressive} |\n"
-         "volume bitrot <volname> scrub-frequency {hourly|daily|weekly|biweekly"
-         "|monthly} |\n"
-         "volume bitrot <volname> scrub {pause|resume|status}",
-         cli_cmd_bitrot_cbk,
-         "Bitrot translator specific operation. For more information about "
-         "bitrot command type  'man gluster'"
-        },
-        { NULL, NULL, NULL }
-};
+    {"volume profile <VOLNAME> {start|info [peek|incremental "
+     "[peek]|cumulative|clear]|stop} [nfs]",
+     cli_cmd_volume_profile_cbk, "volume profile operations"},
+
+    {"volume top <VOLNAME> {open|read|write|opendir|readdir|clear} [nfs|brick "
+     "<brick>] [list-cnt <value>] | "
+     "{read-perf|write-perf} [bs <size> count <count>] "
+     "[brick <brick>] [list-cnt <value>]",
+     cli_cmd_volume_top_cbk, "volume top operations"},
+
+    {"volume status [all | <VOLNAME> [nfs|shd|<BRICK>|quotad]]"
+     " [detail|clients|mem|inode|fd|callpool|tasks|client-list]",
+     cli_cmd_volume_status_cbk,
+     "display status of all or specified volume(s)/brick"},
+
+    {"volume heal <VOLNAME> [enable | disable | full |"
+     "statistics [heal-count [replica <HOSTNAME:BRICKNAME>]] |"
+     "info [summary | split-brain] |"
+     "split-brain {bigger-file <FILE> | latest-mtime <FILE> |"
+     "source-brick <HOSTNAME:BRICKNAME> [<FILE>]} |"
+     "granular-entry-heal {enable | disable}]",
+     cli_cmd_volume_heal_cbk,
+     "self-heal commands on volume specified by <VOLNAME>"},
+
+    {"volume statedump <VOLNAME> [[nfs|quotad] [all|mem|iobuf|callpool|"
+     "priv|fd|inode|history]... | [client <hostname:process-id>]]",
+     cli_cmd_volume_statedump_cbk, "perform statedump on bricks"},
+
+    {"volume list", cli_cmd_volume_list_cbk, "list all volumes in cluster"},
+
+    {"volume clear-locks <VOLNAME> <path> kind {blocked|granted|all}"
+     "{inode [range]|entry [basename]|posix [range]}",
+     cli_cmd_volume_clearlocks_cbk, "Clear locks held on path"},
+    {"volume barrier <VOLNAME> {enable|disable}", cli_cmd_volume_barrier_cbk,
+     "Barrier/unbarrier file operations on a volume"},
+    {"volume get <VOLNAME|all> <key|all>", cli_cmd_volume_getopt_cbk,
+     "Get the value of the all options or given option for volume <VOLNAME>"
+     " or all option. gluster volume get all all is to get all global "
+     "options"},
+
+    {"volume reset-brick <VOLNAME> <SOURCE-BRICK> {{start} |"
+     " {<NEW-BRICK> commit}}",
+     cli_cmd_volume_reset_brick_cbk, "reset-brick operations"},
+
+    {NULL, NULL, NULL}};
 
 int
-cli_cmd_volume_help_cbk (struct cli_state *state, struct cli_cmd_word *in_word,
-                      const char **words, int wordcount)
+cli_cmd_quota_help_cbk(struct cli_state *state, struct cli_cmd_word *in_word,
+                       const char **words, int wordcount)
 {
-        struct cli_cmd        *cmd     = NULL;
-        struct cli_cmd        *vol_cmd = NULL;
-        int                   count    = 0;
+    struct cli_cmd *cmd = NULL;
+    struct cli_cmd *quota_cmd = NULL;
+    int count = 0;
+
+    cmd = GF_MALLOC(sizeof(quota_cmds), cli_mt_cli_cmd);
+    memcpy(cmd, quota_cmds, sizeof(quota_cmds));
+    count = (sizeof(quota_cmds) / sizeof(struct cli_cmd));
+    cli_cmd_sort(cmd, count);
 
-        cmd = GF_CALLOC (1, sizeof (volume_cmds), cli_mt_cli_cmd);
-        memcpy (cmd, volume_cmds, sizeof (volume_cmds));
-        count = (sizeof (volume_cmds) / sizeof (struct cli_cmd));
-        cli_cmd_sort (cmd, count);
+    cli_out("\ngluster quota commands");
+    cli_out("=======================\n");
 
-        for (vol_cmd = cmd; vol_cmd->pattern; vol_cmd++)
-                if (_gf_false == vol_cmd->disable)
-                        cli_out ("%s - %s", vol_cmd->pattern, vol_cmd->desc);
+    for (quota_cmd = cmd; quota_cmd->pattern; quota_cmd++)
+        if ((_gf_false == quota_cmd->disable) && (quota_cmd->desc))
+            cli_out("%s - %s", quota_cmd->pattern, quota_cmd->desc);
 
-        GF_FREE (cmd);
-        return 0;
+    cli_out("\n");
+    GF_FREE(cmd);
+
+    return 0;
 }
 
 int
-cli_cmd_volume_register (struct cli_state *state)
+cli_cmd_bitrot_help_cbk(struct cli_state *state, struct cli_cmd_word *in_word,
+                        const char **words, int wordcount)
 {
-        int  ret = 0;
-        struct cli_cmd *cmd = NULL;
+    struct cli_cmd *cmd = NULL;
+    struct cli_cmd *bitrot_cmd = NULL;
+    int count = 0;
 
-        for (cmd = volume_cmds; cmd->pattern; cmd++) {
+    cmd = GF_MALLOC(sizeof(bitrot_cmds), cli_mt_cli_cmd);
+    memcpy(cmd, bitrot_cmds, sizeof(bitrot_cmds));
+    count = (sizeof(bitrot_cmds) / sizeof(struct cli_cmd));
+    cli_cmd_sort(cmd, count);
+
+    cli_out("\ngluster bitrot commands");
+    cli_out("========================\n");
+
+    for (bitrot_cmd = cmd; bitrot_cmd->pattern; bitrot_cmd++)
+        if ((_gf_false == bitrot_cmd->disable) && (bitrot_cmd->desc))
+            cli_out("%s - %s", bitrot_cmd->pattern, bitrot_cmd->desc);
+
+    cli_out("\n");
+    GF_FREE(cmd);
+
+    return 0;
+}
+
+int
+cli_cmd_volume_help_cbk(struct cli_state *state, struct cli_cmd_word *in_word,
+                        const char **words, int wordcount)
+{
+    struct cli_cmd *cmd = NULL;
+    struct cli_cmd *vol_cmd = NULL;
+    int count = 0;
+
+    cmd = GF_MALLOC(sizeof(volume_cmds), cli_mt_cli_cmd);
+    memcpy(cmd, volume_cmds, sizeof(volume_cmds));
+    count = (sizeof(volume_cmds) / sizeof(struct cli_cmd));
+    cli_cmd_sort(cmd, count);
+
+    cli_out("\ngluster volume commands");
+    cli_out("========================\n");
+
+    for (vol_cmd = cmd; vol_cmd->pattern; vol_cmd++)
+        if (_gf_false == vol_cmd->disable)
+            cli_out("%s - %s", vol_cmd->pattern, vol_cmd->desc);
+
+    cli_out("\n");
+    GF_FREE(cmd);
+    return 0;
+}
+
+int
+cli_cmd_volume_register(struct cli_state *state)
+{
+    int ret = 0;
+    struct cli_cmd *cmd = NULL;
+
+    for (cmd = volume_cmds; cmd->pattern; cmd++) {
+        ret = cli_cmd_register(&state->tree, cmd);
+        if (ret)
+            goto out;
+    }
+
+    for (cmd = bitrot_cmds; cmd->pattern; cmd++) {
+        ret = cli_cmd_register(&state->tree, cmd);
+        if (ret)
+            goto out;
+    }
+
+    for (cmd = quota_cmds; cmd->pattern; cmd++) {
+        ret = cli_cmd_register(&state->tree, cmd);
+        if (ret)
+            goto out;
+    }
 
-                ret = cli_cmd_register (&state->tree, cmd);
-                if (ret)
-                        goto out;
-        }
 out:
-        return ret;
+    return ret;
+}
+
+static int
+gf_asprintf_append(char **string_ptr, const char *format, ...)
+{
+    va_list arg;
+    int rv = 0;
+    char *tmp = *string_ptr;
+
+    va_start(arg, format);
+    rv = gf_vasprintf(string_ptr, format, arg);
+    va_end(arg);
+
+    if (tmp)
+        GF_FREE(tmp);
+
+    return rv;
 }
diff --git a/cli/src/cli-cmd.c b/cli/src/cli-cmd.c
index a1e4e909a0d..2d458b16a56 100644
--- a/cli/src/cli-cmd.c
+++ b/cli/src/cli-cmd.c
@@ -22,396 +22,390 @@
 
 static int cmd_done;
 static int cmd_sent;
-static pthread_cond_t      cond  = PTHREAD_COND_INITIALIZER;
-static pthread_mutex_t     cond_mutex = PTHREAD_MUTEX_INITIALIZER;
-static pthread_cond_t      conn  = PTHREAD_COND_INITIALIZER;
-static pthread_mutex_t     conn_mutex = PTHREAD_MUTEX_INITIALIZER;
+static pthread_cond_t cond = PTHREAD_COND_INITIALIZER;
+static pthread_mutex_t cond_mutex = PTHREAD_MUTEX_INITIALIZER;
+static pthread_cond_t conn = PTHREAD_COND_INITIALIZER;
+static pthread_mutex_t conn_mutex = PTHREAD_MUTEX_INITIALIZER;
 
-int    cli_op_ret = 0;
-int    connected = 0;
-
-int cli_cmd_log_help_cbk (struct cli_state *state, struct cli_cmd_word *in_word,
-                      const char **words, int wordcount);
+int cli_op_ret = 0;
+static gf_boolean_t connected = _gf_false;
 
 static unsigned
-cli_cmd_needs_connection (struct cli_cmd_word *word)
+cli_cmd_needs_connection(struct cli_cmd_word *word)
 {
-        if (!strcasecmp ("quit", word->word))
-                return 0;
+    if (!strcasecmp("quit", word->word))
+        return 0;
 
-        if (!strcasecmp ("help", word->word))
-                return 0;
+    if (!strcasecmp("help", word->word))
+        return 0;
 
-        if (!strcasecmp ("getwd", word->word))
-                return 1;
+    if (!strcasecmp("getwd", word->word))
+        return 1;
 
-        if (!strcasecmp ("exit", word->word))
-                return 0;
+    if (!strcasecmp("exit", word->word))
+        return 0;
 
-        return CLI_DEFAULT_CONN_TIMEOUT;
+    return cli_default_conn_timeout;
 }
 
 int
-cli_cmd_status_reset (void)
+cli_cmd_status_reset(void)
 {
-        int ret = 0;
-
-        ret = cli_cmd_lock ();
-        {
-                if (ret == 0) {
-                        cmd_sent = 0;
-                        cmd_done = 0;
-                }
-        }
-        ret = cli_cmd_unlock ();
-        return ret;
+    int ret = 0;
 
+    ret = cli_cmd_lock();
+    {
+        if (ret == 0) {
+            cmd_sent = 0;
+            cmd_done = 0;
+        }
+    }
+    ret = cli_cmd_unlock();
+    return ret;
 }
 
 int
-cli_cmd_sent_status_get (int *status)
+cli_cmd_sent_status_get(int *status)
 {
-        int ret = 0;
-        GF_ASSERT (status);
-
-        ret = cli_cmd_lock ();
-        {
-                if (ret == 0)
-                        *status = cmd_sent;
-        }
-        ret = cli_cmd_unlock ();
-        return ret;
+    int ret = 0;
+    GF_ASSERT(status);
+
+    ret = cli_cmd_lock();
+    {
+        if (ret == 0)
+            *status = cmd_sent;
+    }
+    ret = cli_cmd_unlock();
+    return ret;
 }
 
 int
-cli_cmd_process (struct cli_state *state, int argc, char **argv)
+cli_cmd_process(struct cli_state *state, int argc, char **argv)
 {
-        int                  ret = 0;
-        struct cli_cmd_word *word = NULL;
-        struct cli_cmd_word *next = NULL;
-        int                  i = 0;
+    int ret = 0;
+    struct cli_cmd_word *word = NULL;
+    struct cli_cmd_word *next = NULL;
+    int i = 0;
 
-        word = &state->tree.root;
+    word = &state->tree.root;
 
-        if (!argc)
-                return 0;
+    if (!argc)
+        return 0;
 
-        for (i = 0; i < argc; i++) {
-                next = cli_cmd_nextword (word, argv[i]);
+    for (i = 0; i < argc; i++) {
+        next = cli_cmd_nextword(word, argv[i]);
 
-                word = next;
-                if (!word)
-                        break;
+        word = next;
+        if (!word)
+            break;
 
-                if (word->cbkfn)
-                        break;
-        }
+        if (word->cbkfn)
+            break;
+    }
 
-        if (!word) {
-                cli_out ("unrecognized word: %s (position %d)",
-                         argv[i], i);
-                return -1;
-        }
+    if (!word) {
+        cli_out("unrecognized word: %s (position %d)\n", argv[i], i);
+        usage();
+        return -1;
+    }
 
-        if (!word->cbkfn) {
-                cli_out ("unrecognized command");
-                return -1;
-        }
+    if (!word->cbkfn) {
+        cli_out("unrecognized command\n");
+        usage();
+        return -1;
+    }
 
-	if ( strcmp (word->word,"help")==0 )
-		goto callback;
+    if (strcmp(word->word, "help") == 0)
+        goto callback;
 
-        state->await_connected = cli_cmd_needs_connection (word);
+    state->await_connected = cli_cmd_needs_connection(word);
 
-        ret = cli_cmd_await_connected (state->await_connected);
-        if (ret) {
-                cli_out ("Connection failed. Please check if gluster "
-                          "daemon is operational.");
-                gf_log ("", GF_LOG_INFO, "Exiting with: %d", ret);
-                exit (ret);
-        }
+    ret = cli_cmd_await_connected(state->await_connected);
+    if (ret) {
+        cli_out(
+            "Connection failed. Please check if gluster "
+            "daemon is operational.");
+        gf_log("", GF_LOG_INFO, "Exiting with: %d", ret);
+        exit(ret);
+    }
 
 callback:
-        ret = word->cbkfn (state, word, (const char **)argv, argc);
-        (void) cli_cmd_status_reset ();
-        return ret;
+    ret = word->cbkfn(state, word, (const char **)argv, argc);
+    (void)cli_cmd_status_reset();
+    return ret;
 }
 
 int
-cli_cmd_input_token_count (const char *text)
+cli_cmd_input_token_count(const char *text)
 {
-        int          count = 0;
-        const char  *trav = NULL;
-        int          is_spc = 1;
-
-        for (trav = text; *trav; trav++) {
-                if (*trav == ' ') {
-                        is_spc = 1;
-                } else {
-                        if (is_spc) {
-                                count++;
-                                is_spc = 0;
-                        }
-                }
+    int count = 0;
+    const char *trav = NULL;
+    int is_spc = 1;
+
+    for (trav = text; *trav; trav++) {
+        if (*trav == ' ') {
+            is_spc = 1;
+        } else {
+            if (is_spc) {
+                count++;
+                is_spc = 0;
+            }
         }
+    }
 
-        return count;
+    return count;
 }
 
-
 int
-cli_cmd_process_line (struct cli_state *state, const char *text)
+cli_cmd_process_line(struct cli_state *state, const char *text)
 {
-        int     count = 0;
-        char  **tokens = NULL;
-        char  **tokenp = NULL;
-        char   *token = NULL;
-        char   *copy = NULL;
-        char   *saveptr = NULL;
-        int     i = 0;
-        int     ret = -1;
-
-        count = cli_cmd_input_token_count (text);
-
-        tokens = calloc (count + 1, sizeof (*tokens));
-        if (!tokens)
-                return -1;
-
-        copy = strdup (text);
-        if (!copy)
-                goto out;
-
-        tokenp = tokens;
-
-        for (token = strtok_r (copy, " \t\r\n", &saveptr); token;
-             token = strtok_r (NULL, " \t\r\n", &saveptr)) {
-                *tokenp = strdup (token);
-
-                if (!*tokenp)
-                        goto out;
-                tokenp++;
-                i++;
-
-        }
-
-        ret = cli_cmd_process (state, count, tokens);
+    int count = 0;
+    char **tokens = NULL;
+    char **tokenp = NULL;
+    char *token = NULL;
+    char *copy = NULL;
+    char *saveptr = NULL;
+    int i = 0;
+    int ret = -1;
+
+    count = cli_cmd_input_token_count(text);
+
+    tokens = calloc(count + 1, sizeof(*tokens));
+    if (!tokens)
+        return -1;
+
+    copy = strdup(text);
+    if (!copy)
+        goto out;
+
+    tokenp = tokens;
+
+    for (token = strtok_r(copy, " \t\r\n", &saveptr); token;
+         token = strtok_r(NULL, " \t\r\n", &saveptr)) {
+        *tokenp = strdup(token);
+
+        if (!*tokenp)
+            goto out;
+        tokenp++;
+        i++;
+    }
+
+    ret = cli_cmd_process(state, count, tokens);
 out:
-        free (copy);
+    free(copy);
 
-        if (tokens)
-                cli_cmd_tokens_destroy (tokens);
+    if (tokens)
+        cli_cmd_tokens_destroy(tokens);
 
-        return ret;
+    return ret;
 }
 
-
 int
-cli_cmds_register (struct cli_state *state)
+cli_cmds_register(struct cli_state *state)
 {
-        int  ret = 0;
-
-        ret = cli_cmd_volume_register (state);
-        if (ret)
-                goto out;
-
-        ret = cli_cmd_probe_register (state);
-        if (ret)
-                goto out;
-
-        ret = cli_cmd_system_register (state);
-        if (ret)
-                goto out;
-
-        ret = cli_cmd_misc_register (state);
-        if (ret)
-                goto out;
-
-        ret = cli_cmd_snapshot_register (state);
-        if (ret)
-                goto out;
-        ret = cli_cmd_global_register (state);
-        if (ret)
-                goto out;
+    int ret = 0;
+
+    ret = cli_cmd_volume_register(state);
+    if (ret)
+        goto out;
+
+    ret = cli_cmd_probe_register(state);
+    if (ret)
+        goto out;
+
+    ret = cli_cmd_system_register(state);
+    if (ret)
+        goto out;
+
+    ret = cli_cmd_misc_register(state);
+    if (ret)
+        goto out;
+
+    ret = cli_cmd_snapshot_register(state);
+    if (ret)
+        goto out;
+    ret = cli_cmd_global_register(state);
+    if (ret)
+        goto out;
 out:
-        return ret;
-}
-
-int
-cli_cmd_cond_init ()
-{
-
-       pthread_mutex_init (&cond_mutex, NULL);
-       pthread_cond_init (&cond, NULL);
-
-       pthread_mutex_init (&conn_mutex, NULL);
-       pthread_cond_init (&conn, NULL);
-
-       return 0;
+    return ret;
 }
 
 int
-cli_cmd_lock ()
+cli_cmd_lock()
 {
-       pthread_mutex_lock (&cond_mutex);
-       return 0;
+    pthread_mutex_lock(&cond_mutex);
+    return 0;
 }
 
 int
-cli_cmd_unlock ()
+cli_cmd_unlock()
 {
-        pthread_mutex_unlock (&cond_mutex);
-        return 0;
+    pthread_mutex_unlock(&cond_mutex);
+    return 0;
 }
 
 static void
-seconds_from_now (unsigned secs, struct timespec *ts)
+seconds_from_now(unsigned secs, struct timespec *ts)
 {
-        struct timeval tv = {0,};
+    struct timeval tv = {
+        0,
+    };
 
-        gettimeofday (&tv, NULL);
+    gettimeofday(&tv, NULL);
 
-        ts->tv_sec = tv.tv_sec + secs;
-        ts->tv_nsec = tv.tv_usec * 1000;
+    ts->tv_sec = tv.tv_sec + secs;
+    ts->tv_nsec = tv.tv_usec * 1000;
 }
 
 int
-cli_cmd_await_response (unsigned time)
+cli_cmd_await_response(unsigned time)
 {
-        struct  timespec        ts = {0,};
-        int                     ret = 0;
+    struct timespec ts = {
+        0,
+    };
+    int ret = 0;
 
-        cli_op_ret = -1;
+    cli_op_ret = -1;
 
-        seconds_from_now (time, &ts);
-        while (!cmd_done && !ret) {
-                ret = pthread_cond_timedwait (&cond, &cond_mutex,
-                                        &ts);
-        }
+    seconds_from_now(time, &ts);
+    while (!cmd_done && !ret) {
+        ret = pthread_cond_timedwait(&cond, &cond_mutex, &ts);
+    }
 
-        if (!cmd_done) {
-                if (ret == ETIMEDOUT)
-                        cli_out ("Error : Request timed out");
-                else
-                        cli_out ("Error : Command returned with error code:%d",
-                                 ret);
-        }
-        cmd_done = 0;
+    if (!cmd_done) {
+        if (ret == ETIMEDOUT)
+            cli_out("Error : Request timed out");
+        else
+            cli_out("Error : Command returned with error code:%d", ret);
+    }
+    cmd_done = 0;
 
-        return cli_op_ret;
+    return cli_op_ret;
 }
 
 /* This function must be called _only_ after all actions associated with
  * command processing is complete. Otherwise, gluster process may exit before
  * reporting results to stdout/stderr. */
 int
-cli_cmd_broadcast_response (int32_t status)
+cli_cmd_broadcast_response(int32_t status)
 {
-
-        pthread_mutex_lock (&cond_mutex);
-        {
-                if (!cmd_sent)
-                        goto out;
-                cmd_done = 1;
-                cli_op_ret = status;
-                pthread_cond_broadcast (&cond);
-        }
-
+    pthread_mutex_lock(&cond_mutex);
+    {
+        if (!cmd_sent)
+            goto out;
+        cmd_done = 1;
+        cli_op_ret = status;
+        pthread_cond_broadcast(&cond);
+    }
 
 out:
-        pthread_mutex_unlock (&cond_mutex);
-        return 0;
+    pthread_mutex_unlock(&cond_mutex);
+    return 0;
 }
 
 int32_t
-cli_cmd_await_connected (unsigned conn_timo)
+cli_cmd_await_connected(unsigned conn_timo)
 {
-        int32_t                 ret = 0;
-        struct  timespec        ts = {0,};
-
-        if (!conn_timo)
-                return 0;
-
-        pthread_mutex_lock (&conn_mutex);
-        {
-                seconds_from_now (conn_timo, &ts);
-                while (!connected && !ret) {
-                        ret = pthread_cond_timedwait (&conn, &conn_mutex,
-                                                      &ts);
-                }
-        }
-        pthread_mutex_unlock (&conn_mutex);
+    int32_t ret = 0;
+    struct timespec ts = {
+        0,
+    };
+
+    if (!conn_timo)
+        return 0;
 
+    pthread_mutex_lock(&conn_mutex);
+    {
+        seconds_from_now(conn_timo, &ts);
+        while (!connected && !ret) {
+            ret = pthread_cond_timedwait(&conn, &conn_mutex, &ts);
+        }
+    }
+    pthread_mutex_unlock(&conn_mutex);
 
-        return ret;
+    return ret;
 }
 
 int32_t
-cli_cmd_broadcast_connected ()
+cli_cmd_broadcast_connected(gf_boolean_t status)
 {
-        pthread_mutex_lock (&conn_mutex);
-        {
-                connected = 1;
-                pthread_cond_broadcast (&conn);
-        }
-
-        pthread_mutex_unlock (&conn_mutex);
-
-        return 0;
+    pthread_mutex_lock(&conn_mutex);
+    {
+        connected = status;
+        pthread_cond_broadcast(&conn);
+    }
+    pthread_mutex_unlock(&conn_mutex);
+
+    return 0;
 }
 
-int
-cli_cmd_submit (struct rpc_clnt* rpc, void *req, call_frame_t *frame,
-                rpc_clnt_prog_t *prog,
-                int procnum, struct iobref *iobref,
-                xlator_t *this, fop_cbk_fn_t cbkfn, xdrproc_t xdrproc)
+gf_boolean_t
+cli_cmd_connected(void)
 {
-        int             ret = -1;
-        unsigned        timeout = 0;
-
-        if ((GLUSTER_CLI_PROFILE_VOLUME == procnum) ||
-            (GLUSTER_CLI_HEAL_VOLUME == procnum) ||
-            (GLUSTER_CLI_GANESHA == procnum))
-                timeout = CLI_TEN_MINUTES_TIMEOUT;
-        else
-                timeout = CLI_DEFAULT_CMD_TIMEOUT;
-
-        cli_cmd_lock ();
-        cmd_sent = 0;
-        ret = cli_submit_request (rpc, req, frame, prog,
-                                  procnum, NULL, this, cbkfn, xdrproc);
+    gf_boolean_t status;
 
-        if (!ret) {
-                cmd_sent = 1;
-                ret = cli_cmd_await_response (timeout);
-        }
+    pthread_mutex_lock(&conn_mutex);
+    {
+        status = connected;
+    }
+    pthread_mutex_unlock(&conn_mutex);
 
-        cli_cmd_unlock ();
+    return status;
+}
 
-        gf_log ("cli", GF_LOG_DEBUG, "Returning %d", ret);
-        return ret;
+int
+cli_cmd_submit(struct rpc_clnt *rpc, void *req, call_frame_t *frame,
+               rpc_clnt_prog_t *prog, int procnum, struct iobref *iobref,
+               xlator_t *this, fop_cbk_fn_t cbkfn, xdrproc_t xdrproc)
+{
+    int ret = -1;
+    unsigned timeout = 0;
+
+    if ((GLUSTER_CLI_PROFILE_VOLUME == procnum) ||
+        (GLUSTER_CLI_HEAL_VOLUME == procnum) ||
+        (GLUSTER_CLI_GANESHA == procnum))
+        timeout = cli_ten_minutes_timeout;
+    else
+        timeout = cli_default_conn_timeout;
+
+    cli_cmd_lock();
+    cmd_sent = 0;
+    ret = cli_submit_request(rpc, req, frame, prog, procnum, NULL, this, cbkfn,
+                             xdrproc);
+
+    if (!ret) {
+        cmd_sent = 1;
+        ret = cli_cmd_await_response(timeout);
+    }
+
+    cli_cmd_unlock();
+
+    gf_log("cli", GF_LOG_DEBUG, "Returning %d", ret);
+    return ret;
 }
 
 int
-cli_cmd_pattern_cmp (void *a, void *b)
+cli_cmd_pattern_cmp(void *a, void *b)
 {
-        struct cli_cmd *ia = NULL;
-        struct cli_cmd *ib = NULL;
-        int            ret = 0;
-
-        ia = a;
-        ib = b;
-        if (strcmp (ia->pattern, ib->pattern) > 0)
-                ret = 1;
-        else if (strcmp (ia->pattern, ib->pattern) < 0)
-                ret = -1;
-        else
-                ret = 0;
-        return ret;
+    struct cli_cmd *ia = NULL;
+    struct cli_cmd *ib = NULL;
+    int ret = 0;
+
+    ia = a;
+    ib = b;
+    if (strcmp(ia->pattern, ib->pattern) > 0)
+        ret = 1;
+    else if (strcmp(ia->pattern, ib->pattern) < 0)
+        ret = -1;
+    else
+        ret = 0;
+    return ret;
 }
 
 void
-cli_cmd_sort (struct cli_cmd *cmd, int count)
+cli_cmd_sort(struct cli_cmd *cmd, int count)
 {
-        gf_array_insertionsort (cmd, 1, count - 2, sizeof(struct cli_cmd),
-                                cli_cmd_pattern_cmp);
+    gf_array_insertionsort(cmd, 1, count - 2, sizeof(struct cli_cmd),
+                           cli_cmd_pattern_cmp);
 }
diff --git a/cli/src/cli-cmd.h b/cli/src/cli-cmd.h
index 2f96bdda2ff..c1c068c7085 100644
--- a/cli/src/cli-cmd.h
+++ b/cli/src/cli-cmd.h
@@ -13,110 +13,117 @@
 #include <netdb.h>
 
 #include "cli.h"
-#include "list.h"
-
-#define GLUSTER_SHARED_STORAGE      "gluster_shared_storage"
-
-#define CLI_LOCAL_INIT(local, words, frame, dictionary) \
-        do {                                                 \
-                local = cli_local_get ();                    \
-                                                             \
-                if (local) {                                 \
-                        local->words = words;                \
-                        if (dictionary)                      \
-                                local->dict = dictionary;    \
-                        if (frame)                           \
-                                frame->local = local;        \
-                }                                            \
-        } while (0)
-
-#define CLI_STACK_DESTROY(_frame)                                       \
-        do {                                                            \
-                if (_frame) {                                           \
-                        if (_frame->local) {                            \
-                                gf_log ("cli", GF_LOG_DEBUG, "frame->local " \
-                                        "is not NULL (%p)", _frame->local); \
-                                cli_local_wipe (_frame->local);         \
-                                _frame->local = NULL;                   \
-                        }                                               \
-                        STACK_DESTROY (_frame->root);                   \
-                }                                                       \
-        } while (0);
-
-typedef enum {
-        GF_ANSWER_YES = 1,
-        GF_ANSWER_NO  = 2
-} gf_answer_t;
+#include <glusterfs/list.h>
+
+#define GLUSTER_SHARED_STORAGE "gluster_shared_storage"
+
+#define CLI_LOCAL_INIT(local, words, frame, dictionary)                        \
+    do {                                                                       \
+        local = cli_local_get();                                               \
+                                                                               \
+        if (local) {                                                           \
+            local->words = words;                                              \
+            if (dictionary)                                                    \
+                local->dict = dictionary;                                      \
+            if (frame)                                                         \
+                frame->local = local;                                          \
+        }                                                                      \
+    } while (0)
+
+#define CLI_STACK_DESTROY(_frame)                                              \
+    do {                                                                       \
+        if (_frame) {                                                          \
+            if (_frame->local) {                                               \
+                gf_log("cli", GF_LOG_DEBUG,                                    \
+                       "frame->local "                                         \
+                       "is not NULL (%p)",                                     \
+                       _frame->local);                                         \
+                cli_local_wipe(_frame->local);                                 \
+                _frame->local = NULL;                                          \
+            }                                                                  \
+            STACK_DESTROY(_frame->root);                                       \
+        }                                                                      \
+    } while (0);
+
+typedef enum { GF_ANSWER_YES = 1, GF_ANSWER_NO = 2 } gf_answer_t;
 
 struct cli_cmd {
-        const char         *pattern;
-        cli_cmd_cbk_t      *cbk;
-        const char         *desc;
-        cli_cmd_reg_cbk_t  *reg_cbk; /* callback to check in runtime if the   *
-                                      * command should be enabled or disabled */
-        gf_boolean_t       disable;
+    const char *pattern;
+    cli_cmd_cbk_t *cbk;
+    const char *desc;
+    cli_cmd_reg_cbk_t *reg_cbk; /* callback to check in runtime if the   *
+                                 * command should be enabled or disabled */
+    gf_boolean_t disable;
 };
 
 struct cli_cmd_volume_get_ctx_ {
-        char            *volname;
-        int             flags;
+    char *volname;
+    int flags;
 };
 
 typedef struct cli_profile_info_ {
-        uint64_t fop_hits;
-        double min_latency;
-        double max_latency;
-        double avg_latency;
-        char   *fop_name;
-        double percentage_avg_latency;
+    uint64_t fop_hits;
+    double min_latency;
+    double max_latency;
+    double avg_latency;
+    char *fop_name;
+    double percentage_avg_latency;
 } cli_profile_info_t;
 
 typedef struct cli_cmd_volume_get_ctx_ cli_cmd_volume_get_ctx_t;
 
-int cli_cmd_volume_register (struct cli_state *state);
-
-int cli_cmd_probe_register (struct cli_state *state);
+int
+cli_cmd_volume_register(struct cli_state *state);
 
-int cli_cmd_system_register (struct cli_state *state);
+int
+cli_cmd_probe_register(struct cli_state *state);
 
-int cli_cmd_snapshot_register (struct cli_state *state);
+int
+cli_cmd_system_register(struct cli_state *state);
 
-int cli_cmd_global_register (struct cli_state *state);
+int
+cli_cmd_snapshot_register(struct cli_state *state);
 
-int cli_cmd_misc_register (struct cli_state *state);
+int
+cli_cmd_global_register(struct cli_state *state);
 
-struct cli_cmd_word *cli_cmd_nextword (struct cli_cmd_word *word,
-                                       const char *text);
-void cli_cmd_tokens_destroy (char **tokens);
+int
+cli_cmd_misc_register(struct cli_state *state);
 
-int cli_cmd_await_response (unsigned time);
+struct cli_cmd_word *
+cli_cmd_nextword(struct cli_cmd_word *word, const char *text);
+void
+cli_cmd_tokens_destroy(char **tokens);
 
-int cli_cmd_broadcast_response (int32_t status);
+int
+cli_cmd_await_response(unsigned time);
 
-int cli_cmd_cond_init ();
+int
+cli_cmd_broadcast_response(int32_t status);
 
-int cli_cmd_lock ();
+int
+cli_cmd_lock();
 
-int cli_cmd_unlock ();
+int
+cli_cmd_unlock();
 
 int
-cli_cmd_submit (struct rpc_clnt *rpc, void *req, call_frame_t *frame,
-                rpc_clnt_prog_t *prog,
-                int procnum, struct iobref *iobref,
-                xlator_t *this, fop_cbk_fn_t cbkfn, xdrproc_t xdrproc);
+cli_cmd_submit(struct rpc_clnt *rpc, void *req, call_frame_t *frame,
+               rpc_clnt_prog_t *prog, int procnum, struct iobref *iobref,
+               xlator_t *this, fop_cbk_fn_t cbkfn, xdrproc_t xdrproc);
 
-int cli_cmd_pattern_cmp (void *a, void *b);
+int
+cli_cmd_pattern_cmp(void *a, void *b);
 
-void cli_cmd_sort (struct cli_cmd *cmd, int count);
+void
+cli_cmd_sort(struct cli_cmd *cmd, int count);
 
 gf_answer_t
-cli_cmd_get_confirmation (struct cli_state *state, const char *question);
-int cli_cmd_sent_status_get (int *status);
-
-gf_boolean_t
-_limits_set_on_volume (char *volname, int type);
+cli_cmd_get_confirmation(struct cli_state *state, const char *question);
+int
+cli_cmd_sent_status_get(int *status);
 
 gf_boolean_t
-_quota_aux_mount_online (char *volname);
+_limits_set_on_volume(char *volname, int type);
 
 #endif /* __CLI_CMD_H__ */
diff --git a/cli/src/cli-mem-types.h b/cli/src/cli-mem-types.h
index 5468b25cc0c..b42b4dd86c2 100644
--- a/cli/src/cli-mem-types.h
+++ b/cli/src/cli-mem-types.h
@@ -10,21 +10,21 @@
 #ifndef __CLI_MEM_TYPES_H__
 #define __CLI_MEM_TYPES_H__
 
-#include "mem-types.h"
+#include <glusterfs/mem-types.h>
 
 #define CLI_MEM_TYPE_START (gf_common_mt_end + 1)
 
 enum cli_mem_types_ {
-        cli_mt_xlator_list_t = CLI_MEM_TYPE_START,
-        cli_mt_xlator_t,
-        cli_mt_xlator_cmdline_option_t,
-        cli_mt_char,
-        cli_mt_call_pool_t,
-        cli_mt_cli_local_t,
-        cli_mt_cli_get_vol_ctx_t,
-        cli_mt_append_str,
-        cli_mt_cli_cmd,
-        cli_mt_end
+    cli_mt_xlator_list_t = CLI_MEM_TYPE_START,
+    cli_mt_xlator_t,
+    cli_mt_xlator_cmdline_option_t,
+    cli_mt_char,
+    cli_mt_call_pool_t,
+    cli_mt_cli_local_t,
+    cli_mt_cli_get_vol_ctx_t,
+    cli_mt_append_str,
+    cli_mt_cli_cmd,
+    cli_mt_end
 
 };
 
diff --git a/cli/src/cli-quotad-client.c b/cli/src/cli-quotad-client.c
index 5be9c80c858..772b8f75bd9 100644
--- a/cli/src/cli-quotad-client.c
+++ b/cli/src/cli-quotad-client.c
@@ -10,140 +10,137 @@
 
 #include "cli-quotad-client.h"
 
-extern struct rpc_clnt global_quotad_rpc;
-extern struct rpc_clnt_program cli_quotad_clnt;
-
 int
-cli_quotad_submit_request (void *req, call_frame_t *frame,
-                           rpc_clnt_prog_t *prog,
-                           int procnum, struct iobref *iobref,
-                           xlator_t *this, fop_cbk_fn_t cbkfn,
-                           xdrproc_t xdrproc)
+cli_quotad_submit_request(void *req, call_frame_t *frame, rpc_clnt_prog_t *prog,
+                          int procnum, struct iobref *iobref, xlator_t *this,
+                          fop_cbk_fn_t cbkfn, xdrproc_t xdrproc)
 {
-        int           ret        = -1;
-        int           count      = 0;
-        struct iovec  iov        = {0, };
-        struct iobuf *iobuf      = NULL;
-        char          new_iobref = 0;
-        ssize_t       xdr_size   = 0;
-
-        GF_ASSERT (this);
-
-        if (req) {
-                xdr_size = xdr_sizeof (xdrproc, req);
-                iobuf = iobuf_get2 (this->ctx->iobuf_pool, xdr_size);
-                if (!iobuf) {
-                        goto out;
-                };
-
-                if (!iobref) {
-                        iobref = iobref_new ();
-                        if (!iobref) {
-                                goto out;
-                        }
-
-                        new_iobref = 1;
-                }
-
-                iobref_add (iobref, iobuf);
-
-                iov.iov_base = iobuf->ptr;
-                iov.iov_len  = iobuf_size (iobuf);
-
-                /* Create the xdr payload */
-                ret = xdr_serialize_generic (iov, req, xdrproc);
-                if (ret == -1) {
-                        goto out;
-                }
-                iov.iov_len = ret;
-                count = 1;
+    int ret = -1;
+    int count = 0;
+    struct iovec iov = {
+        0,
+    };
+    struct iobuf *iobuf = NULL;
+    char new_iobref = 0;
+    ssize_t xdr_size = 0;
+
+    GF_ASSERT(this);
+
+    if (req) {
+        xdr_size = xdr_sizeof(xdrproc, req);
+        iobuf = iobuf_get2(this->ctx->iobuf_pool, xdr_size);
+        if (!iobuf) {
+            goto out;
+        };
+
+        if (!iobref) {
+            iobref = iobref_new();
+            if (!iobref) {
+                goto out;
+            }
+
+            new_iobref = 1;
+        }
+
+        iobref_add(iobref, iobuf);
+
+        iov.iov_base = iobuf->ptr;
+        iov.iov_len = iobuf_size(iobuf);
+
+        /* Create the xdr payload */
+        ret = xdr_serialize_generic(iov, req, xdrproc);
+        if (ret == -1) {
+            goto out;
         }
+        iov.iov_len = ret;
+        count = 1;
+    }
 
-        /* Send the msg */
-        ret = rpc_clnt_submit (&global_quotad_rpc, prog, procnum, cbkfn,
-                               &iov, count,
-                               NULL, 0, iobref, frame, NULL, 0, NULL, 0, NULL);
-        ret = 0;
+    /* Send the msg */
+    ret = rpc_clnt_submit(global_quotad_rpc, prog, procnum, cbkfn, &iov, count,
+                          NULL, 0, iobref, frame, NULL, 0, NULL, 0, NULL);
+    ret = 0;
 
 out:
-        if (new_iobref)
-                iobref_unref (iobref);
-        if (iobuf)
-                iobuf_unref (iobuf);
+    if (new_iobref)
+        iobref_unref(iobref);
+    if (iobuf)
+        iobuf_unref(iobuf);
 
-        return ret;
+    return ret;
 }
 
 int
-cli_quotad_notify (struct rpc_clnt *rpc, void *mydata,
-                   rpc_clnt_event_t event, void *data)
+cli_quotad_notify(struct rpc_clnt *rpc, void *mydata, rpc_clnt_event_t event,
+                  void *data)
 {
-        xlator_t                *this = NULL;
-        int                     ret = 0;
+    xlator_t *this = NULL;
+    int ret = 0;
 
-        this = mydata;
+    this = mydata;
 
-        switch (event) {
-        case RPC_CLNT_CONNECT:
-        {
-                gf_log (this->name, GF_LOG_TRACE, "got RPC_CLNT_CONNECT");
-                break;
+    switch (event) {
+        case RPC_CLNT_CONNECT: {
+            gf_log(this->name, GF_LOG_TRACE, "got RPC_CLNT_CONNECT");
+            break;
         }
 
-        case RPC_CLNT_DISCONNECT:
-        {
-                gf_log (this->name, GF_LOG_TRACE, "got RPC_CLNT_DISCONNECT");
-                break;
+        case RPC_CLNT_DISCONNECT: {
+            gf_log(this->name, GF_LOG_TRACE, "got RPC_CLNT_DISCONNECT");
+            break;
         }
 
         default:
-                gf_log (this->name, GF_LOG_TRACE,
-                        "got some other RPC event %d", event);
-                ret = 0;
-                break;
-        }
+            gf_log(this->name, GF_LOG_TRACE, "got some other RPC event %d",
+                   event);
+            ret = 0;
+            break;
+    }
 
-        return ret;
+    return ret;
 }
 
 struct rpc_clnt *
-cli_quotad_clnt_init (xlator_t *this, dict_t *options)
+cli_quotad_clnt_init(xlator_t *this, dict_t *options)
 {
-        struct rpc_clnt *rpc  = NULL;
-        int              ret  = -1;
-
-
-        ret = dict_set_str (options, "transport.address-family", "unix");
-        if (ret)
-                goto out;
-
-        ret = dict_set_str (options, "transport-type", "socket");
-        if (ret)
-                goto out;
-
-        ret = dict_set_str (options, "transport.socket.connect-path",
-                            "/var/run/gluster/quotad.socket");
-        if (ret)
-                goto out;
-
-        rpc = rpc_clnt_new (options, this, this->name, 16);
-        if (!rpc)
-                goto out;
-
-        ret = rpc_clnt_register_notify (rpc, cli_quotad_notify, this);
-        if (ret) {
-                gf_log ("cli", GF_LOG_ERROR, "failed to register notify");
-                goto out;
-        }
-
-        rpc_clnt_start (rpc);
+    struct rpc_clnt *rpc = NULL;
+    int ret = -1;
+
+    ret = dict_set_nstrn(options, "transport.address-family",
+                         SLEN("transport.address-family"), "unix",
+                         SLEN("unix"));
+    if (ret)
+        goto out;
+
+    ret = dict_set_nstrn(options, "transport-type", SLEN("transport-type"),
+                         "socket", SLEN("socket"));
+    if (ret)
+        goto out;
+
+    ret = dict_set_nstrn(options, "transport.socket.connect-path",
+                         SLEN("transport.socket.connect-path"),
+                         "/var/run/gluster/quotad.socket",
+                         SLEN("/var/run/gluster/quotad.socket"));
+    if (ret)
+        goto out;
+
+    rpc = rpc_clnt_new(options, this, this->name, 16);
+    if (!rpc)
+        goto out;
+
+    ret = rpc_clnt_register_notify(rpc, cli_quotad_notify, this);
+    if (ret) {
+        gf_log("cli", GF_LOG_ERROR, "failed to register notify");
+        goto out;
+    }
+
+    rpc_clnt_start(rpc);
 out:
-        if (ret) {
-                if (rpc)
-                        rpc_clnt_unref (rpc);
-                rpc = NULL;
-        }
+    if (ret) {
+        if (rpc)
+            rpc_clnt_unref(rpc);
+        rpc = NULL;
+    }
 
-        return rpc;
+    return rpc;
 }
-
diff --git a/cli/src/cli-quotad-client.h b/cli/src/cli-quotad-client.h
index aa0b42af38d..71a44e5916b 100644
--- a/cli/src/cli-quotad-client.h
+++ b/cli/src/cli-quotad-client.h
@@ -8,26 +8,22 @@
    cases as published by the Free Software Foundation.
 */
 #include "cli.h"
-#include "compat-errno.h"
-#include "compat.h"
+#include <glusterfs/compat-errno.h>
+#include <glusterfs/compat.h>
 #include "cli-cmd.h"
 #include "cli1-xdr.h"
 #include "xdr-generic.h"
 #include "protocol-common.h"
 #include "cli-mem-types.h"
 
-
 int
-cli_quotad_submit_request (void *req, call_frame_t *frame,
-                           rpc_clnt_prog_t *prog,
-                           int procnum, struct iobref *iobref,
-                           xlator_t *this, fop_cbk_fn_t cbkfn,
-                           xdrproc_t xdrproc);
+cli_quotad_submit_request(void *req, call_frame_t *frame, rpc_clnt_prog_t *prog,
+                          int procnum, struct iobref *iobref, xlator_t *this,
+                          fop_cbk_fn_t cbkfn, xdrproc_t xdrproc);
 
 struct rpc_clnt *
-cli_quotad_clnt_init (xlator_t *this, dict_t *options);
+cli_quotad_clnt_init(xlator_t *this, dict_t *options);
 
 int
-cli_quotad_notify (struct rpc_clnt *rpc, void *mydata,
-                   rpc_clnt_event_t event, void *data);
-
+cli_quotad_notify(struct rpc_clnt *rpc, void *mydata, rpc_clnt_event_t event,
+                  void *data);
diff --git a/cli/src/cli-rl.c b/cli/src/cli-rl.c
index bca37d9c509..7a38a0b882a 100644
--- a/cli/src/cli-rl.c
+++ b/cli/src/cli-rl.c
@@ -17,7 +17,7 @@
 #include "cli-cmd.h"
 #include "cli-mem-types.h"
 
-#include "event.h"
+#include <glusterfs/gf-event.h>
 
 #include <fnmatch.h>
 
@@ -27,384 +27,376 @@
 #include <readline/readline.h>
 #include <readline/history.h>
 
-
 int
-cli_rl_out (struct cli_state *state, const char *fmt, va_list ap)
+cli_rl_out(struct cli_state *state, const char *fmt, va_list ap)
 {
-        int tmp_rl_point = rl_point;
-        int            n = rl_end;
-        int            ret = 0;
+    int tmp_rl_point = rl_point;
+    int n = rl_end;
+    int ret = 0;
 
-        if (rl_end >= 0 ) {
-                rl_kill_text (0, rl_end);
-                rl_redisplay ();
-        }
+    if (rl_end >= 0) {
+        rl_kill_text(0, rl_end);
+        rl_redisplay();
+    }
 
-        printf ("\r%*s\r", (int)strlen (state->prompt), "");
+    printf("\r%*s\r", (int)strlen(state->prompt), "");
 
-        ret = vprintf (fmt, ap);
+    ret = vprintf(fmt, ap);
 
-        printf ("\n");
-        fflush(stdout);
+    printf("\n");
+    fflush(stdout);
 
-        if (n) {
-                rl_do_undo ();
-                rl_point = tmp_rl_point;
-                rl_reset_line_state ();
-        }
+    if (n) {
+        rl_do_undo();
+        rl_point = tmp_rl_point;
+        rl_reset_line_state();
+    }
 
-        return ret;
+    return ret;
 }
 
 int
-cli_rl_err (struct cli_state *state, const char *fmt, va_list ap)
+cli_rl_err(struct cli_state *state, const char *fmt, va_list ap)
 {
-        int tmp_rl_point = rl_point;
-        int            n = rl_end;
-        int            ret = 0;
+    int tmp_rl_point = rl_point;
+    int n = rl_end;
+    int ret = 0;
 
-        if (rl_end >= 0 ) {
-                rl_kill_text (0, rl_end);
-                rl_redisplay ();
-        }
+    if (rl_end >= 0) {
+        rl_kill_text(0, rl_end);
+        rl_redisplay();
+    }
 
-        fprintf (stderr, "\r%*s\r", (int)strlen (state->prompt), "");
+    fprintf(stderr, "\r%*s\r", (int)strlen(state->prompt), "");
 
-        ret = vfprintf (stderr, fmt, ap);
+    ret = vfprintf(stderr, fmt, ap);
 
-        fprintf (stderr, "\n");
-        fflush(stderr);
+    fprintf(stderr, "\n");
+    fflush(stderr);
 
-        if (n) {
-                rl_do_undo ();
-                rl_point = tmp_rl_point;
-                rl_reset_line_state ();
-        }
+    if (n) {
+        rl_do_undo();
+        rl_point = tmp_rl_point;
+        rl_reset_line_state();
+    }
 
-        return ret;
+    return ret;
 }
 
-
 void
-cli_rl_process_line (char *line)
+cli_rl_process_line(char *line)
 {
-        struct cli_state *state = NULL;
-        int               ret = 0;
-
-        state = global_state;
+    struct cli_state *state = NULL;
+    int ret = 0;
 
-        state->rl_processing = 1;
-        {
-                ret = cli_cmd_process_line (state, line);
-                if (ret)
-                        gf_log (THIS->name, GF_LOG_WARNING,
-                                "failed to process line");
+    state = global_state;
 
-                add_history (line);
-        }
-        state->rl_processing = 0;
+    state->rl_processing = 1;
+    {
+        ret = cli_cmd_process_line(state, line);
+        if (ret)
+            gf_log(THIS->name, GF_LOG_WARNING, "failed to process line");
 
+        add_history(line);
+    }
+    state->rl_processing = 0;
 }
 
-
-int
-cli_rl_stdin (int fd, int idx, void *data,
-              int poll_out, int poll_in, int poll_err)
+void
+cli_rl_stdin(int fd, int idx, int gen, void *data, int poll_out, int poll_in,
+             int poll_err, char event_thread_died)
 {
-        rl_callback_read_char ();
+    struct cli_state *state = NULL;
 
-        return 0;
-}
+    state = data;
+
+    rl_callback_read_char();
 
+    gf_event_handled(state->ctx->event_pool, fd, idx, gen);
+
+    return;
+}
 
 char *
-cli_rl_autocomplete_entry (const char *text, int times)
+cli_rl_autocomplete_entry(const char *text, int times)
 {
-        struct cli_state  *state = NULL;
-        char              *retp = NULL;
+    struct cli_state *state = NULL;
+    char *retp = NULL;
 
-        state = global_state;
+    state = global_state;
 
-        if (!state->matchesp)
-                return NULL;
+    if (!state->matchesp)
+        return NULL;
 
-        retp = *state->matchesp;
+    retp = *state->matchesp;
 
-        state->matchesp++;
+    state->matchesp++;
 
-        return retp ? strdup (retp) : NULL;
+    return retp ? strdup(retp) : NULL;
 }
 
-
 int
-cli_rl_token_count (const char *text)
+cli_rl_token_count(const char *text)
 {
-        int          count = 0;
-        const char  *trav = NULL;
-        int          is_spc = 1;
-
-        for (trav = text; *trav; trav++) {
-                if (*trav == ' ') {
-                        is_spc = 1;
-                } else {
-                        if (is_spc) {
-                                count++;
-                                is_spc = 0;
-                        }
-                }
+    int count = 0;
+    const char *trav = NULL;
+    int is_spc = 1;
+
+    for (trav = text; *trav; trav++) {
+        if (*trav == ' ') {
+            is_spc = 1;
+        } else {
+            if (is_spc) {
+                count++;
+                is_spc = 0;
+            }
         }
+    }
 
-        if (is_spc)
-                /* what needs to be autocompleted is a full
-                   new word, and not extend the last word
-                */
-                count++;
+    if (is_spc)
+        /* what needs to be autocompleted is a full
+           new word, and not extend the last word
+        */
+        count++;
 
-        return count;
+    return count;
 }
 
-
 char **
-cli_rl_tokenize (const char *text)
+cli_rl_tokenize(const char *text)
 {
-        int     count = 0;
-        char  **tokens = NULL;
-        char  **tokenp = NULL;
-        char   *token = NULL;
-        char   *copy = NULL;
-        char   *saveptr = NULL;
-        int     i = 0;
-
-        count = cli_rl_token_count (text);
-
-        tokens = calloc (count + 1, sizeof (*tokens));
-        if (!tokens)
-                return NULL;
-
-        copy = strdup (text);
-        if (!copy)
-                goto out;
-
-        tokenp = tokens;
-
-        for (token = strtok_r (copy, " \t\r\n", &saveptr); token;
-             token = strtok_r (NULL, " \t\r\n", &saveptr)) {
-                *tokenp = strdup (token);
-
-                if (!*tokenp)
-                        goto out;
-                tokenp++;
-                i++;
-
-        }
+    int count = 0;
+    char **tokens = NULL;
+    char **tokenp = NULL;
+    char *token = NULL;
+    char *copy = NULL;
+    char *saveptr = NULL;
+    int i = 0;
+
+    count = cli_rl_token_count(text);
+
+    tokens = calloc(count + 1, sizeof(*tokens));
+    if (!tokens)
+        return NULL;
 
-        if (i < count) {
-                /* symbolize that what needs to be autocompleted is
-                   the full set of possible nextwords, and not extend
-                   the last word
-                */
-                *tokenp = strdup ("");
-                if (!*tokenp)
-                        goto out;
-                tokenp++;
-                i++;
-        }
+    copy = strdup(text);
+    if (!copy)
+        goto out;
+
+    tokenp = tokens;
+
+    for (token = strtok_r(copy, " \t\r\n", &saveptr); token;
+         token = strtok_r(NULL, " \t\r\n", &saveptr)) {
+        *tokenp = strdup(token);
+
+        if (!*tokenp)
+            goto out;
+        tokenp++;
+        i++;
+    }
+
+    if (i < count) {
+        /* symbolize that what needs to be autocompleted is
+           the full set of possible nextwords, and not extend
+           the last word
+        */
+        *tokenp = strdup("");
+        if (!*tokenp)
+            goto out;
+        tokenp++;
+        i++;
+    }
 
 out:
-        free (copy);
+    free(copy);
 
-        if (i < count) {
-                cli_cmd_tokens_destroy (tokens);
-                tokens = NULL;
-        }
+    if (i < count) {
+        cli_cmd_tokens_destroy(tokens);
+        tokens = NULL;
+    }
 
-        return tokens;
+    return tokens;
 }
 
-
 char **
-cli_rl_get_matches (struct cli_state *state, struct cli_cmd_word *word,
-                    const char *text)
+cli_rl_get_matches(struct cli_state *state, struct cli_cmd_word *word,
+                   const char *text)
 {
-        char                 **matches = NULL;
-        char                 **matchesp = NULL;
-        struct cli_cmd_word  **next = NULL;
-        int                    count = 0;
-        int                    len = 0;
+    char **matches = NULL;
+    char **matchesp = NULL;
+    struct cli_cmd_word **next = NULL;
+    int count = 0;
+    int len = 0;
 
-        len = strlen (text);
+    len = strlen(text);
 
-        if (!word->nextwords)
-                return NULL;
+    if (!word->nextwords)
+        return NULL;
 
-        for (next = word->nextwords; *next; next++)
-                count++;
+    for (next = word->nextwords; *next; next++)
+        count++;
 
-        matches = calloc (count + 1, sizeof (*matches));
-        matchesp = matches;
+    matches = calloc(count + 1, sizeof(*matches));
+    matchesp = matches;
 
-        for (next = word->nextwords; *next; next++) {
-                if ((*next)->match) {
-                        continue;
-                }
+    for (next = word->nextwords; *next; next++) {
+        if ((*next)->match) {
+            continue;
+        }
 
-                if (strncmp ((*next)->word, text, len) == 0) {
-                        *matchesp = strdup ((*next)->word);
-                        matchesp++;
-                }
+        if (strncmp((*next)->word, text, len) == 0) {
+            *matchesp = strdup((*next)->word);
+            matchesp++;
         }
+    }
 
-        return matches;
+    return matches;
 }
 
-
 int
-cli_rl_autocomplete_prepare (struct cli_state *state, const char *text)
+cli_rl_autocomplete_prepare(struct cli_state *state, const char *text)
 {
-        struct cli_cmd_word   *word = NULL;
-        struct cli_cmd_word   *next = NULL;
-        char                 **tokens = NULL;
-        char                 **tokenp = NULL;
-        char                  *token = NULL;
-        char                 **matches = NULL;
-
-        tokens = cli_rl_tokenize (text);
-        if (!tokens)
-                return 0;
-
-        word = &state->tree.root;
-
-        for (tokenp = tokens; (token = *tokenp); tokenp++) {
-                if (!*(tokenp+1)) {
-                        /* last word */
-                        break;
-                }
-
-                next = cli_cmd_nextword (word, token);
-                word = next;
-                if (!word)
-                        break;
+    struct cli_cmd_word *word = NULL;
+    struct cli_cmd_word *next = NULL;
+    char **tokens = NULL;
+    char **tokenp = NULL;
+    char *token = NULL;
+    char **matches = NULL;
+
+    tokens = cli_rl_tokenize(text);
+    if (!tokens)
+        return 0;
+
+    word = &state->tree.root;
+
+    for (tokenp = tokens; (token = *tokenp); tokenp++) {
+        if (!*(tokenp + 1)) {
+            /* last word */
+            break;
         }
 
+        next = cli_cmd_nextword(word, token);
+        word = next;
         if (!word)
-                goto out;
+            break;
+    }
+
+    if (!word || !token)
+        goto out;
 
-        matches = cli_rl_get_matches (state, word, token);
+    matches = cli_rl_get_matches(state, word, token);
 
-        state->matches = matches;
-        state->matchesp = matches;
+    state->matches = matches;
+    state->matchesp = matches;
 
 out:
-        cli_cmd_tokens_destroy (tokens);
-        return 0;
+    cli_cmd_tokens_destroy(tokens);
+    return 0;
 }
 
-
 int
-cli_rl_autocomplete_cleanup (struct cli_state *state)
+cli_rl_autocomplete_cleanup(struct cli_state *state)
 {
-        if (state->matches)
-                cli_cmd_tokens_destroy (state->matches);
+    if (state->matches)
+        cli_cmd_tokens_destroy(state->matches);
 
-        state->matches = NULL;
-        state->matchesp = NULL;
+    state->matches = NULL;
+    state->matchesp = NULL;
 
-        return 0;
+    return 0;
 }
 
-
 char **
-cli_rl_autocomplete (const char *text, int start, int end)
+cli_rl_autocomplete(const char *text, int start, int end)
 {
-        struct cli_state  *state = NULL;
-        char             **matches = NULL;
-        char               save = 0;
+    struct cli_state *state = NULL;
+    char **matches = NULL;
+    char save = 0;
 
-        state = global_state;
+    state = global_state;
 
-        /* hack to make the autocompletion code neater */
-        /* fake it as though the cursor is at the end of line */
+    /* hack to make the autocompletion code neater */
+    /* fake it as though the cursor is at the end of line */
 
-        save = rl_line_buffer[rl_point];
-        rl_line_buffer[rl_point] = 0;
+    save = rl_line_buffer[rl_point];
+    rl_line_buffer[rl_point] = 0;
 
-        cli_rl_autocomplete_prepare (state, rl_line_buffer);
+    cli_rl_autocomplete_prepare(state, rl_line_buffer);
 
-        matches = rl_completion_matches (text, cli_rl_autocomplete_entry);
+    matches = rl_completion_matches(text, cli_rl_autocomplete_entry);
 
-        cli_rl_autocomplete_cleanup (state);
+    cli_rl_autocomplete_cleanup(state);
 
-        rl_line_buffer[rl_point] = save;
+    rl_line_buffer[rl_point] = save;
 
-        return matches;
+    return matches;
 }
 
-
 static char *
-complete_none (const char *txt, int times)
+complete_none(const char *txt, int times)
 {
-        return NULL;
+    return NULL;
 }
 
-
 void *
-cli_rl_input (void *_data)
+cli_rl_input(void *_data)
 {
-        struct cli_state *state = NULL;
-        char             *line = NULL;
+    struct cli_state *state = NULL;
+    char *line = NULL;
 
-        state = _data;
+    state = _data;
 
-        for (;;) {
-                line = readline (state->prompt);
-                if (!line)
-                        exit(0);  //break;
+    fprintf(stderr,
+            "Welcome to gluster prompt, type 'help' to see the available "
+            "commands.\n");
+    for (;;) {
+        line = readline(state->prompt);
+        if (!line)
+            exit(0);  // break;
 
-                if (*line)
-                        cli_rl_process_line (line);
+        if (*line)
+            cli_rl_process_line(line);
 
-                free (line);
-        }
+        free(line);
+    }
 
-        return NULL;
+    return NULL;
 }
 
-
 int
-cli_rl_enable (struct cli_state *state)
+cli_rl_enable(struct cli_state *state)
 {
-        int ret = 0;
-
-        rl_pre_input_hook = NULL;
-        rl_attempted_completion_function = cli_rl_autocomplete;
-        rl_completion_entry_function = complete_none;
-
-        if (!state->rl_async) {
-                ret = pthread_create (&state->input, NULL,
-                                      cli_rl_input, state);
-                if (ret == 0)
-                        state->rl_enabled = 1;
-                goto out;
-        }
+    int ret = 0;
 
-        ret = event_register (state->ctx->event_pool, 0, cli_rl_stdin, state,
-                              1, 0);
-        if (ret == -1)
-                goto out;
+    rl_pre_input_hook = NULL;
+    rl_attempted_completion_function = cli_rl_autocomplete;
+    rl_completion_entry_function = complete_none;
 
-        state->rl_enabled = 1;
-        rl_callback_handler_install (state->prompt, cli_rl_process_line);
+    if (!state->rl_async) {
+        ret = pthread_create(&state->input, NULL, cli_rl_input, state);
+        if (ret == 0)
+            state->rl_enabled = 1;
+        goto out;
+    }
+
+    ret = gf_event_register(state->ctx->event_pool, 0, cli_rl_stdin, state, 1,
+                            0, 0);
+    if (ret == -1)
+        goto out;
+
+    state->rl_enabled = 1;
+    rl_callback_handler_install(state->prompt, cli_rl_process_line);
 
 out:
-        return state->rl_enabled;
+    return state->rl_enabled;
 }
 
 #else /* HAVE_READLINE */
 
 int
-cli_rl_enable (struct cli_state *state)
+cli_rl_enable(struct cli_state *state)
 {
-        return 0;
+    return 0;
 }
 
 #endif /* HAVE_READLINE */
diff --git a/cli/src/cli-rpc-ops.c b/cli/src/cli-rpc-ops.c
index da9df4cc05e..9b6b0c7fa50 100644
--- a/cli/src/cli-rpc-ops.c
+++ b/cli/src/cli-rpc-ops.c
@@ -14,63 +14,49 @@
  */
 #define VOL_TOP_PERF_FILENAME_DEF_WIDTH 47
 #define VOL_TOP_PERF_FILENAME_ALT_WIDTH 44
-#define VOL_TOP_PERF_SPEED_WIDTH        4
-#define VOL_TOP_PERF_TIME_WIDTH         26
+#define VOL_TOP_PERF_SPEED_WIDTH 4
+#define VOL_TOP_PERF_TIME_WIDTH 26
 
 #define INDENT_MAIN_HEAD "%-25s %s "
 
+#define RETURNING "Returning %d"
+#define XML_ERROR "Error outputting to xml"
+#define XDR_DECODE_FAIL "Failed to decode xdr response"
+#define DICT_SERIALIZE_FAIL "Failed to serialize to data to dictionary"
+#define DICT_UNSERIALIZE_FAIL "Failed to unserialize the dictionary"
+
+/* Do not show estimates if greater than this number */
+#define REBAL_ESTIMATE_SEC_UPPER_LIMIT (60 * 24 * 3600)
+#define REBAL_ESTIMATE_START_TIME 600
+
 #include "cli.h"
-#include "compat-errno.h"
+#include <glusterfs/compat-errno.h>
 #include "cli-cmd.h"
 #include <sys/uio.h>
 #include <stdlib.h>
 #include <sys/mount.h>
-#include "cli1-xdr.h"
-#include "xdr-generic.h"
-#include "protocol-common.h"
+#include <glusterfs/compat.h>
 #include "cli-mem-types.h"
-#include "compat.h"
-
-#include "syscall.h"
+#include <glusterfs/syscall.h>
 #include "glusterfs3.h"
 #include "portmap-xdr.h"
-#include "byte-order.h"
+#include <glusterfs/byte-order.h>
 
-#include "cli-quotad-client.h"
-#include "run.h"
-#include "quota-common-utils.h"
+#include <glusterfs/run.h>
+#include <glusterfs/events.h>
 
-enum gf_task_types {
-        GF_TASK_TYPE_REBALANCE,
-        GF_TASK_TYPE_REMOVE_BRICK
-};
+enum gf_task_types { GF_TASK_TYPE_REBALANCE, GF_TASK_TYPE_REMOVE_BRICK };
 
-extern struct rpc_clnt *global_quotad_rpc;
-extern rpc_clnt_prog_t cli_quotad_clnt;
-extern rpc_clnt_prog_t *cli_rpc_prog;
-extern int              cli_op_ret;
-extern int              connected;
+rpc_clnt_prog_t cli_quotad_clnt;
 
-int32_t
-gf_cli_remove_brick (call_frame_t *frame, xlator_t *this,
-                     void *data);
-
-char *cli_vol_type_str[] = {"Distribute",
-                            "Stripe",
-                            "Replicate",
-                            "Striped-Replicate",
-                            "Disperse",
-                            "Tier",
-                            "Distributed-Stripe",
-                            "Distributed-Replicate",
-                            "Distributed-Striped-Replicate",
-                            "Distributed-Disperse",
-                           };
-
-char *cli_vol_status_str[] = {"Created",
-                              "Started",
-                              "Stopped",
-                             };
+static int32_t
+gf_cli_remove_brick(call_frame_t *frame, xlator_t *this, void *data);
+
+char *cli_vol_status_str[] = {
+    "Created",
+    "Started",
+    "Stopped",
+};
 
 char *cli_vol_task_status_str[] = {"not started",
                                    "in progress",
@@ -81,9020 +67,8582 @@ char *cli_vol_task_status_str[] = {"not started",
                                    "fix-layout stopped",
                                    "fix-layout completed",
                                    "fix-layout failed",
-                                   "unknown"
-};
+                                   "unknown"};
 
-int32_t
-gf_cli_snapshot (call_frame_t *frame, xlator_t *this, void *data);
+static int32_t
+gf_cli_snapshot(call_frame_t *frame, xlator_t *this, void *data);
 
-int32_t
-gf_cli_get_volume (call_frame_t *frame, xlator_t *this,
-                      void *data);
+static int32_t
+gf_cli_get_volume(call_frame_t *frame, xlator_t *this, void *data);
 
-int
-cli_to_glusterd (gf_cli_req *req, call_frame_t *frame, fop_cbk_fn_t cbkfn,
-                 xdrproc_t xdrproc, dict_t *dict, int procnum, xlator_t *this,
-                 rpc_clnt_prog_t *prog, struct iobref *iobref);
-
-rpc_clnt_prog_t cli_handshake_prog = {
-        .progname  = "cli handshake",
-        .prognum   = GLUSTER_HNDSK_PROGRAM,
-        .progver   = GLUSTER_HNDSK_VERSION,
+static int
+cli_to_glusterd(gf_cli_req *req, call_frame_t *frame, fop_cbk_fn_t cbkfn,
+                xdrproc_t xdrproc, dict_t *dict, int procnum, xlator_t *this,
+                rpc_clnt_prog_t *prog, struct iobref *iobref);
+
+static int
+add_cli_cmd_timeout_to_dict(dict_t *dict);
+
+static rpc_clnt_prog_t cli_handshake_prog = {
+    .progname = "cli handshake",
+    .prognum = GLUSTER_HNDSK_PROGRAM,
+    .progver = GLUSTER_HNDSK_VERSION,
 };
 
-rpc_clnt_prog_t cli_pmap_prog = {
-        .progname   = "cli portmap",
-        .prognum    = GLUSTER_PMAP_PROGRAM,
-        .progver    = GLUSTER_PMAP_VERSION,
+static rpc_clnt_prog_t cli_pmap_prog = {
+    .progname = "cli portmap",
+    .prognum = GLUSTER_PMAP_PROGRAM,
+    .progver = GLUSTER_PMAP_VERSION,
 };
 
-int
-gf_cli_probe_cbk (struct rpc_req *req, struct iovec *iov,
-                        int count, void *myframe)
+static void
+gf_free_xdr_cli_rsp(gf_cli_rsp rsp)
 {
-        gf_cli_rsp            rsp   = {0,};
-        int                   ret   = -1;
-        char                  msg[1024] = {0,};
-
-        if (-1 == req->rpc_status) {
-                goto out;
-        }
+    if (rsp.dict.dict_val) {
+        free(rsp.dict.dict_val);
+    }
+    if (rsp.op_errstr) {
+        free(rsp.op_errstr);
+    }
+}
 
-        ret = xdr_to_generic (*iov, &rsp, (xdrproc_t)xdr_gf_cli_rsp);
-        if (ret < 0) {
-                gf_log (((call_frame_t *) myframe)->this->name, GF_LOG_ERROR,
-                        "Failed to decode xdr response");
-                //rsp.op_ret   = -1;
-                //rsp.op_errno = EINVAL;
-                goto out;
-        }
+static void
+gf_free_xdr_getspec_rsp(gf_getspec_rsp rsp)
+{
+    if (rsp.spec) {
+        free(rsp.spec);
+    }
+    if (rsp.xdata.xdata_val) {
+        free(rsp.xdata.xdata_val);
+    }
+}
 
-        gf_log ("cli", GF_LOG_INFO, "Received resp to probe");
+static void
+gf_free_xdr_fsm_log_rsp(gf1_cli_fsm_log_rsp rsp)
+{
+    if (rsp.op_errstr) {
+        free(rsp.op_errstr);
+    }
+    if (rsp.fsm_log.fsm_log_val) {
+        free(rsp.fsm_log.fsm_log_val);
+    }
+}
 
-        if (rsp.op_errstr && (strlen (rsp.op_errstr) > 0)) {
-                snprintf (msg, sizeof (msg), "%s", rsp.op_errstr);
-                if (rsp.op_ret)
-                        gf_log ("cli", GF_LOG_ERROR, "%s", msg);
+static int
+gf_cli_probe_cbk(struct rpc_req *req, struct iovec *iov, int count,
+                 void *myframe)
+{
+    gf_cli_rsp rsp = {
+        0,
+    };
+    int ret = -1;
+    char msg[1024] = "success";
+
+    GF_ASSERT(myframe);
+
+    if (-1 == req->rpc_status) {
+        goto out;
+    }
+
+    ret = xdr_to_generic(*iov, &rsp, (xdrproc_t)xdr_gf_cli_rsp);
+    if (ret < 0) {
+        gf_log(((call_frame_t *)myframe)->this->name, GF_LOG_ERROR,
+               XDR_DECODE_FAIL);
+        // rsp.op_ret   = -1;
+        // rsp.op_errno = EINVAL;
+        goto out;
+    }
+
+    gf_log("cli", GF_LOG_INFO, "Received resp to probe");
+
+    if (rsp.op_errstr && rsp.op_errstr[0] != '\0') {
+        snprintf(msg, sizeof(msg), "%s", rsp.op_errstr);
+        if (rsp.op_ret) {
+            gf_log("cli", GF_LOG_ERROR, "%s", msg);
         }
+    }
 
-        if (global_state->mode & GLUSTER_MODE_XML) {
-                ret = cli_xml_output_str (NULL,
-                                          (rsp.op_ret)? NULL : msg,
-                                          rsp.op_ret, rsp.op_errno,
-                                          (rsp.op_ret)? msg : NULL);
-                if (ret)
-                        gf_log ("cli", GF_LOG_ERROR,
-                                "Error outputting to xml");
-                goto out;
-        }
+    if (global_state->mode & GLUSTER_MODE_XML) {
+        ret = cli_xml_output_str(NULL, (rsp.op_ret) ? NULL : msg, rsp.op_ret,
+                                 rsp.op_errno, (rsp.op_ret) ? msg : NULL);
+        if (ret)
+            gf_log("cli", GF_LOG_ERROR, XML_ERROR);
+        goto out;
+    }
 
-        if (!rsp.op_ret)
-                cli_out ("peer probe: success. %s", msg);
-        else
-                cli_err ("peer probe: failed: %s", msg);
+    if (!rsp.op_ret)
+        cli_out("peer probe: %s", msg);
+    else
+        cli_err("peer probe: failed: %s", msg);
 
-        ret = rsp.op_ret;
+    ret = rsp.op_ret;
 
 out:
-        cli_cmd_broadcast_response (ret);
-        return ret;
+    cli_cmd_broadcast_response(ret);
+    gf_free_xdr_cli_rsp(rsp);
+    return ret;
 }
 
-int
-gf_cli_deprobe_cbk (struct rpc_req *req, struct iovec *iov,
-                       int count, void *myframe)
-{
-        gf_cli_rsp            rsp   = {0,};
-        int                   ret   = -1;
-        char              msg[1024] = {0,};
-
-        if (-1 == req->rpc_status) {
-                goto out;
-        }
-
-        ret = xdr_to_generic (*iov, &rsp, (xdrproc_t)xdr_gf_cli_rsp);
-        if (ret < 0) {
-                gf_log (((call_frame_t *) myframe)->this->name, GF_LOG_ERROR,
-                        "Failed to decode xdr response");
-                //rsp.op_ret   = -1;
-                //rsp.op_errno = EINVAL;
-                goto out;
-        }
-
-        gf_log ("cli", GF_LOG_INFO, "Received resp to deprobe");
-
-        if (rsp.op_ret) {
-                if (strlen (rsp.op_errstr) > 0) {
-                        snprintf (msg, sizeof (msg), "%s", rsp.op_errstr);
-                        gf_log ("cli", GF_LOG_ERROR, "%s", rsp.op_errstr);
-                }
-        } else {
-                snprintf (msg, sizeof (msg), "success");
-        }
-
-        if (global_state->mode & GLUSTER_MODE_XML) {
-                ret = cli_xml_output_str (NULL,
-                                          (rsp.op_ret)? NULL : msg,
-                                          rsp.op_ret, rsp.op_errno,
-                                          (rsp.op_ret)? msg : NULL);
-                if (ret)
-                        gf_log ("cli", GF_LOG_ERROR,
-                                "Error outputting to xml");
-                goto out;
-        }
+static int
+gf_cli_deprobe_cbk(struct rpc_req *req, struct iovec *iov, int count,
+                   void *myframe)
+{
+    gf_cli_rsp rsp = {
+        0,
+    };
+    int ret = -1;
+    char msg[1024] = "success";
+
+    GF_ASSERT(myframe);
+
+    if (-1 == req->rpc_status) {
+        goto out;
+    }
+
+    ret = xdr_to_generic(*iov, &rsp, (xdrproc_t)xdr_gf_cli_rsp);
+    if (ret < 0) {
+        gf_log(((call_frame_t *)myframe)->this->name, GF_LOG_ERROR,
+               XDR_DECODE_FAIL);
+        // rsp.op_ret   = -1;
+        // rsp.op_errno = EINVAL;
+        goto out;
+    }
+
+    gf_log("cli", GF_LOG_INFO, "Received resp to deprobe");
+
+    if (rsp.op_ret) {
+        if (rsp.op_errstr[0] != '\0') {
+            snprintf(msg, sizeof(msg), "%s", rsp.op_errstr);
+            gf_log("cli", GF_LOG_ERROR, "%s", rsp.op_errstr);
+        }
+    }
+
+    if (global_state->mode & GLUSTER_MODE_XML) {
+        ret = cli_xml_output_str(NULL, (rsp.op_ret) ? NULL : msg, rsp.op_ret,
+                                 rsp.op_errno, (rsp.op_ret) ? msg : NULL);
+        if (ret)
+            gf_log("cli", GF_LOG_ERROR, XML_ERROR);
+        goto out;
+    }
 
-        if (!rsp.op_ret)
-                cli_out ("peer detach: %s", msg);
-        else
-                cli_err ("peer detach: failed: %s", msg);
+    if (!rsp.op_ret)
+        cli_out("peer detach: %s", msg);
+    else
+        cli_err("peer detach: failed: %s", msg);
 
-        ret = rsp.op_ret;
+    ret = rsp.op_ret;
 
 out:
-        cli_cmd_broadcast_response (ret);
-        return ret;
+    cli_cmd_broadcast_response(ret);
+    gf_free_xdr_cli_rsp(rsp);
+    return ret;
 }
 
-int
-gf_cli_output_peer_hostnames (dict_t *dict, int count, char *prefix)
-{
-        int ret = -1;
-        char key[256] = {0,};
-        int i = 0;
-        char *hostname = NULL;
-
-        cli_out ("Other names:");
-        /* Starting from friend.hostname1, as friend.hostname0 will be the same
-         * as friend.hostname
-         */
-        for (i = 1; i < count; i++) {
-                memset (key, 0, sizeof (key));
-                snprintf (key, sizeof (key), "%s.hostname%d", prefix, i);
-                ret = dict_get_str (dict, key, &hostname);
-                if (ret)
-                        break;
-                cli_out ("%s", hostname);
-                hostname = NULL;
-        }
+static int
+gf_cli_output_peer_hostnames(dict_t *dict, int count, const char *prefix)
+{
+    int ret = -1;
+    char key[512] = {
+        0,
+    };
+    int i = 0;
+    char *hostname = NULL;
+
+    cli_out("Other names:");
+    /* Starting from friend.hostname1, as friend.hostname0 will be the same
+     * as friend.hostname
+     */
+    for (i = 1; i < count; i++) {
+        ret = snprintf(key, sizeof(key), "%s.hostname%d", prefix, i);
+        ret = dict_get_strn(dict, key, ret, &hostname);
+        if (ret)
+            break;
+        cli_out("%s", hostname);
+        hostname = NULL;
+    }
 
-        return ret;
+    return ret;
 }
 
-int
-gf_cli_output_peer_status (dict_t *dict, int count)
-{
-        int                        ret   = -1;
-        char                       *uuid_buf = NULL;
-        char                       *hostname_buf = NULL;
-        int32_t                    i = 1;
-        char                       key[256] = {0,};
-        char                       *state = NULL;
-        int32_t                    connected = 0;
-        char                       *connected_str = NULL;
-        int                        hostname_count = 0;
-
-        cli_out ("Number of Peers: %d", count);
-        i = 1;
-        while ( i <= count) {
-                snprintf (key, 256, "friend%d.uuid", i);
-                ret = dict_get_str (dict, key, &uuid_buf);
-                if (ret)
-                        goto out;
-
-                snprintf (key, 256, "friend%d.hostname", i);
-                ret = dict_get_str (dict, key, &hostname_buf);
-                if (ret)
-                        goto out;
+static int
+gf_cli_output_peer_status(dict_t *dict, int count)
+{
+    int ret = -1;
+    char *uuid_buf = NULL;
+    char *hostname_buf = NULL;
+    int32_t i = 1;
+    char key[256] = {
+        0,
+    };
+    int keylen;
+    char *state = NULL;
+    int32_t connected = 0;
+    const char *connected_str = NULL;
+    int hostname_count = 0;
+
+    cli_out("Number of Peers: %d", count);
+    i = 1;
+    while (i <= count) {
+        keylen = snprintf(key, sizeof(key), "friend%d.uuid", i);
+        ret = dict_get_strn(dict, key, keylen, &uuid_buf);
+        if (ret)
+            goto out;
 
-                snprintf (key, 256, "friend%d.connected", i);
-                ret = dict_get_int32 (dict, key, &connected);
-                if (ret)
-                        goto out;
-                if (connected)
-                        connected_str = "Connected";
-                else
-                        connected_str = "Disconnected";
+        keylen = snprintf(key, sizeof(key), "friend%d.hostname", i);
+        ret = dict_get_strn(dict, key, keylen, &hostname_buf);
+        if (ret)
+            goto out;
 
+        keylen = snprintf(key, sizeof(key), "friend%d.connected", i);
+        ret = dict_get_int32n(dict, key, keylen, &connected);
+        if (ret)
+            goto out;
+        if (connected)
+            connected_str = "Connected";
+        else
+            connected_str = "Disconnected";
 
-                snprintf (key, 256, "friend%d.state", i);
-                ret = dict_get_str (dict, key, &state);
-                if (ret)
-                        goto out;
+        keylen = snprintf(key, sizeof(key), "friend%d.state", i);
+        ret = dict_get_strn(dict, key, keylen, &state);
+        if (ret)
+            goto out;
 
-                cli_out ("\nHostname: %s\nUuid: %s\nState: %s (%s)",
-                         hostname_buf, uuid_buf, state, connected_str);
+        cli_out("\nHostname: %s\nUuid: %s\nState: %s (%s)", hostname_buf,
+                uuid_buf, state, connected_str);
 
-                memset (key, 0, sizeof (key));
-                snprintf (key, sizeof (key), "friend%d.hostname_count", i);
-                ret = dict_get_int32 (dict, key, &hostname_count);
-                /* Print other addresses only if there are more than 1.
-                 */
-                if ((ret == 0) && (hostname_count > 1)) {
-                        memset (key, 0, sizeof (key));
-                        snprintf (key, sizeof (key), "friend%d", i);
-                        ret = gf_cli_output_peer_hostnames (dict,
-                                                            hostname_count,
-                                                            key);
-                }
-                i++;
+        keylen = snprintf(key, sizeof(key), "friend%d.hostname_count", i);
+        ret = dict_get_int32n(dict, key, keylen, &hostname_count);
+        /* Print other addresses only if there are more than 1.
+         */
+        if ((ret == 0) && (hostname_count > 1)) {
+            snprintf(key, sizeof(key), "friend%d", i);
+            ret = gf_cli_output_peer_hostnames(dict, hostname_count, key);
+            if (ret) {
+                gf_log("cli", GF_LOG_WARNING,
+                       "error outputting peer other names");
+                goto out;
+            }
         }
+        i++;
+    }
 
-        ret = 0;
+    ret = 0;
 out:
-        return ret;
+    return ret;
 }
 
-int
-gf_cli_output_pool_list (dict_t *dict, int count)
-{
-        int                         ret           = -1;
-        char                       *uuid_buf      = NULL;
-        char                       *hostname_buf  = NULL;
-        int32_t                     hostname_len  = 8; /*min len 8 chars*/
-        int32_t                     i             = 1;
-        char                        key[256]      = {0,};
-        int32_t                     connected     = 0;
-        char                       *connected_str = NULL;
-
-        if (count <= 0)
-                goto out;
+static int
+gf_cli_output_pool_list(dict_t *dict, int count)
+{
+    int ret = -1;
+    char *uuid_buf = NULL;
+    char *hostname_buf = NULL;
+    int32_t hostname_len = 8; /*min len 8 chars*/
+    int32_t i = 1;
+    char key[64] = {
+        0,
+    };
+    int keylen;
+    int32_t connected = 0;
+    const char *connected_str = NULL;
+
+    if (count <= 0)
+        goto out;
+
+    while (i <= count) {
+        keylen = snprintf(key, sizeof(key), "friend%d.hostname", i);
+        ret = dict_get_strn(dict, key, keylen, &hostname_buf);
+        if (ret)
+            goto out;
 
-        while (i <= count) {
-                snprintf (key, 256, "friend%d.hostname", i);
-                ret = dict_get_str (dict, key, &hostname_buf);
-                if (ret)
-                        goto out;
+        ret = strlen(hostname_buf);
+        if (ret > hostname_len)
+            hostname_len = ret;
 
-                ret = strlen(hostname_buf);
-                if (ret > hostname_len)
-                        hostname_len = ret;
+        i++;
+    }
 
-                i++;
-        }
+    cli_out("UUID\t\t\t\t\t%-*s\tState", hostname_len, "Hostname");
 
-        cli_out ("UUID\t\t\t\t\t%-*s\tState", hostname_len, "Hostname");
-
-        i = 1;
-        while ( i <= count) {
-                snprintf (key, 256, "friend%d.uuid", i);
-                ret = dict_get_str (dict, key, &uuid_buf);
-                if (ret)
-                        goto out;
+    i = 1;
+    while (i <= count) {
+        keylen = snprintf(key, sizeof(key), "friend%d.uuid", i);
+        ret = dict_get_strn(dict, key, keylen, &uuid_buf);
+        if (ret)
+            goto out;
 
-                snprintf (key, 256, "friend%d.hostname", i);
-                ret = dict_get_str (dict, key, &hostname_buf);
-                if (ret)
-                        goto out;
+        keylen = snprintf(key, sizeof(key), "friend%d.hostname", i);
+        ret = dict_get_strn(dict, key, keylen, &hostname_buf);
+        if (ret)
+            goto out;
 
-                snprintf (key, 256, "friend%d.connected", i);
-                ret = dict_get_int32 (dict, key, &connected);
-                if (ret)
-                        goto out;
-                if (connected)
-                        connected_str = "Connected";
-                else
-                        connected_str = "Disconnected";
+        keylen = snprintf(key, sizeof(key), "friend%d.connected", i);
+        ret = dict_get_int32n(dict, key, keylen, &connected);
+        if (ret)
+            goto out;
+        if (connected)
+            connected_str = "Connected";
+        else
+            connected_str = "Disconnected";
 
-                cli_out ("%s\t%-*s\t%s ", uuid_buf, hostname_len, hostname_buf,
-                         connected_str);
-                i++;
-        }
+        cli_out("%s\t%-*s\t%s ", uuid_buf, hostname_len, hostname_buf,
+                connected_str);
+        i++;
+    }
 
-        ret = 0;
+    ret = 0;
 out:
-        return ret;
+    return ret;
 }
 
 /* function pointer for gf_cli_output_{pool_list,peer_status} */
-typedef int (*cli_friend_output_fn) (dict_t*, int);
-
-int
-gf_cli_list_friends_cbk (struct rpc_req *req, struct iovec *iov,
-                             int count, void *myframe)
-{
-        gf1_cli_peer_list_rsp      rsp   = {0,};
-        int                        ret   = -1;
-        dict_t                    *dict = NULL;
-        char                       msg[1024] = {0,};
-        char                      *cmd = NULL;
-        cli_friend_output_fn       friend_output_fn;
-        call_frame_t              *frame = NULL;
-        unsigned long              flags = 0;
-
-        frame = myframe;
-        flags = (long)frame->local;
-
-        if (flags == GF_CLI_LIST_POOL_NODES) {
-                cmd = "pool list";
-                friend_output_fn = &gf_cli_output_pool_list;
-        } else {
-                cmd = "peer status";
-                friend_output_fn = &gf_cli_output_peer_status;
-        }
-
-        /* 'free' the flags set by gf_cli_list_friends */
-        frame->local = NULL;
-
-        if (-1 == req->rpc_status) {
-                goto out;
-        }
+typedef int (*cli_friend_output_fn)(dict_t *, int);
 
-        ret = xdr_to_generic (*iov, &rsp, (xdrproc_t)xdr_gf1_cli_peer_list_rsp);
-        if (ret < 0) {
-                gf_log (((call_frame_t *) myframe)->this->name, GF_LOG_ERROR,
-                        "Failed to decode xdr response");
-                //rsp.op_ret   = -1;
-                //rsp.op_errno = EINVAL;
-                goto out;
-        }
-
-        gf_log ("cli", GF_LOG_DEBUG, "Received resp to list: %d",
-                rsp.op_ret);
+static int
+gf_cli_list_friends_cbk(struct rpc_req *req, struct iovec *iov, int count,
+                        void *myframe)
+{
+    gf1_cli_peer_list_rsp rsp = {
+        0,
+    };
+    int ret = -1;
+    dict_t *dict = NULL;
+    char msg[1024] = {
+        0,
+    };
+    const char *cmd = NULL;
+    cli_friend_output_fn friend_output_fn;
+    call_frame_t *frame = NULL;
+    unsigned long flags = 0;
 
-        ret = rsp.op_ret;
+    if (-1 == req->rpc_status) {
+        goto out;
+    }
 
-        if (!rsp.op_ret) {
-
-                if (!rsp.friends.friends_len) {
-                        snprintf (msg, sizeof (msg),
-                                  "%s: No peers present", cmd);
-                        if (global_state->mode & GLUSTER_MODE_XML) {
-                                ret = cli_xml_output_peer_status (dict,
-                                                                  rsp.op_ret,
-                                                                  rsp.op_errno,
-                                                                  msg);
-                                if (ret)
-                                        gf_log ("cli", GF_LOG_ERROR,
-                                                "Error outputting to xml");
-                                goto out;
-                        }
-                        cli_err ("%s", msg);
-                        ret = 0;
-                        goto out;
-                }
+    GF_ASSERT(myframe);
 
-                dict = dict_new ();
+    frame = myframe;
 
-                if (!dict) {
-                        ret = -1;
-                        goto out;
-                }
+    flags = (long)frame->local;
 
-                ret = dict_unserialize (rsp.friends.friends_val,
-                                        rsp.friends.friends_len,
-                                        &dict);
+    if (flags == GF_CLI_LIST_POOL_NODES) {
+        cmd = "pool list";
+        friend_output_fn = &gf_cli_output_pool_list;
+    } else {
+        cmd = "peer status";
+        friend_output_fn = &gf_cli_output_peer_status;
+    }
 
-                if (ret) {
-                        gf_log ("", GF_LOG_ERROR,
-                                        "Unable to allocate memory");
-                        goto out;
-                }
+    /* 'free' the flags set by gf_cli_list_friends */
+    frame->local = NULL;
 
-                if (global_state->mode & GLUSTER_MODE_XML) {
-                        ret = cli_xml_output_peer_status (dict, rsp.op_ret,
-                                                          rsp.op_errno, msg);
-                        if (ret)
-                                gf_log ("cli", GF_LOG_ERROR,
-                                        "Error outputting to xml");
-                        goto out;
-                }
+    ret = xdr_to_generic(*iov, &rsp, (xdrproc_t)xdr_gf1_cli_peer_list_rsp);
+    if (ret < 0) {
+        gf_log(frame->this->name, GF_LOG_ERROR, XDR_DECODE_FAIL);
+        // rsp.op_ret   = -1;
+        // rsp.op_errno = EINVAL;
+        goto out;
+    }
 
-                ret = dict_get_int32 (dict, "count", &count);
-                if (ret) {
-                        goto out;
-                }
+    gf_log("cli", GF_LOG_DEBUG, "Received resp to list: %d", rsp.op_ret);
 
-                ret = friend_output_fn (dict, count);
-                if (ret) {
-                        goto out;
-                }
-        } else {
-                if (global_state->mode & GLUSTER_MODE_XML) {
-                        ret = cli_xml_output_peer_status (dict, rsp.op_ret,
-                                                          rsp.op_errno, NULL);
-                        if (ret)
-                                gf_log ("cli", GF_LOG_ERROR,
-                                        "Error outputting to xml");
-                } else {
-                        ret = -1;
-                }
+    if (!rsp.op_ret) {
+        if (!rsp.friends.friends_len) {
+            snprintf(msg, sizeof(msg), "%s: No peers present", cmd);
+            if (global_state->mode & GLUSTER_MODE_XML) {
+                ret = cli_xml_output_peer_status(dict, rsp.op_ret, rsp.op_errno,
+                                                 msg);
+                if (ret)
+                    gf_log("cli", GF_LOG_ERROR, XML_ERROR);
                 goto out;
+            }
+            cli_err("%s", msg);
+            ret = 0;
+            goto out;
         }
 
+        dict = dict_new();
 
-        ret = 0;
-
-out:
-        if (ret)
-                cli_err ("%s: failed", cmd);
-
-        cli_cmd_broadcast_response (ret);
+        if (!dict) {
+            ret = -1;
+            goto out;
+        }
 
-        if (dict)
-                dict_destroy (dict);
+        ret = dict_unserialize(rsp.friends.friends_val, rsp.friends.friends_len,
+                               &dict);
 
-        return ret;
-}
+        if (ret) {
+            gf_log("", GF_LOG_ERROR, DICT_UNSERIALIZE_FAIL);
+            goto out;
+        }
 
-void
-cli_out_options ( char *substr, char *optstr, char *valstr)
-{
-        char                    *ptr1 = NULL;
-        char                    *ptr2 = NULL;
+        if (global_state->mode & GLUSTER_MODE_XML) {
+            ret = cli_xml_output_peer_status(dict, rsp.op_ret, rsp.op_errno,
+                                             msg);
+            if (ret)
+                gf_log("cli", GF_LOG_ERROR, XML_ERROR);
+            goto out;
+        }
 
-        ptr1 = substr;
-        ptr2 = optstr;
+        ret = dict_get_int32_sizen(dict, "count", &count);
+        if (ret) {
+            goto out;
+        }
 
-        while (ptr1)
-        {
- /* Avoiding segmentation fault. */
-                if (!ptr2)
-                        return;
-                if (*ptr1 != *ptr2)
-                        break;
-                ptr1++;
-                ptr2++;
+        ret = friend_output_fn(dict, count);
+        if (ret) {
+            goto out;
         }
+    } else {
+        if (global_state->mode & GLUSTER_MODE_XML) {
+            ret = cli_xml_output_peer_status(dict, rsp.op_ret, rsp.op_errno,
+                                             NULL);
+            if (ret)
+                gf_log("cli", GF_LOG_ERROR, XML_ERROR);
+        } else {
+            ret = -1;
+        }
+        goto out;
+    }
 
-        if (*ptr2 == '\0')
-                return;
-        cli_out ("%s: %s",ptr2 , valstr);
-}
+    ret = 0;
 
-static int
-_gf_cli_output_volinfo_opts (dict_t *d, char *k,
-                             data_t *v, void *tmp)
-{
-        int     ret   = 0;
-        char   *key   = NULL;
-        char   *ptr   = NULL;
-        data_t *value = NULL;
-
-        key = tmp;
-
-        ptr = strstr (k, "option.");
-        if (ptr) {
-                value = v;
-                if (!value) {
-                        ret = -1;
-                        goto out;
-                }
-                cli_out_options (key, k, v->data);
-        }
 out:
-        return ret;
-}
-
-static int
-print_brick_details (dict_t *dict, int volcount, int start_index,
-                     int end_index, int replica_count, int arbiter_count)
-{
-        char           key[1024]   = {0,};
-        int            index       = start_index;
-        int            ret         = -1;
-        char          *brick       = NULL;
-#ifdef HAVE_BD_XLATOR
-        char          *caps        = NULL;
-#endif
+    if (ret)
+        cli_err("%s: failed", cmd);
 
-        while (index <= end_index) {
-                snprintf (key, 1024, "volume%d.brick%d", volcount, index);
-                ret = dict_get_str (dict, key, &brick);
-                if (ret)
-                        goto out;
+    cli_cmd_broadcast_response(ret);
 
-                if (arbiter_count && index % replica_count == 0)
-                        cli_out ("Brick%d: %s (arbiter)", index, brick);
-                else
-                        cli_out ("Brick%d: %s", index, brick);
-#ifdef HAVE_BD_XLATOR
-                snprintf (key, 1024, "volume%d.vg%d", volcount, index);
-                ret = dict_get_str (dict, key, &caps);
-                if (!ret)
-                        cli_out ("Brick%d VG: %s", index, caps);
-#endif
-                index++;
-        }
-        ret = 0;
-out:
-        return ret;
-}
-void
-gf_cli_print_number_of_bricks (int type, int brick_count, int dist_count,
-                               int stripe_count, int replica_count,
-                               int disperse_count, int redundancy_count,
-                               int arbiter_count)
-{
-       if (type == GF_CLUSTER_TYPE_STRIPE_REPLICATE) {
-               if (arbiter_count == 0) {
-                       cli_out ("Number of Bricks: %d x %d x %d = %d",
-                                (brick_count / dist_count),
-                                stripe_count,
-                                replica_count,
-                                brick_count);
-               } else {
-                       cli_out ("Number of Bricks: %d x %d x (%d + %d) = %d",
-                                (brick_count / dist_count),
-                                stripe_count, replica_count - arbiter_count,
-                                arbiter_count, brick_count);
-               }
-       } else if (type == GF_CLUSTER_TYPE_NONE ||
-                  type == GF_CLUSTER_TYPE_TIER) {
-               cli_out ("Number of Bricks: %d", brick_count);
-       } else if (type == GF_CLUSTER_TYPE_DISPERSE) {
-               cli_out ("Number of Bricks: %d x (%d + %d) = %d",
-                        (brick_count / dist_count),
-                        disperse_count - redundancy_count,
-                        redundancy_count, brick_count);
-       } else {
-               /* For both replicate and stripe, dist_count is
-                  good enough */
-               if (arbiter_count == 0) {
-                       cli_out ("Number of Bricks: %d x %d = %d",
-                                (brick_count / dist_count),
-                                dist_count, brick_count);
-               } else {
-                       cli_out ("Number of Bricks: %d x (%d + %d) = %d",
-                                (brick_count / dist_count),
-                                dist_count - arbiter_count, arbiter_count,
-                                brick_count);
-               }
-       }
+    if (dict)
+        dict_unref(dict);
 
+    if (rsp.friends.friends_val) {
+        free(rsp.friends.friends_val);
+    }
+    return ret;
 }
 
-int
-gf_cli_print_tier_info (dict_t *dict, int i, int brick_count)
-{
-
-        int                  hot_brick_count        = -1;
-        int                  cold_type              = 0;
-        int                  cold_brick_count       = 0;
-        int                  cold_replica_count     = 0;
-        int                  cold_arbiter_count     = 0;
-        int                  cold_disperse_count    = 0;
-        int                  cold_redundancy_count  = 0;
-        int                  cold_dist_count        = 0;
-        int                  hot_type               = 0;
-        int                  hot_replica_count      = 0;
-        int                  hot_dist_count         = 0;
-        int                  ret                    = -1;
-        int                  vol_type               = -1;
-        char                 key[256]               = {0,};
-
-        GF_ASSERT (dict);
-
-        memset (key, 0, sizeof (key));
-        snprintf (key, 256, "volume%d.cold_brick_count", i);
-        ret = dict_get_int32 (dict, key, &cold_brick_count);
+static int
+gf_cli_get_state_cbk(struct rpc_req *req, struct iovec *iov, int count,
+                     void *myframe)
+{
+    gf_cli_rsp rsp = {
+        0,
+    };
+    int ret = -1;
+    dict_t *dict = NULL;
+    char *daemon_name = NULL;
+    char *ofilepath = NULL;
+
+    GF_VALIDATE_OR_GOTO("cli", myframe, out);
+
+    if (-1 == req->rpc_status) {
+        goto out;
+    }
+    ret = xdr_to_generic(*iov, &rsp, (xdrproc_t)xdr_gf_cli_rsp);
+    if (ret < 0) {
+        gf_log(((call_frame_t *)myframe)->this->name, GF_LOG_ERROR,
+               XDR_DECODE_FAIL);
+        goto out;
+    }
+
+    dict = dict_new();
+
+    if (!dict) {
+        ret = -1;
+        goto out;
+    }
+
+    ret = dict_unserialize(rsp.dict.dict_val, rsp.dict.dict_len, &dict);
+    if (ret)
+        goto out;
+
+    if (rsp.op_ret) {
+        if (strcmp(rsp.op_errstr, ""))
+            cli_err("Failed to get daemon state: %s", rsp.op_errstr);
+        else
+            cli_err(
+                "Failed to get daemon state. Check glusterd"
+                " log file for more details");
+    } else {
+        ret = dict_get_str_sizen(dict, "daemon", &daemon_name);
         if (ret)
-                goto out;
+            gf_log("cli", GF_LOG_ERROR, "Couldn't get daemon name");
 
-        memset (key, 0, sizeof (key));
-        snprintf (key, 256, "volume%d.cold_type", i);
-        ret = dict_get_int32 (dict, key, &cold_type);
+        ret = dict_get_str_sizen(dict, "ofilepath", &ofilepath);
         if (ret)
-                goto out;
+            gf_log("cli", GF_LOG_ERROR, "Couldn't get filepath");
 
-        memset (key, 0, sizeof (key));
-        snprintf (key, 256, "volume%d.cold_dist_count", i);
-        ret = dict_get_int32 (dict, key, &cold_dist_count);
-        if (ret)
-                goto out;
+        if (daemon_name && ofilepath)
+            cli_out("%s state dumped to %s", daemon_name, ofilepath);
+    }
 
-        memset (key, 0, sizeof (key));
-        snprintf (key, 256, "volume%d.cold_replica_count", i);
-        ret = dict_get_int32 (dict, key, &cold_replica_count);
-        if (ret)
-                goto out;
+    ret = rsp.op_ret;
 
-        memset (key, 0, sizeof (key));
-        snprintf (key, 256, "volume%d.cold_arbiter_count", i);
-        ret = dict_get_int32 (dict, key, &cold_arbiter_count);
-        if (ret)
-                goto out;
+out:
+    if (dict)
+        dict_unref(dict);
 
-        memset (key, 0, sizeof (key));
-        snprintf (key, 256, "volume%d.cold_disperse_count", i);
-        ret = dict_get_int32 (dict, key, &cold_disperse_count);
-        if (ret)
-                goto out;
+    cli_cmd_broadcast_response(ret);
+    gf_free_xdr_cli_rsp(rsp);
 
-        memset (key, 0, sizeof (key));
-        snprintf (key, 256,
-                  "volume%d.cold_redundancy_count", i);
-        ret = dict_get_int32 (dict, key,
-                              &cold_redundancy_count);
-        if (ret)
-                goto out;
+    return ret;
+}
 
-        memset (key, 0, sizeof (key));
-        snprintf (key, 256, "volume%d.hot_brick_count", i);
-        ret = dict_get_int32 (dict, key, &hot_brick_count);
-        if (ret)
-                goto out;
+static void
+cli_out_options(char *substr, char *optstr, char *valstr)
+{
+    char *ptr1 = NULL;
+    char *ptr2 = NULL;
 
-        memset (key, 0, sizeof (key));
-        snprintf (key, 256, "volume%d.hot_type", i);
-        ret = dict_get_int32 (dict, key, &hot_type);
-        if (ret)
-                goto out;
-        memset (key, 0, sizeof (key));
-        snprintf (key, 256, "volume%d.hot_replica_count", i);
-        ret = dict_get_int32 (dict, key, &hot_replica_count);
-        if (ret)
-                goto out;
+    ptr1 = substr;
+    ptr2 = optstr;
 
-        cli_out ("Hot Tier :");
-        vol_type = hot_type;
-        hot_dist_count = (hot_replica_count ?
-                          hot_replica_count : 1);
-        if ((hot_type != GF_CLUSTER_TYPE_TIER) &&
-            (hot_type > 0) &&
-            (hot_dist_count < hot_brick_count))
-                vol_type = hot_type + GF_CLUSTER_TYPE_MAX - 1;
+    while (ptr1) {
+        /* Avoiding segmentation fault. */
+        if (!ptr2)
+            return;
+        if (*ptr1 != *ptr2)
+            break;
+        ptr1++;
+        ptr2++;
+    }
 
-        cli_out ("Hot Tier Type : %s",
-                 cli_vol_type_str[vol_type]);
-        gf_cli_print_number_of_bricks (hot_type,
-                        hot_brick_count, hot_dist_count, 0,
-                        hot_replica_count, 0, 0, 0);
+    if (*ptr2 == '\0')
+        return;
+    cli_out("%s: %s", ptr2, valstr);
+}
 
-        ret = print_brick_details (dict, i, 1, hot_brick_count,
-                                   hot_replica_count, 0);
-        if (ret)
-                goto out;
+static int
+_gf_cli_output_volinfo_opts(dict_t *d, char *k, data_t *v, void *tmp)
+{
+    int ret = 0;
+    char *key = NULL;
+    char *ptr = NULL;
+    data_t *value = NULL;
 
-        cli_out ("Cold Tier:");
-        vol_type = cold_type;
-        if ((cold_type != GF_CLUSTER_TYPE_TIER) &&
-            (cold_type > 0) &&
-            (cold_dist_count < cold_brick_count))
-                vol_type = cold_type + GF_CLUSTER_TYPE_MAX - 1;
+    key = tmp;
 
-        cli_out ("Cold Tier Type : %s",
-                        cli_vol_type_str[vol_type]);
-        gf_cli_print_number_of_bricks (cold_type,
-                cold_brick_count,
-                cold_dist_count, 0, cold_replica_count,
-                cold_disperse_count, cold_redundancy_count, cold_arbiter_count);
+    ptr = strstr(k, "option.");
+    if (ptr) {
+        value = v;
+        if (!value) {
+            ret = -1;
+            goto out;
+        }
+        cli_out_options(key, k, v->data);
+    }
+out:
+    return ret;
+}
 
-        ret = print_brick_details (dict, i, hot_brick_count+1,
-                                   brick_count, cold_replica_count,
-                                   cold_arbiter_count);
+static int
+print_brick_details(dict_t *dict, int volcount, int start_index, int end_index,
+                    int replica_count)
+{
+    char key[64] = {
+        0,
+    };
+    int keylen;
+    int index = start_index;
+    int isArbiter = 0;
+    int ret = -1;
+    char *brick = NULL;
+
+    while (index <= end_index) {
+        keylen = snprintf(key, sizeof(key), "volume%d.brick%d", volcount,
+                          index);
+        ret = dict_get_strn(dict, key, keylen, &brick);
         if (ret)
-                goto out;
+            goto out;
+        keylen = snprintf(key, sizeof(key), "volume%d.brick%d.isArbiter",
+                          volcount, index);
+        if (dict_getn(dict, key, keylen))
+            isArbiter = 1;
+        else
+            isArbiter = 0;
+
+        if (isArbiter)
+            cli_out("Brick%d: %s (arbiter)", index, brick);
+        else
+            cli_out("Brick%d: %s", index, brick);
+        index++;
+    }
+    ret = 0;
 out:
-        return ret;
+    return ret;
 }
 
-int
-gf_cli_get_volume_cbk (struct rpc_req *req, struct iovec *iov,
-                          int count, void *myframe)
-{
-        int                        ret                  = -1;
-        int                        opt_count            = 0;
-        int32_t                    i                    = 0;
-        int32_t                    j                    = 1;
-        int32_t                    status               = 0;
-        int32_t                    type                 = 0;
-        int32_t                    brick_count          = 0;
-        int32_t                    dist_count           = 0;
-        int32_t                    stripe_count         = 0;
-        int32_t                    replica_count        = 0;
-        int32_t                    disperse_count       = 0;
-        int32_t                    redundancy_count     = 0;
-        int32_t                    arbiter_count        = 0;
-        int32_t                    vol_type             = 0;
-        int32_t                    transport            = 0;
-        char                      *volume_id_str        = NULL;
-        char                      *brick                = NULL;
-        char                      *volname              = NULL;
-        dict_t                    *dict                 = NULL;
-        cli_local_t               *local                = NULL;
-        char                       key[1024]            = {0};
-        char                       err_str[2048]        = {0};
-        gf_cli_rsp                 rsp                  = {0};
-        char                      *caps                 = NULL;
-        int                        k __attribute__((unused)) = 0;
-
-        if (-1 == req->rpc_status)
-                goto out;
-
-        ret = xdr_to_generic (*iov, &rsp, (xdrproc_t)xdr_gf_cli_rsp);
-        if (ret < 0) {
-                gf_log (((call_frame_t *) myframe)->this->name, GF_LOG_ERROR,
-                        "Failed to decode xdr response");
-                goto out;
+static void
+gf_cli_print_number_of_bricks(int type, int brick_count, int dist_count,
+                              int stripe_count, int replica_count,
+                              int disperse_count, int redundancy_count,
+                              int arbiter_count)
+{
+    if (type == GF_CLUSTER_TYPE_NONE) {
+        cli_out("Number of Bricks: %d", brick_count);
+    } else if (type == GF_CLUSTER_TYPE_DISPERSE) {
+        cli_out("Number of Bricks: %d x (%d + %d) = %d",
+                (brick_count / dist_count), disperse_count - redundancy_count,
+                redundancy_count, brick_count);
+    } else {
+        /* For both replicate and stripe, dist_count is
+           good enough */
+        if (arbiter_count == 0) {
+            cli_out("Number of Bricks: %d x %d = %d",
+                    (brick_count / dist_count), dist_count, brick_count);
+        } else {
+            cli_out("Number of Bricks: %d x (%d + %d) = %d",
+                    (brick_count / dist_count), dist_count - arbiter_count,
+                    arbiter_count, brick_count);
         }
+    }
+}
 
-        gf_log ("cli", GF_LOG_INFO, "Received resp to get vol: %d",
-                rsp.op_ret);
+static int
+gf_cli_get_volume_cbk(struct rpc_req *req, struct iovec *iov, int count,
+                      void *myframe)
+{
+    int ret = -1;
+    int opt_count = 0;
+    int32_t i = 0;
+    int32_t j = 1;
+    int32_t status = 0;
+    int32_t type = 0;
+    int32_t brick_count = 0;
+    int32_t dist_count = 0;
+    int32_t stripe_count = 0;
+    int32_t replica_count = 0;
+    int32_t disperse_count = 0;
+    int32_t redundancy_count = 0;
+    int32_t arbiter_count = 0;
+    int32_t snap_count = 0;
+    int32_t thin_arbiter_count = 0;
+    int32_t vol_type = 0;
+    int32_t transport = 0;
+    char *volume_id_str = NULL;
+    char *volname = NULL;
+    char *ta_brick = NULL;
+    dict_t *dict = NULL;
+    cli_local_t *local = NULL;
+    char key[64] = {0};
+    int keylen;
+    char err_str[2048] = {0};
+    gf_cli_rsp rsp = {0};
+    char *caps __attribute__((unused)) = NULL;
+    int k __attribute__((unused)) = 0;
+    call_frame_t *frame = NULL;
+
+    GF_ASSERT(myframe);
+
+    if (-1 == req->rpc_status)
+        goto out;
+
+    frame = myframe;
+
+    GF_ASSERT(frame->local);
+
+    local = frame->local;
+
+    ret = xdr_to_generic(*iov, &rsp, (xdrproc_t)xdr_gf_cli_rsp);
+    if (ret < 0) {
+        gf_log(frame->this->name, GF_LOG_ERROR, XDR_DECODE_FAIL);
+        goto out;
+    }
+
+    gf_log("cli", GF_LOG_INFO, "Received resp to get vol: %d", rsp.op_ret);
+
+    if (!rsp.dict.dict_len) {
+        if (global_state->mode & GLUSTER_MODE_XML)
+            goto xml_output;
+        cli_err("No volumes present");
+        ret = 0;
+        goto out;
+    }
 
-        if (rsp.op_ret) {
-                ret = -1;
-                goto out;
-        }
+    dict = dict_new();
 
-        if (!rsp.dict.dict_len) {
-                if (global_state->mode & GLUSTER_MODE_XML)
-                        goto xml_output;
-                cli_err ("No volumes present");
+    if (!dict) {
+        ret = -1;
+        goto out;
+    }
+
+    ret = dict_unserialize(rsp.dict.dict_val, rsp.dict.dict_len, &dict);
+
+    if (ret) {
+        gf_log("cli", GF_LOG_ERROR, DICT_UNSERIALIZE_FAIL);
+        goto out;
+    }
+
+    ret = dict_get_int32_sizen(dict, "count", &count);
+    if (ret)
+        goto out;
+
+    if (!count) {
+        switch (local->get_vol.flags) {
+            case GF_CLI_GET_NEXT_VOLUME:
+                GF_FREE(local->get_vol.volname);
+                local->get_vol.volname = NULL;
                 ret = 0;
                 goto out;
-        }
-
-        dict = dict_new ();
 
-        if (!dict) {
+            case GF_CLI_GET_VOLUME:
+                snprintf(err_str, sizeof(err_str), "Volume %s does not exist",
+                         local->get_vol.volname);
                 ret = -1;
-                goto out;
+                if (!(global_state->mode & GLUSTER_MODE_XML))
+                    goto out;
         }
+    }
 
-        ret = dict_unserialize (rsp.dict.dict_val,
-                                rsp.dict.dict_len,
-                                &dict);
+    if (rsp.op_ret) {
+        if (global_state->mode & GLUSTER_MODE_XML)
+            goto xml_output;
+        ret = -1;
+        goto out;
+    }
 
-        if (ret) {
-                gf_log ("cli", GF_LOG_ERROR,
-                        "Unable to allocate memory");
+xml_output:
+    if (global_state->mode & GLUSTER_MODE_XML) {
+        /* For GET_NEXT_VOLUME output is already begun in
+         * and will also end in gf_cli_get_next_volume()
+         */
+        if (local->get_vol.flags == GF_CLI_GET_VOLUME) {
+            ret = cli_xml_output_vol_info_begin(local, rsp.op_ret, rsp.op_errno,
+                                                rsp.op_errstr);
+            if (ret) {
+                gf_log("cli", GF_LOG_ERROR, XML_ERROR);
                 goto out;
+            }
         }
 
-        ret = dict_get_int32 (dict, "count", &count);
-        if (ret)
+        if (dict) {
+            ret = cli_xml_output_vol_info(local, dict);
+            if (ret) {
+                gf_log("cli", GF_LOG_ERROR, XML_ERROR);
                 goto out;
-
-        local = ((call_frame_t *)myframe)->local;
-
-        if (!count) {
-                switch (local->get_vol.flags) {
-
-                case GF_CLI_GET_NEXT_VOLUME:
-                        GF_FREE (local->get_vol.volname);
-                        local->get_vol.volname = NULL;
-                        ret = 0;
-                        goto out;
-
-                case GF_CLI_GET_VOLUME:
-                        memset (err_str, 0, sizeof (err_str));
-                        snprintf (err_str, sizeof (err_str),
-                                  "Volume %s does not exist",
-                                  local->get_vol.volname);
-                        ret = -1;
-                        if (!(global_state->mode & GLUSTER_MODE_XML))
-                                goto out;
-                }
+            }
         }
 
-xml_output:
-        if (global_state->mode & GLUSTER_MODE_XML) {
-                /* For GET_NEXT_VOLUME output is already begun in
-                 * and will also end in gf_cli_get_next_volume()
-                 */
-                if (local->get_vol.flags == GF_CLI_GET_VOLUME) {
-                        ret = cli_xml_output_vol_info_begin
-                                (local, rsp.op_ret, rsp.op_errno,
-                                 rsp.op_errstr);
-                        if (ret) {
-                                gf_log ("cli", GF_LOG_ERROR,
-                                        "Error outputting to xml");
-                                goto out;
-                        }
-                }
-
-                if (dict) {
-                        ret = cli_xml_output_vol_info (local, dict);
-                        if (ret) {
-                                gf_log ("cli", GF_LOG_ERROR,
-                                        "Error outputting to xml");
-                                goto out;
-                        }
-                }
-
-                if (local->get_vol.flags == GF_CLI_GET_VOLUME) {
-                        ret = cli_xml_output_vol_info_end (local);
-                        if (ret)
-                                gf_log ("cli", GF_LOG_ERROR,
-                                        "Error outputting to xml");
-                }
-                goto out;
+        if (local->get_vol.flags == GF_CLI_GET_VOLUME) {
+            ret = cli_xml_output_vol_info_end(local);
+            if (ret)
+                gf_log("cli", GF_LOG_ERROR, XML_ERROR);
         }
+        goto out;
+    }
 
-        while ( i < count) {
-                cli_out (" ");
-                snprintf (key, 256, "volume%d.name", i);
-                ret = dict_get_str (dict, key, &volname);
-                if (ret)
-                        goto out;
-
-                snprintf (key, 256, "volume%d.type", i);
-                ret = dict_get_int32 (dict, key, &type);
-                if (ret)
-                        goto out;
+    while (i < count) {
+        cli_out(" ");
+        keylen = snprintf(key, sizeof(key), "volume%d.name", i);
+        ret = dict_get_strn(dict, key, keylen, &volname);
+        if (ret)
+            goto out;
 
-                snprintf (key, 256, "volume%d.status", i);
-                ret = dict_get_int32 (dict, key, &status);
-                if (ret)
-                        goto out;
+        keylen = snprintf(key, sizeof(key), "volume%d.type", i);
+        ret = dict_get_int32n(dict, key, keylen, &type);
+        if (ret)
+            goto out;
 
-                snprintf (key, 256, "volume%d.brick_count", i);
-                ret = dict_get_int32 (dict, key, &brick_count);
-                if (ret)
-                        goto out;
+        keylen = snprintf(key, sizeof(key), "volume%d.status", i);
+        ret = dict_get_int32n(dict, key, keylen, &status);
+        if (ret)
+            goto out;
 
-                snprintf (key, 256, "volume%d.dist_count", i);
-                ret = dict_get_int32 (dict, key, &dist_count);
-                if (ret)
-                        goto out;
+        keylen = snprintf(key, sizeof(key), "volume%d.brick_count", i);
+        ret = dict_get_int32n(dict, key, keylen, &brick_count);
+        if (ret)
+            goto out;
 
-                snprintf (key, 256, "volume%d.stripe_count", i);
-                ret = dict_get_int32 (dict, key, &stripe_count);
-                if (ret)
-                        goto out;
+        keylen = snprintf(key, sizeof(key), "volume%d.dist_count", i);
+        ret = dict_get_int32n(dict, key, keylen, &dist_count);
+        if (ret)
+            goto out;
 
-                snprintf (key, 256, "volume%d.replica_count", i);
-                ret = dict_get_int32 (dict, key, &replica_count);
-                if (ret)
-                        goto out;
+        keylen = snprintf(key, sizeof(key), "volume%d.stripe_count", i);
+        ret = dict_get_int32n(dict, key, keylen, &stripe_count);
+        if (ret)
+            goto out;
 
-                snprintf (key, 256, "volume%d.disperse_count", i);
-                ret = dict_get_int32 (dict, key, &disperse_count);
-                if (ret)
-                        goto out;
+        keylen = snprintf(key, sizeof(key), "volume%d.replica_count", i);
+        ret = dict_get_int32n(dict, key, keylen, &replica_count);
+        if (ret)
+            goto out;
 
-                snprintf (key, 256, "volume%d.redundancy_count", i);
-                ret = dict_get_int32 (dict, key, &redundancy_count);
-                if (ret)
-                        goto out;
+        keylen = snprintf(key, sizeof(key), "volume%d.disperse_count", i);
+        ret = dict_get_int32n(dict, key, keylen, &disperse_count);
+        if (ret)
+            goto out;
 
-                snprintf (key, sizeof(key), "volume%d.arbiter_count", i);
-                ret = dict_get_int32 (dict, key, &arbiter_count);
-                if (ret)
-                        goto out;
+        keylen = snprintf(key, sizeof(key), "volume%d.redundancy_count", i);
+        ret = dict_get_int32n(dict, key, keylen, &redundancy_count);
+        if (ret)
+            goto out;
 
-                snprintf (key, 256, "volume%d.transport", i);
-                ret = dict_get_int32 (dict, key, &transport);
-                if (ret)
-                        goto out;
+        keylen = snprintf(key, sizeof(key), "volume%d.arbiter_count", i);
+        ret = dict_get_int32n(dict, key, keylen, &arbiter_count);
+        if (ret)
+            goto out;
 
-                snprintf (key, 256, "volume%d.volume_id", i);
-                ret = dict_get_str (dict, key, &volume_id_str);
-                if (ret)
-                        goto out;
+        keylen = snprintf(key, sizeof(key), "volume%d.transport", i);
+        ret = dict_get_int32n(dict, key, keylen, &transport);
+        if (ret)
+            goto out;
 
-                vol_type = type;
+        keylen = snprintf(key, sizeof(key), "volume%d.volume_id", i);
+        ret = dict_get_strn(dict, key, keylen, &volume_id_str);
+        if (ret)
+            goto out;
 
-                // Distributed (stripe/replicate/stripe-replica) setups
-                if ((type != GF_CLUSTER_TYPE_TIER) && (type > 0) &&
-                    (dist_count < brick_count))
-                       vol_type = type + GF_CLUSTER_TYPE_MAX - 1;
+        keylen = snprintf(key, sizeof(key), "volume%d.snap_count", i);
+        ret = dict_get_int32n(dict, key, keylen, &snap_count);
+        if (ret)
+            goto out;
 
-                cli_out ("Volume Name: %s", volname);
-                cli_out ("Type: %s", cli_vol_type_str[vol_type]);
-                cli_out ("Volume ID: %s", volume_id_str);
-                cli_out ("Status: %s", cli_vol_status_str[status]);
+        keylen = snprintf(key, sizeof(key), "volume%d.thin_arbiter_count", i);
+        ret = dict_get_int32n(dict, key, keylen, &thin_arbiter_count);
+        if (ret)
+            goto out;
 
-#ifdef HAVE_BD_XLATOR
-                k = 0;
-                memset (key, 0, sizeof (key));
-                snprintf (key, sizeof (key), "volume%d.xlator%d", i, k);
-                ret = dict_get_str (dict, key, &caps);
-                if (ret)
-                        goto next;
-                do {
-                        j = 0;
-                        cli_out ("Xlator %d: %s", k + 1, caps);
-                        do {
-                                memset (key, 0, sizeof (key));
-                                snprintf (key, sizeof (key),
-                                          "volume%d.xlator%d.caps%d",
-                                          i, k, j++);
-                                ret = dict_get_str (dict, key, &caps);
-                                if (ret)
-                                        break;
-                                cli_out ("Capability %d: %s", j, caps);
-                        } while (1);
-
-                        memset (key, 0, sizeof (key));
-                        snprintf (key, sizeof (key),
-                                  "volume%d.xlator%d", i, ++k);
-                        ret = dict_get_str (dict, key, &caps);
-                        if (ret)
-                                break;
-                } while (1);
-
-next:
-#else
-                caps = 0; /* Avoid compiler warnings when BD not enabled */
-#endif
-                gf_cli_print_number_of_bricks (type, brick_count,
-                                dist_count, stripe_count, replica_count,
-                                disperse_count, redundancy_count,
-                                arbiter_count);
-
-                cli_out ("Transport-type: %s",
-                         ((transport == 0)?"tcp":
-                          (transport == 1)?"rdma":
-                          "tcp,rdma"));
-                j = 1;
-
-                GF_FREE (local->get_vol.volname);
-                local->get_vol.volname = gf_strdup (volname);
-
-                if (type == GF_CLUSTER_TYPE_TIER) {
-                        ret = gf_cli_print_tier_info (dict, i, brick_count);
-                        if (ret)
-                                goto out;
-
-                } else {
-                        cli_out ("Bricks:");
-                        ret = print_brick_details (dict, i, j, brick_count,
-                                                  replica_count, arbiter_count);
-                        if (ret)
-                                goto out;
-                }
+        // Distributed (stripe/replicate/stripe-replica) setups
+        vol_type = get_vol_type(type, dist_count, brick_count);
 
-                snprintf (key, 256, "volume%d.opt_count",i);
-                ret = dict_get_int32 (dict, key, &opt_count);
-                if (ret)
-                        goto out;
+        cli_out("Volume Name: %s", volname);
+        cli_out("Type: %s", vol_type_str[vol_type]);
+        cli_out("Volume ID: %s", volume_id_str);
+        cli_out("Status: %s", cli_vol_status_str[status]);
+        cli_out("Snapshot Count: %d", snap_count);
 
-                if (!opt_count)
-                        goto out;
+        gf_cli_print_number_of_bricks(
+            type, brick_count, dist_count, stripe_count, replica_count,
+            disperse_count, redundancy_count, arbiter_count);
 
-                cli_out ("Options Reconfigured:");
+        cli_out("Transport-type: %s",
+                ((transport == 0) ? "tcp"
+                                  : (transport == 1) ? "rdma" : "tcp,rdma"));
+        j = 1;
 
-                snprintf (key, 256, "volume%d.option.",i);
+        GF_FREE(local->get_vol.volname);
+        local->get_vol.volname = gf_strdup(volname);
 
-                ret = dict_foreach (dict, _gf_cli_output_volinfo_opts, key);
-                if (ret)
-                        goto out;
+        cli_out("Bricks:");
+        ret = print_brick_details(dict, i, j, brick_count, replica_count);
+        if (ret)
+            goto out;
 
-                i++;
+        if (thin_arbiter_count) {
+            snprintf(key, sizeof(key), "volume%d.thin_arbiter_brick", i);
+            ret = dict_get_str(dict, key, &ta_brick);
+            if (ret)
+                goto out;
+            cli_out("Thin-arbiter-path: %s", ta_brick);
         }
 
-
-        ret = 0;
-out:
+        snprintf(key, sizeof(key), "volume%d.opt_count", i);
+        ret = dict_get_int32(dict, key, &opt_count);
         if (ret)
-                cli_err ("%s", err_str);
-
-        cli_cmd_broadcast_response (ret);
-
-        if (dict)
-                dict_destroy (dict);
-
-        free (rsp.dict.dict_val);
+            goto out;
 
-        free (rsp.op_errstr);
+        if (!opt_count)
+            goto out;
 
-        gf_log ("cli", GF_LOG_DEBUG, "Returning: %d", ret);
-        return ret;
-}
+        cli_out("Options Reconfigured:");
 
-int
-gf_cli_create_volume_cbk (struct rpc_req *req, struct iovec *iov,
-                             int count, void *myframe)
-{
-        gf_cli_rsp              rsp   = {0,};
-        int                     ret   = -1;
-        cli_local_t             *local = NULL;
-        char                    *volname = NULL;
-        dict_t                  *dict = NULL;
-        dict_t                  *rsp_dict = NULL;
+        snprintf(key, sizeof(key), "volume%d.option.", i);
 
-        if (-1 == req->rpc_status) {
-                goto out;
-        }
+        ret = dict_foreach(dict, _gf_cli_output_volinfo_opts, key);
+        if (ret)
+            goto out;
 
-        local = ((call_frame_t *) (myframe))->local;
+        i++;
+    }
 
-        ret = xdr_to_generic (*iov, &rsp, (xdrproc_t)xdr_gf_cli_rsp);
-        if (ret < 0) {
-                gf_log (((call_frame_t *) myframe)->this->name, GF_LOG_ERROR,
-                        "Failed to decode xdr response");
-                goto out;
-        }
+    ret = 0;
+out:
+    if (ret)
+        cli_err("%s", err_str);
 
-        gf_log ("cli", GF_LOG_INFO, "Received resp to create volume");
+    cli_cmd_broadcast_response(ret);
 
-        dict = local->dict;
+    if (dict)
+        dict_unref(dict);
 
-        ret = dict_get_str (dict, "volname", &volname);
-        if (ret)
-                goto out;
+    gf_free_xdr_cli_rsp(rsp);
 
-        if (global_state->mode & GLUSTER_MODE_XML) {
-                if (rsp.op_ret == 0) {
-                        rsp_dict = dict_new ();
-                        ret = dict_unserialize (rsp.dict.dict_val,
-                                                rsp.dict.dict_len,
-                                                &rsp_dict);
-                        if (ret) {
-                                gf_log ("cli", GF_LOG_ERROR,
-                                        "Failed rsp_dict unserialization");
-                                goto out;
-                        }
-                }
+    gf_log("cli", GF_LOG_DEBUG, RETURNING, ret);
+    return ret;
+}
 
-                ret = cli_xml_output_vol_create (rsp_dict, rsp.op_ret,
-                                                 rsp.op_errno, rsp.op_errstr);
-                if (ret)
-                        gf_log ("cli", GF_LOG_ERROR,
-                                "Error outputting to xml");
-                goto out;
-        }
+static int
+gf_cli_create_volume_cbk(struct rpc_req *req, struct iovec *iov, int count,
+                         void *myframe)
+{
+    gf_cli_rsp rsp = {
+        0,
+    };
+    int ret = -1;
+    cli_local_t *local = NULL;
+    char *volname = NULL;
+    dict_t *rsp_dict = NULL;
+    call_frame_t *frame = NULL;
 
-        if (rsp.op_ret && strcmp (rsp.op_errstr, ""))
-                cli_err ("volume create: %s: failed: %s", volname,
-                         rsp.op_errstr);
-        else if (rsp.op_ret)
-                cli_err ("volume create: %s: failed", volname);
-        else
-                cli_out ("volume create: %s: success: "
-                         "please start the volume to access data", volname);
+    GF_ASSERT(myframe);
 
-        ret = rsp.op_ret;
+    if (-1 == req->rpc_status) {
+        goto out;
+    }
 
-out:
-        cli_cmd_broadcast_response (ret);
-        free (rsp.dict.dict_val);
-        free (rsp.op_errstr);
-        return ret;
-}
+    frame = myframe;
 
-int
-gf_cli_delete_volume_cbk (struct rpc_req *req, struct iovec *iov,
-                             int count, void *myframe)
-{
-        gf_cli_rsp              rsp   = {0,};
-        int                     ret   = -1;
-        cli_local_t             *local = NULL;
-        char                    *volname = NULL;
-        call_frame_t            *frame = NULL;
-        dict_t                  *dict = NULL;
-        dict_t                  *rsp_dict = NULL;
-
-        if (-1 == req->rpc_status) {
-                goto out;
-        }
+    GF_ASSERT(frame->local);
 
-        frame = myframe;
+    local = frame->local;
 
-        ret = xdr_to_generic (*iov, &rsp, (xdrproc_t)xdr_gf_cli_rsp);
-        if (ret < 0) {
-                gf_log (frame->this->name, GF_LOG_ERROR,
-                        "Failed to decode xdr response");
-                goto out;
-        }
+    ret = xdr_to_generic(*iov, &rsp, (xdrproc_t)xdr_gf_cli_rsp);
+    if (ret < 0) {
+        gf_log(frame->this->name, GF_LOG_ERROR, XDR_DECODE_FAIL);
+        goto out;
+    }
 
-        local = frame->local;
+    gf_log("cli", GF_LOG_INFO, "Received resp to create volume");
 
-        if (local)
-                dict = local->dict;
-        ret = dict_get_str (dict, "volname", &volname);
-        if (ret) {
-                gf_log (frame->this->name, GF_LOG_ERROR,
-                        "dict get failed");
+    if (global_state->mode & GLUSTER_MODE_XML) {
+        if (rsp.op_ret == 0) {
+            rsp_dict = dict_new();
+            ret = dict_unserialize(rsp.dict.dict_val, rsp.dict.dict_len,
+                                   &rsp_dict);
+            if (ret) {
+                gf_log("cli", GF_LOG_ERROR, DICT_UNSERIALIZE_FAIL);
                 goto out;
+            }
         }
 
-        gf_log ("cli", GF_LOG_INFO, "Received resp to delete volume");
-
-        if (global_state->mode & GLUSTER_MODE_XML) {
-                if (rsp.op_ret == 0) {
-                        rsp_dict = dict_new ();
-                        ret = dict_unserialize (rsp.dict.dict_val,
-                                                rsp.dict.dict_len,
-                                                &rsp_dict);
-                        if (ret) {
-                                gf_log ("cli", GF_LOG_ERROR,
-                                        "Failed rsp_dict unserialization");
-                                goto out;
-                        }
-                }
+        ret = cli_xml_output_vol_create(rsp_dict, rsp.op_ret, rsp.op_errno,
+                                        rsp.op_errstr);
+        if (ret)
+            gf_log("cli", GF_LOG_ERROR, XML_ERROR);
+        goto out;
+    }
 
-                ret = cli_xml_output_generic_volume ("volDelete", rsp_dict,
-                                                     rsp.op_ret, rsp.op_errno,
-                                                     rsp.op_errstr);
-                if (ret)
-                        gf_log ("cli", GF_LOG_ERROR,
-                                "Error outputting to xml");
-                goto out;
-        }
+    ret = dict_get_str_sizen(local->dict, "volname", &volname);
+    if (ret)
+        goto out;
 
-        if (rsp.op_ret && strcmp (rsp.op_errstr, ""))
-                cli_err ("volume delete: %s: failed: %s", volname,
-                         rsp.op_errstr);
-        else if (rsp.op_ret)
-                cli_err ("volume delete: %s: failed", volname);
-        else
-                cli_out ("volume delete: %s: success", volname);
+    if (rsp.op_ret && strcmp(rsp.op_errstr, ""))
+        cli_err("volume create: %s: failed: %s", volname, rsp.op_errstr);
+    else if (rsp.op_ret)
+        cli_err("volume create: %s: failed", volname);
+    else
+        cli_out(
+            "volume create: %s: success: "
+            "please start the volume to access data",
+            volname);
 
-        ret = rsp.op_ret;
+    ret = rsp.op_ret;
 
 out:
-        cli_cmd_broadcast_response (ret);
-        free (rsp.dict.dict_val);
+    cli_cmd_broadcast_response(ret);
+    gf_free_xdr_cli_rsp(rsp);
 
-        gf_log ("", GF_LOG_DEBUG, "Returning with %d", ret);
-        return ret;
+    if (rsp_dict)
+        dict_unref(rsp_dict);
+    return ret;
 }
 
-int
-gf_cli3_1_uuid_get_cbk (struct rpc_req *req, struct iovec *iov,
-                        int count, void *myframe)
+static int
+gf_cli_delete_volume_cbk(struct rpc_req *req, struct iovec *iov, int count,
+                         void *myframe)
 {
-        char                    *uuid_str = NULL;
-        gf_cli_rsp              rsp   = {0,};
-        int                     ret   = -1;
-        cli_local_t             *local = NULL;
-        call_frame_t            *frame = NULL;
-        dict_t                  *dict = NULL;
+    gf_cli_rsp rsp = {
+        0,
+    };
+    int ret = -1;
+    cli_local_t *local = NULL;
+    char *volname = NULL;
+    call_frame_t *frame = NULL;
+    dict_t *rsp_dict = NULL;
 
-        if (-1 == req->rpc_status)
-                goto out;
+    if (-1 == req->rpc_status) {
+        goto out;
+    }
 
-        frame = myframe;
+    GF_ASSERT(myframe);
+    frame = myframe;
 
-        ret = xdr_to_generic (*iov, &rsp, (xdrproc_t)xdr_gf_cli_rsp);
-        if (ret < 0) {
-                gf_log (frame->this->name, GF_LOG_ERROR,
-                        "Failed to decode xdr response");
-                goto out;
-        }
+    GF_ASSERT(frame->local);
 
-        local = frame->local;
-        frame->local = NULL;
+    local = frame->local;
 
-        gf_log ("cli", GF_LOG_INFO, "Received resp to uuid get");
+    ret = xdr_to_generic(*iov, &rsp, (xdrproc_t)xdr_gf_cli_rsp);
+    if (ret < 0) {
+        gf_log(frame->this->name, GF_LOG_ERROR, XDR_DECODE_FAIL);
+        goto out;
+    }
 
-        dict = dict_new ();
-        if (!dict) {
-                ret = -1;
-                goto out;
-        }
+    gf_log("cli", GF_LOG_INFO, "Received resp to delete volume");
 
-        ret = dict_unserialize (rsp.dict.dict_val, rsp.dict.dict_len,
-                                &dict);
-        if (ret) {
-                gf_log ("cli", GF_LOG_ERROR, "Failed to unserialize "
-                        "response for uuid get");
+    if (global_state->mode & GLUSTER_MODE_XML) {
+        if (rsp.op_ret == 0) {
+            rsp_dict = dict_new();
+            ret = dict_unserialize(rsp.dict.dict_val, rsp.dict.dict_len,
+                                   &rsp_dict);
+            if (ret) {
+                gf_log("cli", GF_LOG_ERROR, DICT_UNSERIALIZE_FAIL);
                 goto out;
+            }
         }
 
-        ret = dict_get_str (dict, "uuid", &uuid_str);
-        if (ret) {
-                gf_log ("cli", GF_LOG_ERROR, "Failed to get uuid "
-                        "from dictionary");
-                goto out;
-        }
+        ret = cli_xml_output_generic_volume("volDelete", rsp_dict, rsp.op_ret,
+                                            rsp.op_errno, rsp.op_errstr);
+        if (ret)
+            gf_log("cli", GF_LOG_ERROR, XML_ERROR);
+        goto out;
+    }
 
-        if (global_state->mode & GLUSTER_MODE_XML) {
-                ret = cli_xml_output_dict ("uuidGenerate", dict, rsp.op_ret,
-                                           rsp.op_errno, rsp.op_errstr);
-                if (ret)
-                        gf_log ("cli", GF_LOG_ERROR,
-                                "Error outputting to xml");
-                goto out;
-        }
+    ret = dict_get_str_sizen(local->dict, "volname", &volname);
+    if (ret) {
+        gf_log(frame->this->name, GF_LOG_ERROR, "dict get failed");
+        goto out;
+    }
 
-        if (rsp.op_ret) {
-                if (strcmp (rsp.op_errstr, "") == 0)
-                        cli_err ("Get uuid was unsuccessful");
-                else
-                        cli_err ("%s", rsp.op_errstr);
+    if (rsp.op_ret && strcmp(rsp.op_errstr, ""))
+        cli_err("volume delete: %s: failed: %s", volname, rsp.op_errstr);
+    else if (rsp.op_ret)
+        cli_err("volume delete: %s: failed", volname);
+    else
+        cli_out("volume delete: %s: success", volname);
 
-        } else {
-                cli_out ("UUID: %s", uuid_str);
-
-        }
-        ret = rsp.op_ret;
+    ret = rsp.op_ret;
 
 out:
-        cli_cmd_broadcast_response (ret);
-        cli_local_wipe (local);
-        if (rsp.dict.dict_val)
-                free (rsp.dict.dict_val);
+    cli_cmd_broadcast_response(ret);
+    gf_free_xdr_cli_rsp(rsp);
 
-        gf_log ("", GF_LOG_DEBUG, "Returning with %d", ret);
-        return ret;
+    if (rsp_dict)
+        dict_unref(rsp_dict);
+    gf_log("", GF_LOG_DEBUG, RETURNING, ret);
+    return ret;
 }
 
-int
-gf_cli3_1_uuid_reset_cbk (struct rpc_req *req, struct iovec *iov,
-                             int count, void *myframe)
+static int
+gf_cli3_1_uuid_get_cbk(struct rpc_req *req, struct iovec *iov, int count,
+                       void *myframe)
 {
-        gf_cli_rsp              rsp   = {0,};
-        int                     ret   = -1;
-        cli_local_t             *local = NULL;
-        call_frame_t            *frame = NULL;
-        dict_t                  *dict = NULL;
+    char *uuid_str = NULL;
+    gf_cli_rsp rsp = {
+        0,
+    };
+    int ret = -1;
+    cli_local_t *local = NULL;
+    call_frame_t *frame = NULL;
+    dict_t *dict = NULL;
 
-        if (-1 == req->rpc_status) {
-                goto out;
-        }
+    GF_ASSERT(myframe);
 
-        frame = myframe;
+    if (-1 == req->rpc_status)
+        goto out;
 
-        ret = xdr_to_generic (*iov, &rsp, (xdrproc_t)xdr_gf_cli_rsp);
-        if (ret < 0) {
-                gf_log (frame->this->name, GF_LOG_ERROR,
-                        "Failed to decode xdr response");
-                goto out;
-        }
+    frame = myframe;
 
-        local = frame->local;
-        frame->local = NULL;
-
-        gf_log ("cli", GF_LOG_INFO, "Received resp to uuid reset");
+    GF_ASSERT(frame->local);
 
-        if (global_state->mode & GLUSTER_MODE_XML) {
-                ret = cli_xml_output_dict ("uuidReset", dict, rsp.op_ret,
-                                           rsp.op_errno, rsp.op_errstr);
-                if (ret)
-                        gf_log ("cli", GF_LOG_ERROR,
-                                "Error outputting to xml");
-                goto out;
-        }
+    local = frame->local;
 
-        if (rsp.op_ret && strcmp (rsp.op_errstr, ""))
-                cli_err ("%s", rsp.op_errstr);
-        else
-                cli_out ("resetting the peer uuid has been %s",
-                         (rsp.op_ret) ? "unsuccessful": "successful");
-        ret = rsp.op_ret;
+    ret = xdr_to_generic(*iov, &rsp, (xdrproc_t)xdr_gf_cli_rsp);
+    if (ret < 0) {
+        gf_log(frame->this->name, GF_LOG_ERROR, XDR_DECODE_FAIL);
+        goto out;
+    }
 
-out:
-        cli_cmd_broadcast_response (ret);
-        cli_local_wipe (local);
-        if (rsp.dict.dict_val)
-                free (rsp.dict.dict_val);
-        if (dict)
-                dict_unref (dict);
+    frame->local = NULL;
 
-        gf_log ("", GF_LOG_DEBUG, "Returning with %d", ret);
-        return ret;
-}
+    gf_log("cli", GF_LOG_INFO, "Received resp to uuid get");
 
-int
-gf_cli_start_volume_cbk (struct rpc_req *req, struct iovec *iov,
-                             int count, void *myframe)
-{
-        gf_cli_rsp              rsp   = {0,};
-        int                     ret   = -1;
-        cli_local_t             *local = NULL;
-        char                    *volname = NULL;
-        call_frame_t            *frame = NULL;
-        dict_t                  *dict = NULL;
-        dict_t                  *rsp_dict = NULL;
-
-        if (-1 == req->rpc_status) {
-                goto out;
-        }
+    dict = dict_new();
+    if (!dict) {
+        ret = -1;
+        goto out;
+    }
 
-        frame = myframe;
+    ret = dict_unserialize(rsp.dict.dict_val, rsp.dict.dict_len, &dict);
+    if (ret) {
+        gf_log("cli", GF_LOG_ERROR, DICT_UNSERIALIZE_FAIL);
+        goto out;
+    }
 
-        ret = xdr_to_generic (*iov, &rsp, (xdrproc_t)xdr_gf_cli_rsp);
-        if (ret < 0) {
-                gf_log (frame->this->name, GF_LOG_ERROR,
-                        "Failed to decode xdr response");
-                goto out;
-        }
-
-        if (frame)
-                local = frame->local;
+    if (global_state->mode & GLUSTER_MODE_XML) {
+        ret = cli_xml_output_dict("uuidGenerate", dict, rsp.op_ret,
+                                  rsp.op_errno, rsp.op_errstr);
+        if (ret)
+            gf_log("cli", GF_LOG_ERROR, XML_ERROR);
+        goto out;
+    }
 
-        if (local)
-                dict = local->dict;
+    if (rsp.op_ret) {
+        if (strcmp(rsp.op_errstr, "") == 0)
+            cli_err("Get uuid was unsuccessful");
+        else
+            cli_err("%s", rsp.op_errstr);
 
-        ret = dict_get_str (dict, "volname", &volname);
+    } else {
+        ret = dict_get_str_sizen(dict, "uuid", &uuid_str);
         if (ret) {
-                gf_log ("cli", GF_LOG_ERROR, "dict get failed");
-                goto out;
+            gf_log("cli", GF_LOG_ERROR, "Failed to get uuid from dictionary");
+            goto out;
         }
+        cli_out("UUID: %s", uuid_str);
+    }
+    ret = rsp.op_ret;
 
-        gf_log ("cli", GF_LOG_INFO, "Received resp to start volume");
-
-        if (global_state->mode & GLUSTER_MODE_XML) {
-                if (rsp.op_ret == 0) {
-                        rsp_dict = dict_new ();
-                        ret = dict_unserialize (rsp.dict.dict_val,
-                                                rsp.dict.dict_len,
-                                                &rsp_dict);
-                        if (ret) {
-                                gf_log ("cli", GF_LOG_ERROR,
-                                        "Failed rsp_dict unserialization");
-                                goto out;
-                        }
-                }
-
-                ret = cli_xml_output_generic_volume ("volStart", rsp_dict,
-                                                     rsp.op_ret, rsp.op_errno,
-                                                     rsp.op_errstr);
-                if (ret)
-                        gf_log ("cli", GF_LOG_ERROR,
-                                "Error outputting to xml");
-                goto out;
-        }
-
-        if (rsp.op_ret && strcmp (rsp.op_errstr, ""))
-                cli_err ("volume start: %s: failed: %s", volname,
-                         rsp.op_errstr);
-        else if (rsp.op_ret)
-                cli_err ("volume start: %s: failed", volname);
-        else
-                cli_out ("volume start: %s: success", volname);
+out:
+    cli_cmd_broadcast_response(ret);
+    cli_local_wipe(local);
+    gf_free_xdr_cli_rsp(rsp);
 
-        ret = rsp.op_ret;
+    if (dict)
+        dict_unref(dict);
 
-out:
-        cli_cmd_broadcast_response (ret);
-        free (rsp.dict.dict_val);
-        free (rsp.op_errstr);
-        return ret;
+    gf_log("", GF_LOG_DEBUG, RETURNING, ret);
+    return ret;
 }
 
-int
-gf_cli_stop_volume_cbk (struct rpc_req *req, struct iovec *iov,
-                             int count, void *myframe)
-{
-        gf_cli_rsp            rsp   = {0,};
-        int                   ret   = -1;
-        cli_local_t           *local = NULL;
-        char                  *volname = NULL;
-        call_frame_t          *frame = NULL;
-        dict_t                *dict = NULL;
-        dict_t                *rsp_dict = NULL;
-
-        if (-1 == req->rpc_status) {
-                goto out;
-        }
+static int
+gf_cli3_1_uuid_reset_cbk(struct rpc_req *req, struct iovec *iov, int count,
+                         void *myframe)
+{
+    gf_cli_rsp rsp = {
+        0,
+    };
+    int ret = -1;
+    cli_local_t *local = NULL;
+    call_frame_t *frame = NULL;
 
-        frame = myframe;
+    GF_ASSERT(myframe);
 
-        ret = xdr_to_generic (*iov, &rsp, (xdrproc_t)xdr_gf_cli_rsp);
-        if (ret < 0) {
-                gf_log (frame->this->name, GF_LOG_ERROR,
-                        "Failed to decode xdr response");
-                goto out;
-        }
+    if (-1 == req->rpc_status) {
+        goto out;
+    }
 
-        if (frame)
-                local = frame->local;
+    frame = myframe;
 
-        if (local) {
-                dict = local->dict;
-                ret = dict_get_str (dict, "volname", &volname);
-                if (ret) {
-                        gf_log (frame->this->name, GF_LOG_ERROR,
-                                "Unable to get volname from dict");
-                        goto out;
-                }
-        }
+    GF_ASSERT(frame->local);
 
-        gf_log ("cli", GF_LOG_INFO, "Received resp to stop volume");
+    local = frame->local;
 
-        if (global_state->mode & GLUSTER_MODE_XML) {
-                if (rsp.op_ret == 0) {
-                        rsp_dict = dict_new ();
-                        ret = dict_unserialize (rsp.dict.dict_val,
-                                                rsp.dict.dict_len,
-                                                &rsp_dict);
-                        if (ret) {
-                                gf_log ("cli", GF_LOG_ERROR,
-                                        "Failed rsp_dict unserialization");
-                                goto out;
-                        }
-                }
+    ret = xdr_to_generic(*iov, &rsp, (xdrproc_t)xdr_gf_cli_rsp);
+    if (ret < 0) {
+        gf_log(frame->this->name, GF_LOG_ERROR, XDR_DECODE_FAIL);
+        goto out;
+    }
 
-                ret = cli_xml_output_generic_volume ("volStop", rsp_dict,
-                                                     rsp.op_ret, rsp.op_errno,
-                                                     rsp.op_errstr);
-                if (ret)
-                        gf_log ("cli", GF_LOG_ERROR,
-                                "Error outputting to xml");
-                goto out;
-        }
+    frame->local = NULL;
 
-        if (rsp.op_ret && strcmp (rsp.op_errstr, ""))
-                cli_err ("volume stop: %s: failed: %s", volname, rsp.op_errstr);
-        else if (rsp.op_ret)
-                cli_err ("volume stop: %s: failed", volname);
-        else
-                cli_out ("volume stop: %s: success", volname);
+    gf_log("cli", GF_LOG_INFO, "Received resp to uuid reset");
 
-        ret = rsp.op_ret;
+    if (global_state->mode & GLUSTER_MODE_XML) {
+        ret = cli_xml_output_dict("uuidReset", NULL, rsp.op_ret, rsp.op_errno,
+                                  rsp.op_errstr);
+        if (ret)
+            gf_log("cli", GF_LOG_ERROR, XML_ERROR);
+        goto out;
+    }
+
+    if (rsp.op_ret && strcmp(rsp.op_errstr, ""))
+        cli_err("%s", rsp.op_errstr);
+    else
+        cli_out("resetting the peer uuid has been %s",
+                (rsp.op_ret) ? "unsuccessful" : "successful");
+    ret = rsp.op_ret;
 
 out:
-        cli_cmd_broadcast_response (ret);
-        free (rsp.op_errstr);
-        free (rsp.dict.dict_val);
+    cli_cmd_broadcast_response(ret);
+    cli_local_wipe(local);
+    gf_free_xdr_cli_rsp(rsp);
 
-        return ret;
+    gf_log("", GF_LOG_DEBUG, RETURNING, ret);
+    return ret;
 }
 
-int
-gf_cli_print_rebalance_status (dict_t *dict, enum gf_task_types task_type)
-{
-        int                ret          = -1;
-        int                count        = 0;
-        int                i            = 1;
-        char               key[256]     = {0,};
-        gf_defrag_status_t status_rcd   = GF_DEFRAG_STATUS_NOT_STARTED;
-        uint64_t           files        = 0;
-        uint64_t           size         = 0;
-        uint64_t           lookup       = 0;
-        char               *node_name   = NULL;
-        uint64_t           failures     = 0;
-        uint64_t           skipped      = 0;
-        double             elapsed      = 0;
-        char               *status_str  = NULL;
-        char               *size_str    = NULL;
-
-        ret = dict_get_int32 (dict, "count", &count);
-        if (ret) {
-                gf_log ("cli", GF_LOG_ERROR, "count not set");
-                goto out;
-        }
-
-
-        cli_out ("%40s %16s %13s %13s %13s %13s %20s %18s", "Node",
-                 "Rebalanced-files", "size", "scanned", "failures", "skipped",
-                 "status", "run time in secs");
-        cli_out ("%40s %16s %13s %13s %13s %13s %20s %18s", "---------",
-                 "-----------", "-----------", "-----------", "-----------",
-                 "-----------", "------------", "--------------");
-        for (i = 1; i <= count; i++) {
-                /* Reset the variables to prevent carryover of values */
-                node_name = NULL;
-                files = 0;
-                size = 0;
-                lookup = 0;
-                skipped = 0;
-                status_str = NULL;
-                elapsed = 0;
-
-                /* Check if status is NOT_STARTED, and continue early */
-                memset (key, 0, 256);
-                snprintf (key, 256, "status-%d", i);
-
-                ret = dict_get_int32 (dict, key, (int32_t *)&status_rcd);
-                if (ret) {
-                        gf_log ("cli", GF_LOG_TRACE, "count %d %d", count, i);
-                        gf_log ("cli", GF_LOG_TRACE, "failed to get status");
-                        goto out;
-                }
+static int
+gf_cli_start_volume_cbk(struct rpc_req *req, struct iovec *iov, int count,
+                        void *myframe)
+{
+    gf_cli_rsp rsp = {
+        0,
+    };
+    int ret = -1;
+    cli_local_t *local = NULL;
+    char *volname = NULL;
+    call_frame_t *frame = NULL;
+    dict_t *rsp_dict = NULL;
 
-                if (GF_DEFRAG_STATUS_NOT_STARTED == status_rcd)
-                        continue;
+    GF_ASSERT(myframe);
 
+    if (-1 == req->rpc_status) {
+        goto out;
+    }
 
-                snprintf (key, 256, "node-name-%d", i);
-                ret = dict_get_str (dict, key, &node_name);
-                if (ret)
-                        gf_log ("cli", GF_LOG_TRACE, "failed to get node-name");
+    frame = myframe;
 
-                memset (key, 0, 256);
-                snprintf (key, 256, "files-%d", i);
-                ret = dict_get_uint64 (dict, key, &files);
-                if (ret)
-                        gf_log ("cli", GF_LOG_TRACE,
-                                "failed to get file count");
+    GF_ASSERT(frame->local);
 
-                memset (key, 0, 256);
-                snprintf (key, 256, "size-%d", i);
-                ret = dict_get_uint64 (dict, key, &size);
-                if (ret)
-                        gf_log ("cli", GF_LOG_TRACE,
-                                "failed to get size of xfer");
+    local = frame->local;
 
-                memset (key, 0, 256);
-                snprintf (key, 256, "lookups-%d", i);
-                ret = dict_get_uint64 (dict, key, &lookup);
-                if (ret)
-                        gf_log ("cli", GF_LOG_TRACE,
-                                "failed to get lookedup file count");
+    ret = xdr_to_generic(*iov, &rsp, (xdrproc_t)xdr_gf_cli_rsp);
+    if (ret < 0) {
+        gf_log(frame->this->name, GF_LOG_ERROR, XDR_DECODE_FAIL);
+        goto out;
+    }
 
-                memset (key, 0, 256);
-                snprintf (key, 256, "failures-%d", i);
-                ret = dict_get_uint64 (dict, key, &failures);
-                if (ret)
-                        gf_log ("cli", GF_LOG_TRACE,
-                                "failed to get failures count");
+    gf_log("cli", GF_LOG_INFO, "Received resp to start volume");
 
-                memset (key, 0, 256);
-                snprintf (key, 256, "skipped-%d", i);
-                ret = dict_get_uint64 (dict, key, &skipped);
-                if (ret)
-                        gf_log ("cli", GF_LOG_TRACE,
-                                "failed to get skipped count");
+    if (global_state->mode & GLUSTER_MODE_XML) {
+        if (rsp.op_ret == 0) {
+            rsp_dict = dict_new();
+            ret = dict_unserialize(rsp.dict.dict_val, rsp.dict.dict_len,
+                                   &rsp_dict);
+            if (ret) {
+                gf_log("cli", GF_LOG_ERROR, DICT_UNSERIALIZE_FAIL);
+                goto out;
+            }
+        }
 
-                /* For remove-brick include skipped count into failure count*/
-                if (task_type != GF_TASK_TYPE_REBALANCE) {
-                        failures += skipped;
-                        skipped = 0;
-                }
+        ret = cli_xml_output_generic_volume("volStart", rsp_dict, rsp.op_ret,
+                                            rsp.op_errno, rsp.op_errstr);
+        if (ret)
+            gf_log("cli", GF_LOG_ERROR, XML_ERROR);
+        goto out;
+    }
 
-                memset (key, 0, 256);
-                snprintf (key, 256, "run-time-%d", i);
-                ret = dict_get_double (dict, key, &elapsed);
-                if (ret)
-                        gf_log ("cli", GF_LOG_TRACE, "failed to get run-time");
-
-                /* Check for array bound */
-                if (status_rcd >= GF_DEFRAG_STATUS_MAX)
-                        status_rcd = GF_DEFRAG_STATUS_MAX;
-
-                status_str = cli_vol_task_status_str[status_rcd];
-                size_str = gf_uint64_2human_readable(size);
-                if (size_str) {
-                        cli_out ("%40s %16"PRIu64 " %13s" " %13"PRIu64 " %13"
-                                 PRIu64" %13"PRIu64 " %20s %18.2f", node_name,
-                                 files, size_str, lookup, failures, skipped,
-                                 status_str, elapsed);
-                } else {
-                        cli_out ("%40s %16"PRIu64 " %13"PRIu64 " %13"PRIu64
-                                 " %13"PRIu64" %13"PRIu64 " %20s %18.2f",
-                                 node_name, files, size, lookup, failures,
-                                 skipped, status_str, elapsed);
-                }
-                GF_FREE(size_str);
-        }
-out:
-        return ret;
-}
+    ret = dict_get_str_sizen(local->dict, "volname", &volname);
+    if (ret) {
+        gf_log("cli", GF_LOG_ERROR, "dict get failed");
+        goto out;
+    }
 
-int
-gf_cli_print_tier_status (dict_t *dict, enum gf_task_types task_type)
-{
-        int                ret          = -1;
-        int                count        = 0;
-        int                i            = 1;
-        uint64_t           promoted     = 0;
-        uint64_t           demoted      = 0;
-        char               key[256]     = {0,};
-        char               *node_name   = NULL;
-        gf_defrag_status_t status_rcd   = GF_DEFRAG_STATUS_NOT_STARTED;
-        char               *status_str  = NULL;
-        char               *size_str    = NULL;
-
-        ret = dict_get_int32 (dict, "count", &count);
-        if (ret) {
-                gf_log ("cli", GF_LOG_ERROR, "count not set");
-                goto out;
-        }
+    if (rsp.op_ret && strcmp(rsp.op_errstr, ""))
+        cli_err("volume start: %s: failed: %s", volname, rsp.op_errstr);
+    else if (rsp.op_ret)
+        cli_err("volume start: %s: failed", volname);
+    else
+        cli_out("volume start: %s: success", volname);
 
-        cli_out ("%-20s %-20s %-20s %-20s", "Node", "Promoted files",
-                 "Demoted files", "Status");
-        cli_out ("%-20s %-20s %-20s %-20s", "---------", "---------",
-                 "---------", "---------");
+    ret = rsp.op_ret;
 
-        for (i = 1; i <= count; i++) {
-                /* Reset the variables to prevent carryover of values */
-                node_name = NULL;
-                promoted = 0;
-                demoted = 0;
+out:
+    cli_cmd_broadcast_response(ret);
+    gf_free_xdr_cli_rsp(rsp);
 
-                memset (key, 0, 256);
-                snprintf (key, 256, "node-name-%d", i);
-                ret = dict_get_str (dict, key, &node_name);
-                if (ret)
-                        gf_log ("cli", GF_LOG_TRACE, "failed to get node-name");
+    if (rsp_dict)
+        dict_unref(rsp_dict);
+    return ret;
+}
 
-                memset (key, 0, 256);
-                snprintf (key, 256, "promoted-%d", i);
-                ret = dict_get_uint64 (dict, key, &promoted);
-                if (ret)
-                        gf_log ("cli", GF_LOG_TRACE,
-                                "failed to get promoted count");
+static int
+gf_cli_stop_volume_cbk(struct rpc_req *req, struct iovec *iov, int count,
+                       void *myframe)
+{
+    gf_cli_rsp rsp = {
+        0,
+    };
+    int ret = -1;
+    cli_local_t *local = NULL;
+    char *volname = NULL;
+    call_frame_t *frame = NULL;
+    dict_t *rsp_dict = NULL;
 
-                memset (key, 0, 256);
-                snprintf (key, 256, "demoted-%d", i);
-                ret = dict_get_uint64 (dict, key, &demoted);
-                if (ret)
-                        gf_log ("cli", GF_LOG_TRACE,
-                                "failed to get demoted count");
+    GF_ASSERT(myframe);
 
-                memset (key, 0, 256);
-                snprintf (key, 256, "status-%d", i);
-                ret = dict_get_int32 (dict, key, (int32_t *)&status_rcd);
-                if (ret)
-                        gf_log ("cli", GF_LOG_TRACE,
-                                "failed to get status");
+    if (-1 == req->rpc_status) {
+        goto out;
+    }
 
-                /* Check for array bound */
-                if (status_rcd >= GF_DEFRAG_STATUS_MAX)
-                        status_rcd = GF_DEFRAG_STATUS_MAX;
+    frame = myframe;
 
-                status_str = cli_vol_task_status_str[status_rcd];
-                cli_out ("%-20s %-20"PRIu64" %-20"PRIu64" %-20s",
-                         node_name, promoted, demoted, status_str);
+    GF_ASSERT(frame->local);
 
-        }
-out:
-        return ret;
-}
+    local = frame->local;
 
-int
-gf_cli_defrag_volume_cbk (struct rpc_req *req, struct iovec *iov,
-                             int count, void *myframe)
-{
-        gf_cli_rsp   rsp          = {0,};
-        cli_local_t  *local       = NULL;
-        char         *volname     = NULL;
-        call_frame_t *frame       = NULL;
-        int          cmd          = 0;
-        int          ret          = -1;
-        dict_t       *dict        = NULL;
-        dict_t       *local_dict  = NULL;
-        char         msg[1024]    = {0,};
-        char         *task_id_str = NULL;
-
-        if (-1 == req->rpc_status) {
-                goto out;
-        }
+    ret = xdr_to_generic(*iov, &rsp, (xdrproc_t)xdr_gf_cli_rsp);
+    if (ret < 0) {
+        gf_log(frame->this->name, GF_LOG_ERROR, XDR_DECODE_FAIL);
+        goto out;
+    }
 
-        frame = myframe;
+    gf_log("cli", GF_LOG_INFO, "Received resp to stop volume");
 
-        ret = xdr_to_generic (*iov, &rsp,
-                              (xdrproc_t)xdr_gf_cli_rsp);
-        if (ret < 0) {
-                gf_log (frame->this->name, GF_LOG_ERROR,
-                        "Failed to decode xdr response");
+    if (global_state->mode & GLUSTER_MODE_XML) {
+        if (rsp.op_ret == 0) {
+            rsp_dict = dict_new();
+            ret = dict_unserialize(rsp.dict.dict_val, rsp.dict.dict_len,
+                                   &rsp_dict);
+            if (ret) {
+                gf_log("cli", GF_LOG_ERROR, DICT_UNSERIALIZE_FAIL);
                 goto out;
+            }
         }
 
-        if (frame)
-                local = frame->local;
+        ret = cli_xml_output_generic_volume("volStop", rsp_dict, rsp.op_ret,
+                                            rsp.op_errno, rsp.op_errstr);
+        if (ret)
+            gf_log("cli", GF_LOG_ERROR, XML_ERROR);
+        goto out;
+    }
 
-        if (local)
-                local_dict = local->dict;
+    ret = dict_get_str_sizen(local->dict, "volname", &volname);
+    if (ret) {
+        gf_log(frame->this->name, GF_LOG_ERROR,
+               "Unable to get volname from dict");
+        goto out;
+    }
 
-        ret = dict_get_str (local_dict, "volname", &volname);
-        if (ret) {
-                gf_log (frame->this->name, GF_LOG_ERROR,
-                        "Failed to get volname");
-                goto out;
-        }
+    if (rsp.op_ret && strcmp(rsp.op_errstr, ""))
+        cli_err("volume stop: %s: failed: %s", volname, rsp.op_errstr);
+    else if (rsp.op_ret)
+        cli_err("volume stop: %s: failed", volname);
+    else
+        cli_out("volume stop: %s: success", volname);
 
-        ret = dict_get_int32 (local_dict, "rebalance-command", (int32_t*)&cmd);
-        if (ret) {
-                gf_log ("cli", GF_LOG_ERROR,
-                        "Failed to get command");
-                goto out;
-        }
+    ret = rsp.op_ret;
 
-        if (rsp.dict.dict_len) {
-                /* Unserialize the dictionary */
-                dict  = dict_new ();
+out:
+    cli_cmd_broadcast_response(ret);
+    gf_free_xdr_cli_rsp(rsp);
 
-                ret = dict_unserialize (rsp.dict.dict_val,
-                                        rsp.dict.dict_len,
-                                        &dict);
-                if (ret < 0) {
-                        gf_log ("glusterd", GF_LOG_ERROR,
-                                "failed to "
-                                "unserialize req-buffer to dictionary");
-                        goto out;
-                }
-        }
+    if (rsp_dict)
+        dict_unref(rsp_dict);
 
-        if (!((cmd == GF_DEFRAG_CMD_STOP) ||
-              (cmd == GF_DEFRAG_CMD_STATUS) ||
-              (cmd == GF_DEFRAG_CMD_STATUS_TIER)) &&
-             !(global_state->mode & GLUSTER_MODE_XML)) {
-                /* All other possibilites are about starting a rebalance */
-                ret = dict_get_str (dict, GF_REBALANCE_TID_KEY, &task_id_str);
-                if (rsp.op_ret && strcmp (rsp.op_errstr, "")) {
-                        snprintf (msg, sizeof (msg), "%s", rsp.op_errstr);
-                } else {
-                         if (!rsp.op_ret) {
-                                /* append errstr in the cli msg for successful
-                                 * case since unlock failures can be highlighted
-                                 * event though rebalance command was successful
-                                 */
-                                 if (cmd == GF_DEFRAG_CMD_START_TIER) {
-                                         snprintf (msg, sizeof (msg),
-                                                  "Attach tier is successful "
-                                                  "on %s. use tier status to "
-                                                  "check the status.\nID: %s"
-                                                  "\n%s",
-                                                  volname, task_id_str,
-                                                  rsp.op_errstr);
-                                 } else {
-                                         snprintf (msg, sizeof (msg),
-                                                  "Rebalance on %s has been "
-                                                  "started successfully. Use "
-                                                  "rebalance status command to"
-                                                  " check status of the "
-                                                  "rebalance process.\nID: %s\n%s",
-                                                  volname, task_id_str,
-                                                  rsp.op_errstr);
-                                 }
-                         } else {
-                                snprintf (msg, sizeof (msg),
-                                          "Starting rebalance on volume %s has "
-                                          "been unsuccessful.", volname);
-                        }
-                }
-                goto done;
-        }
-
-        if (cmd == GF_DEFRAG_CMD_STOP) {
-                if (rsp.op_ret == -1) {
-                        if (strcmp (rsp.op_errstr, ""))
-                                snprintf (msg, sizeof (msg),
-                                          "%s", rsp.op_errstr);
-                        else
-                                snprintf (msg, sizeof (msg),
-                                          "rebalance volume %s stop failed",
-                                          volname);
-                        goto done;
-                } else {
-                        /* append errstr in the cli msg for successful case
-                         * since unlock failures can be highlighted event though
-                         * rebalance command was successful */
-                        snprintf (msg, sizeof (msg),
-                                  "rebalance process may be in the middle of a "
-                                  "file migration.\nThe process will be fully "
-                                  "stopped once the migration of the file is "
-                                  "complete.\nPlease check rebalance process "
-                                  "for completion before doing any further "
-                                  "brick related tasks on the volume.\n%s",
-                                  rsp.op_errstr);
-                }
-        }
-        if (cmd == GF_DEFRAG_CMD_STATUS || cmd == GF_DEFRAG_CMD_STATUS_TIER) {
-                if (rsp.op_ret == -1) {
-                        if (strcmp (rsp.op_errstr, ""))
-                                snprintf (msg, sizeof (msg),
-                                          "%s", rsp.op_errstr);
-                        else
-                                snprintf (msg, sizeof (msg),
-                                          "Failed to get the status of "
-                                          "rebalance process");
-                        goto done;
-                } else {
-                        snprintf (msg, sizeof (msg), "%s", rsp.op_errstr);
-                }
-        }
+    return ret;
+}
 
-        if (global_state->mode & GLUSTER_MODE_XML) {
-                ret = cli_xml_output_vol_rebalance (cmd, dict, rsp.op_ret,
-                                                    rsp.op_errno,
-                                                    rsp.op_errstr);
-                goto out;
-        }
+static int
+gf_cli_print_rebalance_status(dict_t *dict, enum gf_task_types task_type)
+{
+    int ret = -1;
+    int count = 0;
+    int i = 1;
+    char key[64] = {
+        0,
+    };
+    int keylen;
+    gf_defrag_status_t status_rcd = GF_DEFRAG_STATUS_NOT_STARTED;
+    uint64_t files = 0;
+    uint64_t size = 0;
+    uint64_t lookup = 0;
+    char *node_name = NULL;
+    uint64_t failures = 0;
+    uint64_t skipped = 0;
+    double elapsed = 0;
+    char *status_str = NULL;
+    char *size_str = NULL;
+    int32_t hrs = 0;
+    uint32_t min = 0;
+    uint32_t sec = 0;
+    gf_boolean_t fix_layout = _gf_false;
+    uint64_t max_time = 0;
+    uint64_t max_elapsed = 0;
+    uint64_t time_left = 0;
+    gf_boolean_t show_estimates = _gf_false;
+
+    ret = dict_get_int32_sizen(dict, "count", &count);
+    if (ret) {
+        gf_log("cli", GF_LOG_ERROR, "count not set");
+        goto out;
+    }
+
+    for (i = 1; i <= count; i++) {
+        keylen = snprintf(key, sizeof(key), "status-%d", i);
+        ret = dict_get_int32n(dict, key, keylen, (int32_t *)&status_rcd);
+        /* If information from a node is missing we should skip
+         * the node and try to fetch information of other nodes.
+         * If information is not found for all nodes, we should
+         * error out.
+         */
+        if (!ret)
+            break;
+        if (ret && i == count) {
+            gf_log("cli", GF_LOG_TRACE, "failed to get status");
+            goto out;
+        }
+    }
+
+    /* Fix layout will be sent to all nodes for the volume
+       so every status should be of type
+       GF_DEFRAG_STATUS_LAYOUT_FIX*
+    */
+
+    if ((task_type == GF_TASK_TYPE_REBALANCE) &&
+        (status_rcd >= GF_DEFRAG_STATUS_LAYOUT_FIX_STARTED)) {
+        fix_layout = _gf_true;
+    }
+
+    if (fix_layout) {
+        cli_out("%35s %41s %27s", "Node", "status", "run time in h:m:s");
+        cli_out("%35s %41s %27s", "---------", "-----------", "------------");
+    } else {
+        cli_out("%40s %16s %13s %13s %13s %13s %20s %18s", "Node",
+                "Rebalanced-files", "size", "scanned", "failures", "skipped",
+                "status",
+                "run time in"
+                " h:m:s");
+        cli_out("%40s %16s %13s %13s %13s %13s %20s %18s", "---------",
+                "-----------", "-----------", "-----------", "-----------",
+                "-----------", "------------", "--------------");
+    }
+
+    for (i = 1; i <= count; i++) {
+        /* Reset the variables to prevent carryover of values */
+        node_name = NULL;
+        files = 0;
+        size = 0;
+        lookup = 0;
+        skipped = 0;
+        status_str = NULL;
+        elapsed = 0;
+        time_left = 0;
+
+        /* Check if status is NOT_STARTED, and continue early */
+        keylen = snprintf(key, sizeof(key), "status-%d", i);
+
+        ret = dict_get_int32n(dict, key, keylen, (int32_t *)&status_rcd);
+        if (ret == -ENOENT) {
+            gf_log("cli", GF_LOG_TRACE, "count %d %d", count, i);
+            gf_log("cli", GF_LOG_TRACE, "failed to get status");
+            gf_log("cli", GF_LOG_ERROR, "node down and has failed to set dict");
+            continue;
+            /* skip this node if value not available*/
+        } else if (ret) {
+            gf_log("cli", GF_LOG_TRACE, "count %d %d", count, i);
+            gf_log("cli", GF_LOG_TRACE, "failed to get status");
+            continue;
+            /* skip this node if value not available*/
+        }
+
+        if (GF_DEFRAG_STATUS_NOT_STARTED == status_rcd)
+            continue;
+
+        if (GF_DEFRAG_STATUS_STARTED == status_rcd)
+            show_estimates = _gf_true;
+
+        keylen = snprintf(key, sizeof(key), "node-name-%d", i);
+        ret = dict_get_strn(dict, key, keylen, &node_name);
+        if (ret)
+            gf_log("cli", GF_LOG_TRACE, "failed to get node-name");
 
-        if (cmd == GF_DEFRAG_CMD_STATUS_TIER)
-                ret = gf_cli_print_tier_status (dict, GF_TASK_TYPE_REBALANCE);
-        else
-                ret = gf_cli_print_rebalance_status (dict,
-                                                     GF_TASK_TYPE_REBALANCE);
+        snprintf(key, sizeof(key), "files-%d", i);
+        ret = dict_get_uint64(dict, key, &files);
+        if (ret)
+            gf_log("cli", GF_LOG_TRACE, "failed to get file count");
 
+        snprintf(key, sizeof(key), "size-%d", i);
+        ret = dict_get_uint64(dict, key, &size);
         if (ret)
-                gf_log ("cli", GF_LOG_ERROR,
-                        "Failed to print rebalance status");
+            gf_log("cli", GF_LOG_TRACE, "failed to get size of xfer");
 
-done:
-        if (global_state->mode & GLUSTER_MODE_XML)
-                cli_xml_output_str ("volRebalance", msg,
-                                    rsp.op_ret, rsp.op_errno,
-                                    rsp.op_errstr);
-        else {
-                if (rsp.op_ret)
-
-                        if (cmd == GF_DEFRAG_CMD_START_TIER || cmd ==
-                            GF_DEFRAG_CMD_STATUS_TIER) {
-                                cli_err ("Tiering Migration Functionality: %s:"
-                                         " failed%s%s", volname,
-                                         strlen (msg) ? ": " : "", msg);
-                        } else
-                                cli_err ("volume rebalance: %s: failed%s%s",
-                                         volname, strlen (msg) ? ": " : "",
-                                         msg);
-                else
-                        if (cmd == GF_DEFRAG_CMD_START_TIER || cmd ==
-                            GF_DEFRAG_CMD_STATUS_TIER) {
-                                cli_out ("Tiering Migration Functionality: %s:"
-                                         " success%s%s", volname,
-                                         strlen (msg) ? ": " : "", msg);
-                        } else
-                                cli_out ("volume rebalance: %s: success%s%s",
-                                         volname, strlen (msg) ? ": " : "",
-                                         msg);
-        }
-        ret = rsp.op_ret;
+        snprintf(key, sizeof(key), "lookups-%d", i);
+        ret = dict_get_uint64(dict, key, &lookup);
+        if (ret)
+            gf_log("cli", GF_LOG_TRACE, "failed to get lookedup file count");
 
-out:
-        free (rsp.op_errstr); //malloced by xdr
-        free (rsp.dict.dict_val); //malloced by xdr
-        if (dict)
-                dict_unref (dict);
-        cli_cmd_broadcast_response (ret);
-        return ret;
-}
+        snprintf(key, sizeof(key), "failures-%d", i);
+        ret = dict_get_uint64(dict, key, &failures);
+        if (ret)
+            gf_log("cli", GF_LOG_TRACE, "failed to get failures count");
 
-int
-gf_cli_rename_volume_cbk (struct rpc_req *req, struct iovec *iov,
-                             int count, void *myframe)
-{
-        gf_cli_rsp              rsp   = {0,};
-        int                     ret   = -1;
-        char                    msg[1024] = {0,};
+        snprintf(key, sizeof(key), "skipped-%d", i);
+        ret = dict_get_uint64(dict, key, &skipped);
+        if (ret)
+            gf_log("cli", GF_LOG_TRACE, "failed to get skipped count");
 
-        if (-1 == req->rpc_status) {
-                goto out;
+        /* For remove-brick include skipped count into failure count*/
+        if (task_type != GF_TASK_TYPE_REBALANCE) {
+            failures += skipped;
+            skipped = 0;
         }
 
-        ret = xdr_to_generic (*iov, &rsp, (xdrproc_t)xdr_gf_cli_rsp);
-        if (ret < 0) {
-                gf_log (((call_frame_t *) myframe)->this->name, GF_LOG_ERROR,
-                        "Failed to decode xdr response");
-                goto out;
-        }
+        snprintf(key, sizeof(key), "run-time-%d", i);
+        ret = dict_get_double(dict, key, &elapsed);
+        if (ret)
+            gf_log("cli", GF_LOG_TRACE, "failed to get run-time");
 
+        snprintf(key, sizeof(key), "time-left-%d", i);
+        ret = dict_get_uint64(dict, key, &time_left);
+        if (ret)
+            gf_log("cli", GF_LOG_TRACE, "failed to get time left");
 
-        gf_log ("cli", GF_LOG_INFO, "Received resp to probe");
-        snprintf (msg, sizeof (msg), "Rename volume %s",
-                  (rsp.op_ret) ? "unsuccessful": "successful");
+        if (elapsed > max_elapsed)
+            max_elapsed = elapsed;
 
-        if (global_state->mode & GLUSTER_MODE_XML) {
-                ret = cli_xml_output_str ("volRename", msg, rsp.op_ret,
-                                          rsp.op_errno, rsp.op_errstr);
-                if (ret)
-                        gf_log ("cli", GF_LOG_ERROR,
-                                "Error outputting to xml");
-                goto out;
-        }
+        if (time_left > max_time)
+            max_time = time_left;
 
-        if (rsp.op_ret)
-                cli_err ("volume rename: failed");
-        else
-                cli_out ("volume rename: success");
+        /* Check for array bound */
+        if (status_rcd >= GF_DEFRAG_STATUS_MAX)
+            status_rcd = GF_DEFRAG_STATUS_MAX;
 
-        ret = rsp.op_ret;
+        status_str = cli_vol_task_status_str[status_rcd];
+        size_str = gf_uint64_2human_readable(size);
+        hrs = elapsed / 3600;
+        min = ((uint64_t)elapsed % 3600) / 60;
+        sec = ((uint64_t)elapsed % 3600) % 60;
 
+        if (fix_layout) {
+            cli_out("%35s %50s %8d:%d:%d", node_name, status_str, hrs, min,
+                    sec);
+        } else {
+            if (size_str) {
+                cli_out("%40s %16" PRIu64
+                        " %13s"
+                        " %13" PRIu64 " %13" PRIu64 " %13" PRIu64
+                        " %20s "
+                        "%8d:%02d:%02d",
+                        node_name, files, size_str, lookup, failures, skipped,
+                        status_str, hrs, min, sec);
+            } else {
+                cli_out("%40s %16" PRIu64 " %13" PRIu64 " %13" PRIu64
+                        " %13" PRIu64 " %13" PRIu64
+                        " %20s"
+                        " %8d:%02d:%02d",
+                        node_name, files, size, lookup, failures, skipped,
+                        status_str, hrs, min, sec);
+            }
+        }
+        GF_FREE(size_str);
+    }
+
+    /* Max time will be non-zero if rebalance is still running */
+    if (max_time) {
+        hrs = max_time / 3600;
+        min = (max_time % 3600) / 60;
+        sec = (max_time % 3600) % 60;
+
+        if (hrs < REBAL_ESTIMATE_SEC_UPPER_LIMIT) {
+            cli_out(
+                "Estimated time left for rebalance to "
+                "complete : %8d:%02d:%02d",
+                hrs, min, sec);
+        } else {
+            cli_out(
+                "Estimated time left for rebalance to "
+                "complete : > 2 months. Please try again "
+                "later.");
+        }
+    } else {
+        /* Rebalance will return 0 if it could not calculate the
+         * estimates or if it is complete.
+         */
+        if (!show_estimates) {
+            goto out;
+        }
+        if (max_elapsed <= REBAL_ESTIMATE_START_TIME) {
+            cli_out(
+                "The estimated time for rebalance to complete "
+                "will be unavailable for the first 10 "
+                "minutes.");
+        } else {
+            cli_out(
+                "Rebalance estimated time unavailable. Please "
+                "try again later.");
+        }
+    }
 out:
-        cli_cmd_broadcast_response (ret);
-        return ret;
+    return ret;
 }
 
-int
-gf_cli_reset_volume_cbk (struct rpc_req *req, struct iovec *iov,
-                             int count, void *myframe)
-{
-        gf_cli_rsp           rsp   = {0,};
-        int                  ret   = -1;
-        char                 msg[1024] = {0,};
-
-        if (-1 == req->rpc_status) {
-                goto out;
-        }
-
-        ret = xdr_to_generic (*iov, &rsp, (xdrproc_t)xdr_gf_cli_rsp);
+static int
+gf_cli_defrag_volume_cbk(struct rpc_req *req, struct iovec *iov, int count,
+                         void *myframe)
+{
+    gf_cli_rsp rsp = {
+        0,
+    };
+    cli_local_t *local = NULL;
+    char *volname = NULL;
+    call_frame_t *frame = NULL;
+    int cmd = 0;
+    int ret = -1;
+    dict_t *dict = NULL;
+    char msg[1024] = {
+        0,
+    };
+    char *task_id_str = NULL;
+
+    if (-1 == req->rpc_status) {
+        goto out;
+    }
+
+    GF_ASSERT(myframe);
+
+    frame = myframe;
+
+    GF_ASSERT(frame->local);
+
+    local = frame->local;
+
+    ret = xdr_to_generic(*iov, &rsp, (xdrproc_t)xdr_gf_cli_rsp);
+    if (ret < 0) {
+        gf_log(frame->this->name, GF_LOG_ERROR, XDR_DECODE_FAIL);
+        goto out;
+    }
+
+    ret = dict_get_str_sizen(local->dict, "volname", &volname);
+    if (ret) {
+        gf_log(frame->this->name, GF_LOG_ERROR, "Failed to get volname");
+        goto out;
+    }
+
+    ret = dict_get_int32_sizen(local->dict, "rebalance-command",
+                               (int32_t *)&cmd);
+    if (ret) {
+        gf_log("cli", GF_LOG_ERROR, "Failed to get command");
+        goto out;
+    }
+
+    if (rsp.dict.dict_len) {
+        /* Unserialize the dictionary */
+        dict = dict_new();
+
+        ret = dict_unserialize(rsp.dict.dict_val, rsp.dict.dict_len, &dict);
         if (ret < 0) {
-                gf_log (((call_frame_t *) myframe)->this->name, GF_LOG_ERROR,
-                        "Failed to decode xdr response");
-                goto out;
+            gf_log("glusterd", GF_LOG_ERROR, DICT_UNSERIALIZE_FAIL);
+            goto out;
         }
+    }
 
-        gf_log ("cli", GF_LOG_INFO, "Received resp to reset");
+    if (!((cmd == GF_DEFRAG_CMD_STOP) || (cmd == GF_DEFRAG_CMD_STATUS)) &&
+        !(global_state->mode & GLUSTER_MODE_XML)) {
+        ret = dict_get_str_sizen(dict, GF_REBALANCE_TID_KEY, &task_id_str);
+        if (ret) {
+            gf_log("cli", GF_LOG_WARNING, "failed to get %s from dict",
+                   GF_REBALANCE_TID_KEY);
+        }
+        if (rsp.op_ret && strcmp(rsp.op_errstr, "")) {
+            snprintf(msg, sizeof(msg), "%s", rsp.op_errstr);
+        } else {
+            if (!rsp.op_ret) {
+                /* append errstr in the cli msg for successful
+                 * case since unlock failures can be highlighted
+                 * event though rebalance command was successful
+                 */
+                snprintf(msg, sizeof(msg),
+                         "Rebalance on %s has been "
+                         "started successfully. Use "
+                         "rebalance status command to"
+                         " check status of the "
+                         "rebalance process.\nID: %s",
+                         volname, task_id_str);
+            } else {
+                snprintf(msg, sizeof(msg),
+                         "Starting rebalance on volume %s has "
+                         "been unsuccessful.",
+                         volname);
+            }
+        }
+        goto done;
+    }
+
+    if (cmd == GF_DEFRAG_CMD_STOP) {
+        if (rsp.op_ret == -1) {
+            if (strcmp(rsp.op_errstr, ""))
+                snprintf(msg, sizeof(msg), "%s", rsp.op_errstr);
+            else
+                snprintf(msg, sizeof(msg), "rebalance volume %s stop failed",
+                         volname);
+            goto done;
+        } else {
+            /* append errstr in the cli msg for successful case
+             * since unlock failures can be highlighted event though
+             * rebalance command was successful */
+            snprintf(msg, sizeof(msg),
+                     "rebalance process may be in the middle of a "
+                     "file migration.\nThe process will be fully "
+                     "stopped once the migration of the file is "
+                     "complete.\nPlease check rebalance process "
+                     "for completion before doing any further "
+                     "brick related tasks on the volume.\n%s",
+                     rsp.op_errstr);
+        }
+    }
+    if (cmd == GF_DEFRAG_CMD_STATUS) {
+        if (rsp.op_ret == -1) {
+            if (strcmp(rsp.op_errstr, ""))
+                snprintf(msg, sizeof(msg), "%s", rsp.op_errstr);
+            else
+                snprintf(msg, sizeof(msg),
+                         "Failed to get the status of rebalance process");
+            goto done;
+        } else {
+            snprintf(msg, sizeof(msg), "%s", rsp.op_errstr);
+        }
+    }
 
-        if (strcmp (rsp.op_errstr, ""))
-                snprintf (msg, sizeof (msg), "%s", rsp.op_errstr);
-        else
-                snprintf (msg, sizeof (msg), "reset volume %s",
-                          (rsp.op_ret) ? "unsuccessful": "successful");
+    if (global_state->mode & GLUSTER_MODE_XML) {
+        ret = cli_xml_output_vol_rebalance(cmd, dict, rsp.op_ret, rsp.op_errno,
+                                           rsp.op_errstr);
+        goto out;
+    }
 
-        if (global_state->mode & GLUSTER_MODE_XML) {
-                ret = cli_xml_output_str ("volReset", msg, rsp.op_ret,
-                                          rsp.op_errno, rsp.op_errstr);
-                if (ret)
-                        gf_log ("cli", GF_LOG_ERROR,
-                                "Error outputting to xml");
-                goto out;
-        }
+    ret = gf_cli_print_rebalance_status(dict, GF_TASK_TYPE_REBALANCE);
+    if (ret)
+        gf_log("cli", GF_LOG_ERROR, "Failed to print rebalance status");
 
+done:
+    if (global_state->mode & GLUSTER_MODE_XML)
+        cli_xml_output_str("volRebalance", msg, rsp.op_ret, rsp.op_errno,
+                           rsp.op_errstr);
+    else {
         if (rsp.op_ret)
-                cli_err ("volume reset: failed: %s", msg);
+            cli_err("volume rebalance: %s: failed%s%s", volname,
+                    strlen(msg) ? ": " : "", msg);
         else
-                cli_out ("volume reset: success: %s", msg);
-
-        ret = rsp.op_ret;
+            cli_out("volume rebalance: %s: success%s%s", volname,
+                    strlen(msg) ? ": " : "", msg);
+    }
+    ret = rsp.op_ret;
 
 out:
-        cli_cmd_broadcast_response (ret);
-        return ret;
+    gf_free_xdr_cli_rsp(rsp);
+    if (dict)
+        dict_unref(dict);
+    cli_cmd_broadcast_response(ret);
+    return ret;
 }
 
-int
-gf_cli_ganesha_cbk (struct rpc_req *req, struct iovec *iov,
-                        int count, void *myframe)
-{
-        gf_cli_rsp           rsp   = {0,};
-        int                  ret   = -1;
-        dict_t               *dict = NULL;
-        char                 *help_str = NULL;
-        char                 msg[1024] = {0,};
-        char                 tmp_str[512] = {0,};
-
-        if (-1 == req->rpc_status) {
-                goto out;
-        }
-
-        ret = xdr_to_generic (*iov, &rsp, (xdrproc_t)xdr_gf_cli_rsp);
-        if (ret < 0) {
-                gf_log (((call_frame_t *) myframe)->this->name, GF_LOG_ERROR,
-                        "Failed to decode xdr response");
-                goto out;
-        }
-
-        gf_log ("cli", GF_LOG_DEBUG, "Received resp to ganesha");
-
-        dict = dict_new ();
-
-        if (!dict) {
-                ret = -1;
-                goto out;
-        }
-
-        ret = dict_unserialize (rsp.dict.dict_val, rsp.dict.dict_len, &dict);
+static int
+gf_cli_rename_volume_cbk(struct rpc_req *req, struct iovec *iov, int count,
+                         void *myframe)
+{
+    gf_cli_rsp rsp = {
+        0,
+    };
+    int ret = -1;
+    char msg[1024] = {
+        0,
+    };
+
+    GF_ASSERT(myframe);
+
+    if (-1 == req->rpc_status) {
+        goto out;
+    }
+
+    ret = xdr_to_generic(*iov, &rsp, (xdrproc_t)xdr_gf_cli_rsp);
+    if (ret < 0) {
+        gf_log(((call_frame_t *)myframe)->this->name, GF_LOG_ERROR,
+               XDR_DECODE_FAIL);
+        goto out;
+    }
+
+    gf_log("cli", GF_LOG_INFO, "Received resp to probe");
+    snprintf(msg, sizeof(msg), "Rename volume %s",
+             (rsp.op_ret) ? "unsuccessful" : "successful");
+
+    if (global_state->mode & GLUSTER_MODE_XML) {
+        ret = cli_xml_output_str("volRename", msg, rsp.op_ret, rsp.op_errno,
+                                 rsp.op_errstr);
         if (ret)
-                goto out;
+            gf_log("cli", GF_LOG_ERROR, XML_ERROR);
+        goto out;
+    }
 
-        if (rsp.op_ret) {
-                if (strcmp (rsp.op_errstr, ""))
-                        cli_err ("nfs-ganesha: failed: %s", rsp.op_errstr);
-                else
-                        cli_err ("nfs-ganesha: failed");
-        }
-
-        else {
-                cli_out("nfs-ganesha : success ");
-        }
+    if (rsp.op_ret)
+        cli_err("volume rename: failed");
+    else
+        cli_out("volume rename: success");
 
-        ret = rsp.op_ret;
+    ret = rsp.op_ret;
 
 out:
-        if (dict)
-                dict_unref (dict);
-        cli_cmd_broadcast_response (ret);
-        return ret;
-}
-
-char *
-is_server_debug_xlator (void *myframe)
-{
-        call_frame_t         *frame        = NULL;
-        cli_local_t          *local        = NULL;
-        char                 **words       = NULL;
-        char                 *key          = NULL;
-        char                 *value        = NULL;
-        char                 *debug_xlator = NULL;
-
-        frame = myframe;
-        local = frame->local;
-        words = (char **)local->words;
-
-        while (*words != NULL) {
-                if (strstr (*words, "trace") == NULL &&
-                    strstr (*words, "error-gen") == NULL) {
-                        words++;
-                        continue;
-                }
-
-                key = *words;
-                words++;
-                value = *words;
-                if (value == NULL)
-                        break;
-                if (strstr (value, "client")) {
-                        words++;
-                        continue;
-                } else {
-                        if (!(strstr (value, "posix") || strstr (value, "acl")
-                              || strstr (value, "locks") ||
-                              strstr (value, "io-threads") ||
-                              strstr (value, "marker") ||
-                              strstr (value, "index"))) {
-                                words++;
-                                continue;
-                        } else {
-                                debug_xlator = gf_strdup (key);
-                                break;
-                        }
-                }
-        }
-
-        return debug_xlator;
+    cli_cmd_broadcast_response(ret);
+    gf_free_xdr_cli_rsp(rsp);
+    return ret;
 }
 
-int
-gf_cli_set_volume_cbk (struct rpc_req *req, struct iovec *iov,
-                             int count, void *myframe)
-{
-        gf_cli_rsp           rsp   = {0,};
-        int                  ret   = -1;
-        dict_t               *dict = NULL;
-        char                 *help_str = NULL;
-        char                 msg[1024] = {0,};
-        char                 *debug_xlator = NULL;
-        char                 tmp_str[512] = {0,};
-
-        if (-1 == req->rpc_status) {
-                goto out;
-        }
-
-        ret = xdr_to_generic (*iov, &rsp, (xdrproc_t)xdr_gf_cli_rsp);
-        if (ret < 0) {
-                gf_log (((call_frame_t *) myframe)->this->name, GF_LOG_ERROR,
-                        "Failed to decode xdr response");
-                goto out;
-        }
-
-        gf_log ("cli", GF_LOG_INFO, "Received resp to set");
-
-        dict = dict_new ();
-
-        if (!dict) {
-                ret = -1;
-                goto out;
-        }
-
-        ret = dict_unserialize (rsp.dict.dict_val, rsp.dict.dict_len, &dict);
-
-        /* For brick processes graph change does not happen on the fly.
-         * The process has to be restarted. So this is a check from the
-         * volume set option such that if debug xlators such as trace/errorgen
-         * are provided in the set command, warn the user.
-         */
-        debug_xlator = is_server_debug_xlator (myframe);
-
-        if (dict_get_str (dict, "help-str", &help_str) && !msg[0])
-                snprintf (msg, sizeof (msg), "Set volume %s",
-                          (rsp.op_ret) ? "unsuccessful": "successful");
-        if (rsp.op_ret == 0 && debug_xlator) {
-                snprintf (tmp_str, sizeof (tmp_str), "\n%s translator has been "
-                          "added to the server volume file. Please restart the"
-                          " volume for enabling the translator", debug_xlator);
-        }
+static int
+gf_cli_reset_volume_cbk(struct rpc_req *req, struct iovec *iov, int count,
+                        void *myframe)
+{
+    gf_cli_rsp rsp = {
+        0,
+    };
+    int ret = -1;
+    char msg[1024] = {
+        0,
+    };
+
+    GF_ASSERT(myframe);
+
+    if (-1 == req->rpc_status) {
+        goto out;
+    }
+
+    ret = xdr_to_generic(*iov, &rsp, (xdrproc_t)xdr_gf_cli_rsp);
+    if (ret < 0) {
+        gf_log(((call_frame_t *)myframe)->this->name, GF_LOG_ERROR,
+               XDR_DECODE_FAIL);
+        goto out;
+    }
+
+    gf_log("cli", GF_LOG_INFO, "Received resp to reset");
+
+    if (strcmp(rsp.op_errstr, ""))
+        snprintf(msg, sizeof(msg), "%s", rsp.op_errstr);
+    else
+        snprintf(msg, sizeof(msg), "reset volume %s",
+                 (rsp.op_ret) ? "unsuccessful" : "successful");
+
+    if (global_state->mode & GLUSTER_MODE_XML) {
+        ret = cli_xml_output_str("volReset", msg, rsp.op_ret, rsp.op_errno,
+                                 rsp.op_errstr);
+        if (ret)
+            gf_log("cli", GF_LOG_ERROR, XML_ERROR);
+        goto out;
+    }
 
-        if ((global_state->mode & GLUSTER_MODE_XML) && (help_str == NULL)) {
-                ret = cli_xml_output_str ("volSet", msg, rsp.op_ret,
-                                          rsp.op_errno, rsp.op_errstr);
-                if (ret)
-                        gf_log ("cli", GF_LOG_ERROR,
-                                "Error outputting to xml");
-                goto out;
-        }
+    if (rsp.op_ret)
+        cli_err("volume reset: failed: %s", msg);
+    else
+        cli_out("volume reset: success: %s", msg);
 
-        if (rsp.op_ret) {
-                if (strcmp (rsp.op_errstr, ""))
-                        cli_err ("volume set: failed: %s", rsp.op_errstr);
-                else
-                        cli_err ("volume set: failed");
-        } else {
-                if (help_str == NULL) {
-                        if (debug_xlator == NULL)
-                                cli_out ("volume set: success");
-                        else
-                                cli_out ("volume set: success%s", tmp_str);
-                }else {
-                        cli_out ("%s", help_str);
-                }
-        }
-
-        ret = rsp.op_ret;
+    ret = rsp.op_ret;
 
 out:
-        if (dict)
-                dict_unref (dict);
-        GF_FREE (debug_xlator);
-        cli_cmd_broadcast_response (ret);
-        return ret;
+    cli_cmd_broadcast_response(ret);
+    gf_free_xdr_cli_rsp(rsp);
+    return ret;
 }
 
-int
-gf_cli_attach_tier_cbk (struct rpc_req *req, struct iovec *iov,
-                             int count, void *myframe)
+static int
+gf_cli_ganesha_cbk(struct rpc_req *req, struct iovec *iov, int count,
+                   void *myframe)
 {
-        gf_cli_rsp                  rsp   = {0,};
-        int                         ret   = -1;
-        char                        msg[1024] = {0,};
+    gf_cli_rsp rsp = {
+        0,
+    };
+    int ret = -1;
+    dict_t *dict = NULL;
 
-        if (-1 == req->rpc_status) {
-                goto out;
-        }
+    GF_ASSERT(myframe);
 
-        ret = xdr_to_generic (*iov, &rsp, (xdrproc_t)xdr_gf_cli_rsp);
-        if (ret < 0) {
-                gf_log (((call_frame_t *) myframe)->this->name, GF_LOG_ERROR,
-                        "Failed to decode xdr response");
-                goto out;
-        }
+    if (-1 == req->rpc_status) {
+        goto out;
+    }
 
+    ret = xdr_to_generic(*iov, &rsp, (xdrproc_t)xdr_gf_cli_rsp);
+    if (ret < 0) {
+        gf_log(((call_frame_t *)myframe)->this->name, GF_LOG_ERROR,
+               XDR_DECODE_FAIL);
+        goto out;
+    }
 
-        gf_log ("cli", GF_LOG_INFO, "Received resp to attach tier");
+    gf_log("cli", GF_LOG_DEBUG, "Received resp to ganesha");
 
-        if (rsp.op_ret && strcmp (rsp.op_errstr, ""))
-                snprintf (msg, sizeof (msg), "%s", rsp.op_errstr);
-        else
-                snprintf (msg, sizeof (msg), "Attach tier %s",
-                          (rsp.op_ret) ? "unsuccessful" : "successful");
+    dict = dict_new();
 
-        if (global_state->mode & GLUSTER_MODE_XML) {
-                ret = cli_xml_output_str ("volAttachTier", msg, rsp.op_ret,
-                                          rsp.op_errno, rsp.op_errstr);
-                if (ret)
-                        gf_log ("cli", GF_LOG_ERROR,
-                                "Error outputting to xml");
-                goto out;
-        }
+    if (!dict) {
+        ret = -1;
+        goto out;
+    }
 
-        if (rsp.op_ret)
-                cli_err ("volume attach-tier: failed: %s", msg);
+    ret = dict_unserialize(rsp.dict.dict_val, rsp.dict.dict_len, &dict);
+    if (ret)
+        goto out;
+
+    if (rsp.op_ret) {
+        if (strcmp(rsp.op_errstr, ""))
+            cli_err("nfs-ganesha: failed: %s", rsp.op_errstr);
         else
-                cli_out ("volume attach-tier: success");
-        ret = rsp.op_ret;
+            cli_err("nfs-ganesha: failed");
+    }
+
+    else {
+        cli_out("nfs-ganesha : success ");
+    }
+
+    ret = rsp.op_ret;
 
 out:
-        cli_cmd_broadcast_response (ret);
-        free (rsp.dict.dict_val);
-        free (rsp.op_errstr);
-        return ret;
+    if (dict)
+        dict_unref(dict);
+    cli_cmd_broadcast_response(ret);
+    return ret;
 }
 
-int
-gf_cli_detach_tier_cbk (struct rpc_req *req, struct iovec *iov,
-                             int count, void *myframe)
+static char *
+is_server_debug_xlator(void *myframe)
 {
+    call_frame_t *frame = NULL;
+    cli_local_t *local = NULL;
+    char **words = NULL;
+    char *key = NULL;
+    char *value = NULL;
+    char *debug_xlator = NULL;
 
-        gf_cli_rsp                      rsp   = {0,};
-        int                             ret   = -1;
-        char                            msg[1024] = {0,};
-        gf1_op_commands                 cmd = GF_OP_CMD_NONE;
-        char                           *cmd_str = "unknown";
-        cli_local_t                    *local = NULL;
-        call_frame_t                   *frame = NULL;
-        char                           *task_id_str = NULL;
-        dict_t                         *rsp_dict = NULL;
-
-        if (-1 == req->rpc_status) {
-                goto out;
-        }
-
-        frame = myframe;
-        local = frame->local;
-
-        ret = xdr_to_generic (*iov, &rsp, (xdrproc_t)xdr_gf_cli_rsp);
-        if (ret < 0) {
-                gf_log (frame->this->name, GF_LOG_ERROR,
-                        "Failed to decode xdr response");
-                goto out;
-        }
-
-        ret = dict_get_int32 (local->dict, "command", (int32_t *)&cmd);
-        if (ret) {
-                 gf_log ("", GF_LOG_ERROR, "failed to get command");
-                 goto out;
-        }
+    frame = myframe;
+    local = frame->local;
+    words = (char **)local->words;
 
-        if (rsp.dict.dict_len) {
-                rsp_dict = dict_new ();
-                if (!rsp_dict) {
-                        ret = -1;
-                        goto out;
-                }
-
-                ret = dict_unserialize (rsp.dict.dict_val, rsp.dict.dict_len,
-                                        &rsp_dict);
-                if (ret) {
-                        gf_log ("cli", GF_LOG_ERROR,
-                                "Failed to unserialize rsp_dict");
-                        goto out;
-                }
+    while (*words != NULL) {
+        if (strstr(*words, "trace") == NULL &&
+            strstr(*words, "error-gen") == NULL) {
+            words++;
+            continue;
         }
 
-        switch (cmd) {
-        case GF_OP_CMD_DETACH_START:
-                cmd_str = "start";
-
-                ret = dict_get_str (rsp_dict, GF_REMOVE_BRICK_TID_KEY,
-                                    &task_id_str);
-                if (ret) {
-                        gf_log ("cli", GF_LOG_ERROR,
-                                "remove-brick-id is not present in dict");
-                }
-                break;
-        case GF_OP_CMD_DETACH_COMMIT:
-                cmd_str = "commit";
-                break;
-        case GF_OP_CMD_DETACH_COMMIT_FORCE:
-                cmd_str = "commit force";
-                break;
-        default:
-                cmd_str = "unknown";
+        key = *words;
+        words++;
+        value = *words;
+        if (value == NULL)
+            break;
+        if (strstr(value, "client")) {
+            words++;
+            continue;
+        } else {
+            if (!(strstr(value, "posix") || strstr(value, "acl") ||
+                  strstr(value, "locks") || strstr(value, "io-threads") ||
+                  strstr(value, "marker") || strstr(value, "index"))) {
+                words++;
+                continue;
+            } else {
+                debug_xlator = gf_strdup(key);
                 break;
+            }
         }
+    }
 
-        gf_log ("cli", GF_LOG_INFO, "Received resp to detach-tier");
-
-        if (rsp.op_ret && strcmp (rsp.op_errstr, ""))
-                snprintf (msg, sizeof (msg), "%s", rsp.op_errstr);
-        else
-                snprintf (msg, sizeof (msg), "Detach tier %s %s", cmd_str,
-                          (rsp.op_ret) ? "unsuccessful" : "successful");
-
-
-        if (global_state->mode & GLUSTER_MODE_XML) {
-                ret = cli_xml_output_vol_remove_brick_detach_tier (
-                                                       _gf_false, rsp_dict,
-                                                       rsp.op_ret, rsp.op_errno,
-                                                       msg, "volDetachTier");
+    return debug_xlator;
+}
 
-                if (ret)
-                        gf_log ("cli", GF_LOG_ERROR,
-                                "Error outputting to xml");
-                goto out;
-        }
+static int
+gf_cli_set_volume_cbk(struct rpc_req *req, struct iovec *iov, int count,
+                      void *myframe)
+{
+    gf_cli_rsp rsp = {
+        0,
+    };
+    int ret = -1;
+    dict_t *dict = NULL;
+    char *help_str = NULL;
+    char msg[1024] = {
+        0,
+    };
+    char *debug_xlator = NULL;
+    char tmp_str[512] = {
+        0,
+    };
+
+    GF_ASSERT(myframe);
+
+    if (-1 == req->rpc_status) {
+        goto out;
+    }
+
+    ret = xdr_to_generic(*iov, &rsp, (xdrproc_t)xdr_gf_cli_rsp);
+    if (ret < 0) {
+        gf_log(((call_frame_t *)myframe)->this->name, GF_LOG_ERROR,
+               XDR_DECODE_FAIL);
+        goto out;
+    }
+
+    gf_log("cli", GF_LOG_INFO, "Received resp to set");
+
+    dict = dict_new();
+
+    if (!dict) {
+        ret = -1;
+        goto out;
+    }
+
+    ret = dict_unserialize(rsp.dict.dict_val, rsp.dict.dict_len, &dict);
+    if (ret) {
+        gf_log("cli", GF_LOG_ERROR, DICT_UNSERIALIZE_FAIL);
+        goto out;
+    }
+
+    /* For brick processes graph change does not happen on the fly.
+     * The process has to be restarted. So this is a check from the
+     * volume set option such that if debug xlators such as trace/errorgen
+     * are provided in the set command, warn the user.
+     */
+    debug_xlator = is_server_debug_xlator(myframe);
+
+    if (dict_get_str_sizen(dict, "help-str", &help_str) && !msg[0])
+        snprintf(msg, sizeof(msg), "Set volume %s",
+                 (rsp.op_ret) ? "unsuccessful" : "successful");
+    if (rsp.op_ret == 0 && debug_xlator) {
+        snprintf(tmp_str, sizeof(tmp_str),
+                 "\n%s translator has been "
+                 "added to the server volume file. Please restart the"
+                 " volume for enabling the translator",
+                 debug_xlator);
+    }
+
+    if ((global_state->mode & GLUSTER_MODE_XML) && (help_str == NULL)) {
+        ret = cli_xml_output_str("volSet", msg, rsp.op_ret, rsp.op_errno,
+                                 rsp.op_errstr);
+        if (ret)
+            gf_log("cli", GF_LOG_ERROR, XML_ERROR);
+        goto out;
+    }
 
-        if (rsp.op_ret) {
-                cli_err ("volume detach-tier %s: failed: %s", cmd_str,
-                         msg);
+    if (rsp.op_ret) {
+        if (strcmp(rsp.op_errstr, ""))
+            cli_err("volume set: failed: %s", rsp.op_errstr);
+        else
+            cli_err("volume set: failed");
+    } else {
+        if (help_str == NULL) {
+            if (debug_xlator == NULL)
+                cli_out("volume set: success");
+            else
+                cli_out("volume set: success%s", tmp_str);
         } else {
-                cli_out ("volume detach-tier %s: success", cmd_str);
-                if (GF_OP_CMD_DETACH_START == cmd && task_id_str != NULL)
-                        cli_out ("ID: %s", task_id_str);
-                if (GF_OP_CMD_DETACH_COMMIT == cmd)
-                        cli_out ("Check the detached bricks to ensure all files"
-                                 " are migrated.\nIf files with data are "
-                                 "found on the brick path, copy them via a "
-                                 "gluster mount point before re-purposing the "
-                                 "removed brick. ");
+            cli_out("%s", help_str);
         }
+    }
 
-        ret = rsp.op_ret;
+    ret = rsp.op_ret;
 
 out:
-        cli_cmd_broadcast_response (ret);
-        free (rsp.dict.dict_val);
-        free (rsp.op_errstr);
-
-        return ret;
+    if (dict)
+        dict_unref(dict);
+    GF_FREE(debug_xlator);
+    cli_cmd_broadcast_response(ret);
+    gf_free_xdr_cli_rsp(rsp);
+    return ret;
 }
 
 int
-gf_cli_detach_tier_status_cbk (struct rpc_req *req, struct iovec *iov,
-                                 int count, void *myframe)
-{
-        gf_cli_rsp               rsp     = {0,};
-        int                      ret     = -1;
-        dict_t                  *dict    = NULL;
-        char                     msg[1024] = {0,};
-        int32_t                  command = 0;
-        gf1_op_commands          cmd = GF_OP_CMD_NONE;
-        cli_local_t             *local = NULL;
-        call_frame_t            *frame = NULL;
-        char                    *cmd_str = "unknown";
-
-        if (-1 == req->rpc_status) {
-                goto out;
-        }
-
-        frame = myframe;
-
-        ret = xdr_to_generic (*iov, &rsp,
-                              (xdrproc_t)xdr_gf_cli_rsp);
-        if (ret < 0) {
-                gf_log (frame->this->name, GF_LOG_ERROR,
-                        "Failed to decode xdr response");
-                goto out;
-        }
-
-        if (frame)
-                local = frame->local;
-        ret = dict_get_int32 (local->dict, "command", &command);
+gf_cli_add_brick_cbk(struct rpc_req *req, struct iovec *iov, int count,
+                     void *myframe)
+{
+    gf_cli_rsp rsp = {
+        0,
+    };
+    int ret = -1;
+    char msg[1024] = {
+        0,
+    };
+
+    GF_ASSERT(myframe);
+
+    if (-1 == req->rpc_status) {
+        goto out;
+    }
+
+    ret = xdr_to_generic(*iov, &rsp, (xdrproc_t)xdr_gf_cli_rsp);
+    if (ret < 0) {
+        gf_log(((call_frame_t *)myframe)->this->name, GF_LOG_ERROR,
+               XDR_DECODE_FAIL);
+        goto out;
+    }
+
+    gf_log("cli", GF_LOG_INFO, "Received resp to add brick");
+
+    if (rsp.op_ret && strcmp(rsp.op_errstr, ""))
+        snprintf(msg, sizeof(msg), "%s", rsp.op_errstr);
+    else
+        snprintf(msg, sizeof(msg), "Add Brick %s",
+                 (rsp.op_ret) ? "unsuccessful" : "successful");
+
+    if (global_state->mode & GLUSTER_MODE_XML) {
+        ret = cli_xml_output_str("volAddBrick", msg, rsp.op_ret, rsp.op_errno,
+                                 rsp.op_errstr);
         if (ret)
-                goto out;
-        cmd = command;
+            gf_log("cli", GF_LOG_ERROR, XML_ERROR);
+        goto out;
+    }
 
-        switch (cmd) {
-        case GF_OP_CMD_STOP_DETACH_TIER:
-                cmd_str = "stop";
-                break;
-        case GF_OP_CMD_STATUS:
-                cmd_str = "status";
-                break;
-        default:
-                break;
-        }
-
-        ret = rsp.op_ret;
-        if (rsp.op_ret == -1) {
-                if (strcmp (rsp.op_errstr, ""))
-                        snprintf (msg, sizeof (msg), "volume tier detach %s: "
-                                  "failed: %s", cmd_str, rsp.op_errstr);
-                else
-                        snprintf (msg, sizeof (msg), "volume tier detach %s: "
-                                  "failed", cmd_str);
-
-                if (global_state->mode & GLUSTER_MODE_XML)
-                        goto xml_output;
-
-                cli_err ("%s", msg);
-                goto out;
-        }
-
-        if (rsp.dict.dict_len) {
-                /* Unserialize the dictionary */
-                dict  = dict_new ();
-
-                ret = dict_unserialize (rsp.dict.dict_val,
-                                        rsp.dict.dict_len,
-                                        &dict);
-                if (ret < 0) {
-                        strncpy (msg, "failed to unserialize req-buffer to "
-                                 "dictionary", sizeof (msg));
-
-                        if (global_state->mode & GLUSTER_MODE_XML) {
-                                rsp.op_ret = -1;
-                                goto xml_output;
-                        }
-
-                        gf_log ("cli", GF_LOG_ERROR, "%s", msg);
-                        goto out;
-                }
-        }
-xml_output:
-        if (global_state->mode & GLUSTER_MODE_XML) {
-                if (strcmp (rsp.op_errstr, "")) {
-                        ret = cli_xml_output_vol_remove_brick_detach_tier (
-                                                               _gf_true, dict,
-                                                               rsp.op_ret,
-                                                               rsp.op_errno,
-                                                               rsp.op_errstr,
-                                                               "volDetachTier");
-                } else {
-                        ret = cli_xml_output_vol_remove_brick_detach_tier
-                                                               (_gf_true, dict,
-                                                               rsp.op_ret,
-                                                               rsp.op_errno,
-                                                               msg,
-                                                               "volDetachTier");
-                }
-                goto out;
-        }
-
-        ret = gf_cli_print_rebalance_status (dict, GF_TASK_TYPE_REMOVE_BRICK);
-        if (ret) {
-                gf_log ("cli", GF_LOG_ERROR, "Failed to print remove-brick "
-                        "rebalance status");
-                goto out;
-        }
-
-        if ((cmd == GF_OP_CMD_STOP_DETACH_TIER) && (rsp.op_ret == 0)) {
-                cli_out ("'detach-tier' process may be in the middle of a "
-                         "file migration.\nThe process will be fully stopped "
-                         "once the migration of the file is complete.\nPlease "
-                         "check detach-tier process for completion before "
-                         "doing any further brick related tasks on the "
-                         "volume.");
-        }
+    if (rsp.op_ret)
+        cli_err("volume add-brick: failed: %s", rsp.op_errstr);
+    else
+        cli_out("volume add-brick: success");
+    ret = rsp.op_ret;
 
 out:
-        free (rsp.dict.dict_val); /* malloced by xdr */
-        if (dict)
-                dict_unref (dict);
-        cli_cmd_broadcast_response (ret);
-        return ret;
+    cli_cmd_broadcast_response(ret);
+    gf_free_xdr_cli_rsp(rsp);
+    return ret;
 }
 
-
-int
-gf_cli_add_brick_cbk (struct rpc_req *req, struct iovec *iov,
-                             int count, void *myframe)
+static int
+gf_cli3_remove_brick_status_cbk(struct rpc_req *req, struct iovec *iov,
+                                int count, void *myframe)
 {
-        gf_cli_rsp                  rsp   = {0,};
-        int                         ret   = -1;
-        char                        msg[1024] = {0,};
+    gf_cli_rsp rsp = {
+        0,
+    };
+    int ret = -1;
+    dict_t *dict = NULL;
+    char msg[1024] = {
+        0,
+    };
+    int32_t command = 0;
+    gf1_op_commands cmd = GF_OP_CMD_NONE;
+    cli_local_t *local = NULL;
+    call_frame_t *frame = NULL;
+    const char *cmd_str;
 
-        if (-1 == req->rpc_status) {
-                goto out;
-        }
+    GF_ASSERT(myframe);
 
-        ret = xdr_to_generic (*iov, &rsp, (xdrproc_t)xdr_gf_cli_rsp);
-        if (ret < 0) {
-                gf_log (((call_frame_t *) myframe)->this->name, GF_LOG_ERROR,
-                        "Failed to decode xdr response");
-                goto out;
-        }
-
-
-        gf_log ("cli", GF_LOG_INFO, "Received resp to add brick");
-
-        if (rsp.op_ret && strcmp (rsp.op_errstr, ""))
-                snprintf (msg, sizeof (msg), "%s", rsp.op_errstr);
-        else
-                snprintf (msg, sizeof (msg), "Add Brick %s",
-                          (rsp.op_ret) ? "unsuccessful": "successful");
-
-        if (global_state->mode & GLUSTER_MODE_XML) {
-                ret = cli_xml_output_str ("volAddBrick", msg, rsp.op_ret,
-                                          rsp.op_errno, rsp.op_errstr);
-                if (ret)
-                        gf_log ("cli", GF_LOG_ERROR,
-                                "Error outputting to xml");
-                goto out;
-        }
+    if (-1 == req->rpc_status) {
+        goto out;
+    }
 
-        if (rsp.op_ret)
-                cli_err ("volume add-brick: failed: %s", rsp.op_errstr);
-        else
-                cli_out ("volume add-brick: success");
-        ret = rsp.op_ret;
+    frame = myframe;
 
-out:
-        cli_cmd_broadcast_response (ret);
-        free (rsp.dict.dict_val);
-        free (rsp.op_errstr);
-        return ret;
-}
+    GF_ASSERT(frame->local);
 
-int
-gf_cli3_remove_brick_status_cbk (struct rpc_req *req, struct iovec *iov,
-                                 int count, void *myframe)
-{
-        gf_cli_rsp               rsp     = {0,};
-        int                      ret     = -1;
-        dict_t                  *dict    = NULL;
-        char                     msg[1024] = {0,};
-        int32_t                  command = 0;
-        gf1_op_commands          cmd = GF_OP_CMD_NONE;
-        cli_local_t             *local = NULL;
-        call_frame_t            *frame = NULL;
-        char                    *cmd_str = "unknown";
-
-        if (-1 == req->rpc_status) {
-                goto out;
-        }
+    local = frame->local;
 
-        frame = myframe;
+    ret = xdr_to_generic(*iov, &rsp, (xdrproc_t)xdr_gf_cli_rsp);
+    if (ret < 0) {
+        gf_log(frame->this->name, GF_LOG_ERROR, XDR_DECODE_FAIL);
+        goto out;
+    }
 
-        ret = xdr_to_generic (*iov, &rsp,
-                              (xdrproc_t)xdr_gf_cli_rsp);
-        if (ret < 0) {
-                gf_log (frame->this->name, GF_LOG_ERROR,
-                        "Failed to decode xdr response");
-                goto out;
-        }
+    ret = dict_get_int32_sizen(local->dict, "command", &command);
+    if (ret)
+        goto out;
 
-        if (frame)
-                local = frame->local;
-        ret = dict_get_int32 (local->dict, "command", &command);
-        if (ret)
-                goto out;
-        cmd = command;
+    cmd = command;
 
-        switch (cmd) {
+    switch (cmd) {
         case GF_OP_CMD_STOP:
-                cmd_str = "stop";
-                break;
+            cmd_str = "stop";
+            break;
         case GF_OP_CMD_STATUS:
-                cmd_str = "status";
-                break;
+            cmd_str = "status";
+            break;
         default:
-                break;
-        }
-
-        ret = rsp.op_ret;
-        if (rsp.op_ret == -1) {
-                if (strcmp (rsp.op_errstr, ""))
-                        snprintf (msg, sizeof (msg), "volume remove-brick %s: "
-                                  "failed: %s", cmd_str, rsp.op_errstr);
-                else
-                        snprintf (msg, sizeof (msg), "volume remove-brick %s: "
-                                  "failed", cmd_str);
+            cmd_str = "unknown";
+            break;
+    }
+
+    ret = rsp.op_ret;
+    if (rsp.op_ret == -1) {
+        if (strcmp(rsp.op_errstr, ""))
+            snprintf(msg, sizeof(msg), "volume remove-brick %s: failed: %s",
+                     cmd_str, rsp.op_errstr);
+        else
+            snprintf(msg, sizeof(msg), "volume remove-brick %s: failed",
+                     cmd_str);
 
-                if (global_state->mode & GLUSTER_MODE_XML)
-                        goto xml_output;
+        if (global_state->mode & GLUSTER_MODE_XML)
+            goto xml_output;
 
-                cli_err ("%s", msg);
-                goto out;
-        }
+        cli_err("%s", msg);
+        goto out;
+    }
 
-        if (rsp.dict.dict_len) {
-                /* Unserialize the dictionary */
-                dict  = dict_new ();
+    if (rsp.dict.dict_len) {
+        /* Unserialize the dictionary */
+        dict = dict_new();
 
-                ret = dict_unserialize (rsp.dict.dict_val,
-                                        rsp.dict.dict_len,
-                                        &dict);
-                if (ret < 0) {
-                        strncpy (msg, "failed to unserialize req-buffer to "
-                                 "dictionary", sizeof (msg));
+        ret = dict_unserialize(rsp.dict.dict_val, rsp.dict.dict_len, &dict);
+        if (ret < 0) {
+            strncpy(msg, DICT_UNSERIALIZE_FAIL, sizeof(msg));
 
-                        if (global_state->mode & GLUSTER_MODE_XML) {
-                                rsp.op_ret = -1;
-                                goto xml_output;
-                        }
+            if (global_state->mode & GLUSTER_MODE_XML) {
+                rsp.op_ret = -1;
+                goto xml_output;
+            }
 
-                        gf_log ("cli", GF_LOG_ERROR, "%s", msg);
-                        goto out;
-                }
+            gf_log("cli", GF_LOG_ERROR, "%s", msg);
+            goto out;
         }
+    }
 
 xml_output:
-        if (global_state->mode & GLUSTER_MODE_XML) {
-                if (strcmp (rsp.op_errstr, "")) {
-                        ret = cli_xml_output_vol_remove_brick_detach_tier (
-                                                               _gf_true, dict,
-                                                               rsp.op_ret,
-                                                               rsp.op_errno,
-                                                               rsp.op_errstr,
-                                                               "volRemoveBrick");
-                } else {
-                        ret = cli_xml_output_vol_remove_brick_detach_tier (
-                                                               _gf_true, dict,
-                                                               rsp.op_ret,
-                                                               rsp.op_errno,
-                                                               msg,
-                                                               "volRemoveBrick");
-                }
-                goto out;
-        }
+    if (global_state->mode & GLUSTER_MODE_XML) {
+        if (strcmp(rsp.op_errstr, "")) {
+            ret = cli_xml_output_vol_remove_brick(_gf_true, dict, rsp.op_ret,
+                                                  rsp.op_errno, rsp.op_errstr,
+                                                  "volRemoveBrick");
+        } else {
+            ret = cli_xml_output_vol_remove_brick(_gf_true, dict, rsp.op_ret,
+                                                  rsp.op_errno, msg,
+                                                  "volRemoveBrick");
+        }
+        goto out;
+    }
+
+    ret = gf_cli_print_rebalance_status(dict, GF_TASK_TYPE_REMOVE_BRICK);
+    if (ret) {
+        gf_log("cli", GF_LOG_ERROR,
+               "Failed to print remove-brick rebalance status");
+        goto out;
+    }
+
+    if ((cmd == GF_OP_CMD_STOP) && (rsp.op_ret == 0)) {
+        cli_out(
+            "'remove-brick' process may be in the middle of a "
+            "file migration.\nThe process will be fully stopped "
+            "once the migration of the file is complete.\nPlease "
+            "check remove-brick process for completion before "
+            "doing any further brick related tasks on the "
+            "volume.");
+    }
+
+out:
+    if (dict)
+        dict_unref(dict);
+    cli_cmd_broadcast_response(ret);
+    gf_free_xdr_cli_rsp(rsp);
+    return ret;
+}
 
-        ret = gf_cli_print_rebalance_status (dict, GF_TASK_TYPE_REMOVE_BRICK);
-        if (ret) {
-                gf_log ("cli", GF_LOG_ERROR, "Failed to print remove-brick "
-                        "rebalance status");
-                goto out;
-        }
+static int
+gf_cli_remove_brick_cbk(struct rpc_req *req, struct iovec *iov, int count,
+                        void *myframe)
+{
+    gf_cli_rsp rsp = {
+        0,
+    };
+    int ret = -1;
+    char msg[1024] = {
+        0,
+    };
+    gf1_op_commands cmd = GF_OP_CMD_NONE;
+    char *cmd_str = "unknown";
+    cli_local_t *local = NULL;
+    call_frame_t *frame = NULL;
+    char *task_id_str = NULL;
+    dict_t *rsp_dict = NULL;
 
-        if ((cmd == GF_OP_CMD_STOP) && (rsp.op_ret == 0)) {
-                cli_out ("'remove-brick' process may be in the middle of a "
-                         "file migration.\nThe process will be fully stopped "
-                         "once the migration of the file is complete.\nPlease "
-                         "check remove-brick process for completion before "
-                         "doing any further brick related tasks on the "
-                         "volume.");
-        }
+    GF_ASSERT(myframe);
 
-out:
-        free (rsp.dict.dict_val); //malloced by xdr
-        if (dict)
-                dict_unref (dict);
-        cli_cmd_broadcast_response (ret);
-        return ret;
-}
+    if (-1 == req->rpc_status) {
+        goto out;
+    }
 
+    frame = myframe;
 
-int
-gf_cli_remove_brick_cbk (struct rpc_req *req, struct iovec *iov,
-                             int count, void *myframe)
-{
-        gf_cli_rsp                      rsp   = {0,};
-        int                             ret   = -1;
-        char                            msg[1024] = {0,};
-        gf1_op_commands                 cmd = GF_OP_CMD_NONE;
-        char                           *cmd_str = "unknown";
-        cli_local_t                    *local = NULL;
-        call_frame_t                   *frame = NULL;
-        char                           *task_id_str = NULL;
-        dict_t                         *rsp_dict = NULL;
-
-        if (-1 == req->rpc_status) {
-                goto out;
-        }
+    GF_ASSERT(frame->local);
 
-        frame = myframe;
-        local = frame->local;
+    local = frame->local;
 
-        ret = xdr_to_generic (*iov, &rsp, (xdrproc_t)xdr_gf_cli_rsp);
-        if (ret < 0) {
-                gf_log (frame->this->name, GF_LOG_ERROR,
-                        "Failed to decode xdr response");
-                goto out;
-        }
+    ret = xdr_to_generic(*iov, &rsp, (xdrproc_t)xdr_gf_cli_rsp);
+    if (ret < 0) {
+        gf_log(frame->this->name, GF_LOG_ERROR, XDR_DECODE_FAIL);
+        goto out;
+    }
 
-        ret = dict_get_int32 (local->dict, "command", (int32_t *)&cmd);
-        if (ret) {
-                 gf_log ("", GF_LOG_ERROR, "failed to get command");
-                 goto out;
-        }
+    ret = dict_get_int32_sizen(local->dict, "command", (int32_t *)&cmd);
+    if (ret) {
+        gf_log("", GF_LOG_ERROR, "failed to get command");
+        goto out;
+    }
 
-        if (rsp.dict.dict_len) {
-                rsp_dict = dict_new ();
-                if (!rsp_dict) {
-                        ret = -1;
-                        goto out;
-                }
+    if (rsp.dict.dict_len) {
+        rsp_dict = dict_new();
+        if (!rsp_dict) {
+            ret = -1;
+            goto out;
+        }
 
-                ret = dict_unserialize (rsp.dict.dict_val, rsp.dict.dict_len,
-                                        &rsp_dict);
-                if (ret) {
-                        gf_log ("cli", GF_LOG_ERROR,
-                                "Failed to unserialize rsp_dict");
-                        goto out;
-                }
+        ret = dict_unserialize(rsp.dict.dict_val, rsp.dict.dict_len, &rsp_dict);
+        if (ret) {
+            gf_log("cli", GF_LOG_ERROR, DICT_UNSERIALIZE_FAIL);
+            goto out;
         }
+    }
 
-        switch (cmd) {
+    switch (cmd) {
         case GF_OP_CMD_DETACH_START:
         case GF_OP_CMD_START:
-                cmd_str = "start";
-
-                ret = dict_get_str (rsp_dict, GF_REMOVE_BRICK_TID_KEY, &task_id_str);
-                if (ret) {
-                        gf_log ("cli", GF_LOG_ERROR,
-                                "remove-brick-id is not present in dict");
-                }
-                break;
+            cmd_str = "start";
+
+            ret = dict_get_str_sizen(rsp_dict, GF_REMOVE_BRICK_TID_KEY,
+                                     &task_id_str);
+            if (ret) {
+                gf_log("cli", GF_LOG_ERROR,
+                       "remove-brick-id is not present in dict");
+            }
+            break;
         case GF_OP_CMD_COMMIT:
-                cmd_str = "commit";
-                break;
+            cmd_str = "commit";
+            break;
         case GF_OP_CMD_COMMIT_FORCE:
-                cmd_str = "commit force";
-                break;
+            cmd_str = "commit force";
+            break;
         default:
-                cmd_str = "unknown";
-                break;
-        }
+            cmd_str = "unknown";
+            break;
+    }
+
+    gf_log("cli", GF_LOG_INFO, "Received resp to remove brick");
+
+    if (rsp.op_ret && strcmp(rsp.op_errstr, ""))
+        snprintf(msg, sizeof(msg), "%s", rsp.op_errstr);
+    else
+        snprintf(msg, sizeof(msg), "Remove Brick %s %s", cmd_str,
+                 (rsp.op_ret) ? "unsuccessful" : "successful");
+
+    if (global_state->mode & GLUSTER_MODE_XML) {
+        ret = cli_xml_output_vol_remove_brick(_gf_false, rsp_dict, rsp.op_ret,
+                                              rsp.op_errno, msg,
+                                              "volRemoveBrick");
+        if (ret)
+            gf_log("cli", GF_LOG_ERROR, XML_ERROR);
+        goto out;
+    }
 
-        gf_log ("cli", GF_LOG_INFO, "Received resp to remove brick");
+    if (rsp.op_ret) {
+        cli_err("volume remove-brick %s: failed: %s", cmd_str, msg);
+    } else {
+        cli_out("volume remove-brick %s: success", cmd_str);
+        if (GF_OP_CMD_START == cmd && task_id_str != NULL)
+            cli_out("ID: %s", task_id_str);
+        if (GF_OP_CMD_COMMIT == cmd)
+            cli_out(
+                "Check the removed bricks to ensure all files "
+                "are migrated.\nIf files with data are "
+                "found on the brick path, copy them via a "
+                "gluster mount point before re-purposing the "
+                "removed brick. ");
+    }
 
-        if (rsp.op_ret && strcmp (rsp.op_errstr, ""))
-                snprintf (msg, sizeof (msg), "%s", rsp.op_errstr);
-        else
-                snprintf (msg, sizeof (msg), "Remove Brick %s %s", cmd_str,
-                          (rsp.op_ret) ? "unsuccessful": "successful");
-
-        if (global_state->mode & GLUSTER_MODE_XML) {
-                ret = cli_xml_output_vol_remove_brick_detach_tier (
-                                                       _gf_false, rsp_dict,
-                                                       rsp.op_ret, rsp.op_errno,
-                                                       msg, "volRemoveBrick");
-                if (ret)
-                        gf_log ("cli", GF_LOG_ERROR,
-                                "Error outputting to xml");
-                goto out;
-        }
-
-        if (rsp.op_ret) {
-                cli_err ("volume remove-brick %s: failed: %s", cmd_str,
-                         msg);
-        } else {
-                cli_out ("volume remove-brick %s: success", cmd_str);
-                if (GF_OP_CMD_START == cmd && task_id_str != NULL)
-                        cli_out ("ID: %s", task_id_str);
-                if (GF_OP_CMD_COMMIT == cmd)
-                        cli_out ("Check the removed bricks to ensure all files "
-                                 "are migrated.\nIf files with data are "
-                                 "found on the brick path, copy them via a "
-                                 "gluster mount point before re-purposing the "
-                                 "removed brick. ");
-        }
-
-        ret = rsp.op_ret;
+    ret = rsp.op_ret;
 
 out:
-        cli_cmd_broadcast_response (ret);
-        free (rsp.dict.dict_val);
-        free (rsp.op_errstr);
-
-        return ret;
-}
+    cli_cmd_broadcast_response(ret);
+    gf_free_xdr_cli_rsp(rsp);
 
-int
-gf_cli_replace_brick_cbk (struct rpc_req *req, struct iovec *iov,
-                             int count, void *myframe)
-{
-        gf_cli_rsp                       rsp              = {0,};
-        int                              ret              = -1;
-        cli_local_t                     *local            = NULL;
-        call_frame_t                    *frame            = NULL;
-        dict_t                          *dict             = NULL;
-        char                            *src_brick        = NULL;
-        char                            *dst_brick        = NULL;
-        char                            *status_reply     = NULL;
-        char                            *rb_operation_str = NULL;
-        dict_t                          *rsp_dict         = NULL;
-        char                             msg[1024]        = {0,};
-        char                            *task_id_str      = NULL;
-        char                            *replace_op       = 0;
-
-        if (-1 == req->rpc_status) {
-                goto out;
-        }
+    if (rsp_dict)
+        dict_unref(rsp_dict);
 
-        frame = (call_frame_t *) myframe;
+    return ret;
+}
 
-        ret = xdr_to_generic (*iov, &rsp, (xdrproc_t)xdr_gf_cli_rsp);
+static int
+gf_cli_reset_brick_cbk(struct rpc_req *req, struct iovec *iov, int count,
+                       void *myframe)
+{
+    gf_cli_rsp rsp = {
+        0,
+    };
+    int ret = -1;
+    cli_local_t *local = NULL;
+    call_frame_t *frame = NULL;
+    const char *rb_operation_str = NULL;
+    dict_t *rsp_dict = NULL;
+    char msg[1024] = {
+        0,
+    };
+    char *reset_op = NULL;
+
+    GF_ASSERT(myframe);
+
+    if (-1 == req->rpc_status) {
+        goto out;
+    }
+
+    frame = myframe;
+
+    GF_ASSERT(frame->local);
+
+    local = frame->local;
+
+    ret = xdr_to_generic(*iov, &rsp, (xdrproc_t)xdr_gf_cli_rsp);
+    if (ret < 0) {
+        gf_log(frame->this->name, GF_LOG_ERROR, XDR_DECODE_FAIL);
+        goto out;
+    }
+
+    ret = dict_get_str_sizen(local->dict, "operation", &reset_op);
+    if (ret) {
+        gf_log(frame->this->name, GF_LOG_ERROR, "dict_get on operation failed");
+        goto out;
+    }
+
+    if (strcmp(reset_op, "GF_RESET_OP_START") &&
+        strcmp(reset_op, "GF_RESET_OP_COMMIT") &&
+        strcmp(reset_op, "GF_RESET_OP_COMMIT_FORCE")) {
+        ret = -1;
+        goto out;
+    }
+
+    if (rsp.dict.dict_len) {
+        /* Unserialize the dictionary */
+        rsp_dict = dict_new();
+
+        ret = dict_unserialize(rsp.dict.dict_val, rsp.dict.dict_len, &rsp_dict);
         if (ret < 0) {
-                gf_log (frame->this->name, GF_LOG_ERROR,
-                        "Failed to decode xdr response");
-                goto out;
-        }
-
-        local = frame->local;
-        GF_ASSERT (local);
-        dict = local->dict;
-
-        ret = dict_get_str (dict, "operation", &replace_op);
-        if (ret) {
-                gf_log (frame->this->name, GF_LOG_ERROR,
-                        "dict_get on operation failed");
-                goto out;
-        }
-
-        if (rsp.dict.dict_len) {
-                /* Unserialize the dictionary */
-                rsp_dict  = dict_new ();
-
-                ret = dict_unserialize (rsp.dict.dict_val,
-                                rsp.dict.dict_len,
-                                &rsp_dict);
-                if (ret < 0) {
-                        gf_log (frame->this->name, GF_LOG_ERROR, "failed to "
-                                "unserialize rsp buffer to dictionary");
-                        goto out;
-                }
-        }
-
-        if (!strcmp(replace_op, "GF_REPLACE_OP_COMMIT_FORCE")) {
-
-                if (rsp.op_ret || ret)
-                        rb_operation_str = gf_strdup ("replace-brick commit "
-                                                      "force operation failed");
-                else
-                        rb_operation_str = gf_strdup ("replace-brick commit "
-                                                      "force operation "
-                                                      "successful");
-        } else {
-                gf_log (frame->this->name, GF_LOG_DEBUG, "Unknown operation");
-        }
-
-        if (rsp.op_ret && (strcmp (rsp.op_errstr, ""))) {
-                rb_operation_str = gf_strdup (rsp.op_errstr);
-        }
-
-        gf_log ("cli", GF_LOG_INFO, "Received resp to replace brick");
-        snprintf (msg, sizeof (msg), "%s",
-                  rb_operation_str ? rb_operation_str : "Unknown operation");
-
-        if (global_state->mode & GLUSTER_MODE_XML) {
-                ret = cli_xml_output_vol_replace_brick (replace_op, rsp_dict,
-                                                        rsp.op_ret,
-                                                        rsp.op_errno, msg);
-                if (ret)
-                        gf_log ("cli", GF_LOG_ERROR,
-                                "Error outputting to xml");
-                goto out;
-        }
+            gf_log(frame->this->name, GF_LOG_ERROR, DICT_UNSERIALIZE_FAIL);
+            goto out;
+        }
+    }
+
+    if (rsp.op_ret && (strcmp(rsp.op_errstr, ""))) {
+        rb_operation_str = rsp.op_errstr;
+    } else {
+        if (!strcmp(reset_op, "GF_RESET_OP_START")) {
+            if (rsp.op_ret)
+                rb_operation_str = "reset-brick start operation failed";
+            else
+                rb_operation_str = "reset-brick start operation successful";
+        } else if (!strcmp(reset_op, "GF_RESET_OP_COMMIT")) {
+            if (rsp.op_ret)
+                rb_operation_str = "reset-brick commit operation failed";
+            else
+                rb_operation_str = "reset-brick commit operation successful";
+        } else if (!strcmp(reset_op, "GF_RESET_OP_COMMIT_FORCE")) {
+            if (rsp.op_ret)
+                rb_operation_str = "reset-brick commit force operation failed";
+            else
+                rb_operation_str =
+                    "reset-brick commit force operation successful";
+        }
+    }
+
+    gf_log("cli", GF_LOG_INFO, "Received resp to reset brick");
+    snprintf(msg, sizeof(msg), "%s",
+             rb_operation_str ? rb_operation_str : "Unknown operation");
+
+    if (global_state->mode & GLUSTER_MODE_XML) {
+        ret = cli_xml_output_vol_replace_brick(rsp_dict, rsp.op_ret,
+                                               rsp.op_errno, msg);
+        if (ret)
+            gf_log("cli", GF_LOG_ERROR, XML_ERROR);
+        goto out;
+    }
 
-        if (rsp.op_ret)
-                cli_err ("volume replace-brick: failed: %s", msg);
-        else
-                cli_out ("volume replace-brick: success: %s", msg);
-        ret = rsp.op_ret;
+    if (rsp.op_ret)
+        cli_err("volume reset-brick: failed: %s", msg);
+    else
+        cli_out("volume reset-brick: success: %s", msg);
+    ret = rsp.op_ret;
 
 out:
-        if (frame)
-                frame->local = NULL;
-
-        if (local) {
-                dict_unref (local->dict);
-                cli_local_wipe (local);
-        }
+    if (frame)
+        frame->local = NULL;
 
-        if (rb_operation_str)
-                GF_FREE (rb_operation_str);
+    if (local)
+        cli_local_wipe(local);
 
-        cli_cmd_broadcast_response (ret);
-        free (rsp.dict.dict_val);
-        if (rsp_dict)
-                dict_unref (rsp_dict);
+    cli_cmd_broadcast_response(ret);
+    gf_free_xdr_cli_rsp(rsp);
+    if (rsp_dict)
+        dict_unref(rsp_dict);
 
-        return ret;
+    return ret;
 }
-
-
 static int
-gf_cli_log_rotate_cbk (struct rpc_req *req, struct iovec *iov,
-                          int count, void *myframe)
-{
-        gf_cli_rsp             rsp   = {0,};
-        int                    ret   = -1;
-        char                   msg[1024] = {0,};
-
-        if (-1 == req->rpc_status) {
-                goto out;
-        }
-
-        ret = xdr_to_generic (*iov, &rsp, (xdrproc_t)xdr_gf_cli_rsp);
+gf_cli_replace_brick_cbk(struct rpc_req *req, struct iovec *iov, int count,
+                         void *myframe)
+{
+    gf_cli_rsp rsp = {
+        0,
+    };
+    int ret = -1;
+    cli_local_t *local = NULL;
+    call_frame_t *frame = NULL;
+    const char *rb_operation_str = NULL;
+    dict_t *rsp_dict = NULL;
+    char msg[1024] = {
+        0,
+    };
+    char *replace_op = NULL;
+
+    GF_ASSERT(myframe);
+
+    if (-1 == req->rpc_status) {
+        goto out;
+    }
+
+    frame = myframe;
+
+    GF_ASSERT(frame->local);
+
+    local = frame->local;
+
+    ret = xdr_to_generic(*iov, &rsp, (xdrproc_t)xdr_gf_cli_rsp);
+    if (ret < 0) {
+        gf_log(frame->this->name, GF_LOG_ERROR, XDR_DECODE_FAIL);
+        goto out;
+    }
+
+    ret = dict_get_str_sizen(local->dict, "operation", &replace_op);
+    if (ret) {
+        gf_log(frame->this->name, GF_LOG_ERROR, "dict_get on operation failed");
+        goto out;
+    }
+
+    if (rsp.dict.dict_len) {
+        /* Unserialize the dictionary */
+        rsp_dict = dict_new();
+
+        ret = dict_unserialize(rsp.dict.dict_val, rsp.dict.dict_len, &rsp_dict);
         if (ret < 0) {
-                gf_log (((call_frame_t *) myframe)->this->name, GF_LOG_ERROR,
-                        "Failed to decode xdr response");
-                goto out;
+            gf_log(frame->this->name, GF_LOG_ERROR, DICT_UNSERIALIZE_FAIL);
+            goto out;
         }
+    }
 
-        gf_log ("cli", GF_LOG_DEBUG, "Received resp to log rotate");
-
-        if (rsp.op_ret && strcmp (rsp.op_errstr, ""))
-                snprintf (msg, sizeof (msg), "%s", rsp.op_errstr);
+    if (!strcmp(replace_op, "GF_REPLACE_OP_COMMIT_FORCE")) {
+        if (rsp.op_ret || ret)
+            rb_operation_str = "replace-brick commit force operation failed";
         else
-                snprintf (msg, sizeof (msg), "log rotate %s",
-                          (rsp.op_ret) ? "unsuccessful": "successful");
-
-        if (global_state->mode & GLUSTER_MODE_XML) {
-                ret = cli_xml_output_str ("volLogRotate", msg, rsp.op_ret,
-                                          rsp.op_errno, rsp.op_errstr);
-                if (ret)
-                        gf_log ("cli", GF_LOG_ERROR,
-                                "Error outputting to xml");
-                goto out;
-        }
+            rb_operation_str =
+                "replace-brick commit force operation successful";
+    } else {
+        gf_log(frame->this->name, GF_LOG_DEBUG, "Unknown operation");
+    }
+
+    if (rsp.op_ret && (strcmp(rsp.op_errstr, ""))) {
+        rb_operation_str = rsp.op_errstr;
+    }
+
+    gf_log("cli", GF_LOG_INFO, "Received resp to replace brick");
+    snprintf(msg, sizeof(msg), "%s",
+             rb_operation_str ? rb_operation_str : "Unknown operation");
+
+    if (global_state->mode & GLUSTER_MODE_XML) {
+        ret = cli_xml_output_vol_replace_brick(rsp_dict, rsp.op_ret,
+                                               rsp.op_errno, msg);
+        if (ret)
+            gf_log("cli", GF_LOG_ERROR, XML_ERROR);
+        goto out;
+    }
 
-        if (rsp.op_ret)
-                cli_err ("volume log-rotate: failed: %s", msg);
-        else
-                cli_out ("volume log-rotate: success");
-        ret = rsp.op_ret;
+    if (rsp.op_ret)
+        cli_err("volume replace-brick: failed: %s", msg);
+    else
+        cli_out("volume replace-brick: success: %s", msg);
+    ret = rsp.op_ret;
 
 out:
-        cli_cmd_broadcast_response (ret);
-        free (rsp.dict.dict_val);
+    if (frame)
+        frame->local = NULL;
 
-        return ret;
+    if (local)
+        cli_local_wipe(local);
+
+    cli_cmd_broadcast_response(ret);
+    gf_free_xdr_cli_rsp(rsp);
+    if (rsp_dict)
+        dict_unref(rsp_dict);
+
+    return ret;
 }
 
 static int
-gf_cli_sync_volume_cbk (struct rpc_req *req, struct iovec *iov,
-                           int count, void *myframe)
-{
-        gf_cli_rsp                     rsp   = {0,};
-        int                            ret   = -1;
-        char                           msg[1024] = {0,};
-
-        if (-1 == req->rpc_status) {
-                goto out;
-        }
+gf_cli_log_rotate_cbk(struct rpc_req *req, struct iovec *iov, int count,
+                      void *myframe)
+{
+    gf_cli_rsp rsp = {
+        0,
+    };
+    int ret = -1;
+    char msg[1024] = {
+        0,
+    };
+
+    GF_ASSERT(myframe);
+
+    if (-1 == req->rpc_status) {
+        goto out;
+    }
+
+    ret = xdr_to_generic(*iov, &rsp, (xdrproc_t)xdr_gf_cli_rsp);
+    if (ret < 0) {
+        gf_log(((call_frame_t *)myframe)->this->name, GF_LOG_ERROR,
+               XDR_DECODE_FAIL);
+        goto out;
+    }
+
+    gf_log("cli", GF_LOG_DEBUG, "Received resp to log rotate");
+
+    if (rsp.op_ret && strcmp(rsp.op_errstr, ""))
+        snprintf(msg, sizeof(msg), "%s", rsp.op_errstr);
+    else
+        snprintf(msg, sizeof(msg), "log rotate %s",
+                 (rsp.op_ret) ? "unsuccessful" : "successful");
+
+    if (global_state->mode & GLUSTER_MODE_XML) {
+        ret = cli_xml_output_str("volLogRotate", msg, rsp.op_ret, rsp.op_errno,
+                                 rsp.op_errstr);
+        if (ret)
+            gf_log("cli", GF_LOG_ERROR, XML_ERROR);
+        goto out;
+    }
 
-        ret = xdr_to_generic (*iov, &rsp, (xdrproc_t)xdr_gf_cli_rsp);
-        if (ret < 0) {
-                gf_log (((call_frame_t *) myframe)->this->name, GF_LOG_ERROR,
-                        "Failed to decode xdr response");
-                goto out;
-        }
+    if (rsp.op_ret)
+        cli_err("volume log-rotate: failed: %s", msg);
+    else
+        cli_out("volume log-rotate: success");
+    ret = rsp.op_ret;
 
-        gf_log ("cli", GF_LOG_DEBUG, "Received resp to sync");
+out:
+    cli_cmd_broadcast_response(ret);
+    gf_free_xdr_cli_rsp(rsp);
 
-        if (rsp.op_ret && strcmp (rsp.op_errstr, ""))
-                snprintf (msg, sizeof (msg), "volume sync: failed: %s",
-                          rsp.op_errstr);
-        else
-                snprintf (msg, sizeof (msg), "volume sync: %s",
-                          (rsp.op_ret) ? "failed": "success");
+    return ret;
+}
 
-        if (global_state->mode & GLUSTER_MODE_XML) {
-                ret = cli_xml_output_str ("volSync", msg, rsp.op_ret,
-                                          rsp.op_errno, rsp.op_errstr);
-                if (ret)
-                        gf_log ("cli", GF_LOG_ERROR,
-                                "Error outputting to xml");
-                goto out;
-        }
+static int
+gf_cli_sync_volume_cbk(struct rpc_req *req, struct iovec *iov, int count,
+                       void *myframe)
+{
+    gf_cli_rsp rsp = {
+        0,
+    };
+    int ret = -1;
+    char msg[1024] = {
+        0,
+    };
+
+    GF_ASSERT(myframe);
+
+    if (-1 == req->rpc_status) {
+        goto out;
+    }
+
+    ret = xdr_to_generic(*iov, &rsp, (xdrproc_t)xdr_gf_cli_rsp);
+    if (ret < 0) {
+        gf_log(((call_frame_t *)myframe)->this->name, GF_LOG_ERROR,
+               XDR_DECODE_FAIL);
+        goto out;
+    }
+
+    gf_log("cli", GF_LOG_DEBUG, "Received resp to sync");
+
+    if (rsp.op_ret && strcmp(rsp.op_errstr, ""))
+        snprintf(msg, sizeof(msg), "volume sync: failed: %s", rsp.op_errstr);
+    else
+        snprintf(msg, sizeof(msg), "volume sync: %s",
+                 (rsp.op_ret) ? "failed" : "success");
+
+    if (global_state->mode & GLUSTER_MODE_XML) {
+        ret = cli_xml_output_str("volSync", msg, rsp.op_ret, rsp.op_errno,
+                                 rsp.op_errstr);
+        if (ret)
+            gf_log("cli", GF_LOG_ERROR, XML_ERROR);
+        goto out;
+    }
 
-        if (rsp.op_ret)
-                cli_err ("%s", msg);
-        else
-                cli_out ("%s", msg);
-        ret = rsp.op_ret;
+    if (rsp.op_ret)
+        cli_err("%s", msg);
+    else
+        cli_out("%s", msg);
+    ret = rsp.op_ret;
 
 out:
-        cli_cmd_broadcast_response (ret);
-        return ret;
+    cli_cmd_broadcast_response(ret);
+    gf_free_xdr_cli_rsp(rsp);
+    return ret;
 }
 
 static int
-print_quota_list_usage_output (cli_local_t *local, char *path, int64_t avail,
-                               char *sl_str, quota_limits_t *limits,
-                               quota_meta_t *used_space, gf_boolean_t sl,
-                               gf_boolean_t hl, double sl_num,
-                               gf_boolean_t limit_set)
-{
-        int32_t         ret          = -1;
-        char           *used_str     = NULL;
-        char           *avail_str    = NULL;
-        char           *hl_str       = NULL;
-        char           *sl_val       = NULL;
-
-        used_str = gf_uint64_2human_readable (used_space->size);
-
-        if (limit_set) {
-                hl_str = gf_uint64_2human_readable (limits->hl);
-                avail_str = gf_uint64_2human_readable (avail);
-
-                sl_val = gf_uint64_2human_readable (sl_num);
+print_quota_list_usage_output(cli_local_t *local, char *path, int64_t avail,
+                              char *sl_str, quota_limits_t *limits,
+                              quota_meta_t *used_space, gf_boolean_t sl,
+                              gf_boolean_t hl, double sl_num,
+                              gf_boolean_t limit_set)
+{
+    int32_t ret = -1;
+    char *used_str = NULL;
+    char *avail_str = NULL;
+    char *hl_str = NULL;
+    char *sl_val = NULL;
+
+    if (global_state->mode & GLUSTER_MODE_XML) {
+        ret = cli_quota_xml_output(local, path, limits->hl, sl_str, sl_num,
+                                   used_space->size, avail, sl ? "Yes" : "No",
+                                   hl ? "Yes" : "No", limit_set);
+        if (ret) {
+            gf_log("cli", GF_LOG_ERROR,
+                   "Failed to output in xml format for quota list command");
         }
+        goto out;
+    }
 
-        if (global_state->mode & GLUSTER_MODE_XML) {
-                ret = cli_quota_xml_output (local, path, limits->hl,
-                                            sl_str, sl_num, used_space->size,
-                                            avail, sl ? "Yes" : "No",
-                                            hl ? "Yes" : "No", limit_set);
-                if (ret) {
-                        gf_log ("cli", GF_LOG_ERROR, "Failed to "
-                                "output in xml format for quota "
-                                "list command");
-                }
-                goto out;
-        }
+    used_str = gf_uint64_2human_readable(used_space->size);
 
-        if (limit_set) {
-                if (!used_str) {
-                        cli_out ("%-40s %7s %7s(%s) %8"PRIu64 "%9"PRIu64""
-                                 "%15s %18s", path, hl_str, sl_str, sl_val,
-                                 used_space->size, avail,
-                                 sl ? "Yes" : "No", hl ? "Yes" : "No");
-                } else {
-                        cli_out ("%-40s %7s %7s(%s) %8s %7s %15s %20s",
-                                 path, hl_str, sl_str, sl_val, used_str,
-                                 avail_str, sl ? "Yes" : "No",
-                                 hl ? "Yes" : "No");
-                }
+    if (limit_set) {
+        hl_str = gf_uint64_2human_readable(limits->hl);
+        sl_val = gf_uint64_2human_readable(sl_num);
+
+        if (!used_str) {
+            cli_out("%-40s %7s %7s(%s) %8" PRIu64 "%9" PRIu64
+                    ""
+                    "%15s %18s",
+                    path, hl_str, sl_str, sl_val, used_space->size, avail,
+                    sl ? "Yes" : "No", hl ? "Yes" : "No");
         } else {
-                        cli_out ("%-36s %10s %10s %14s %9s %15s %18s",
-                                 path, "N/A", "N/A", used_str, "N/A",
-                                 "N/A", "N/A");
+            avail_str = gf_uint64_2human_readable(avail);
+            cli_out("%-40s %7s %7s(%s) %8s %7s %15s %20s", path, hl_str, sl_str,
+                    sl_val, used_str, avail_str, sl ? "Yes" : "No",
+                    hl ? "Yes" : "No");
         }
+    } else {
+        cli_out("%-36s %10s %10s %14s %9s %15s %18s", path, "N/A", "N/A",
+                used_str, "N/A", "N/A", "N/A");
+    }
 
-        ret = 0;
+    ret = 0;
 out:
-        GF_FREE (hl_str);
-        GF_FREE (used_str);
-        GF_FREE (avail_str);
-        GF_FREE (sl_val);
+    GF_FREE(hl_str);
+    GF_FREE(used_str);
+    GF_FREE(avail_str);
+    GF_FREE(sl_val);
 
-        return ret;
+    return ret;
 }
 
 static int
-print_quota_list_object_output (cli_local_t *local, char *path, int64_t avail,
+print_quota_list_object_output(cli_local_t *local, char *path, int64_t avail,
                                char *sl_str, quota_limits_t *limits,
                                quota_meta_t *used_space, gf_boolean_t sl,
                                gf_boolean_t hl, double sl_num,
                                gf_boolean_t limit_set)
 {
-        int32_t         ret       = -1;
-        int64_t         sl_val    = sl_num;
+    int32_t ret = -1;
+    int64_t sl_val = sl_num;
 
-        if (global_state->mode & GLUSTER_MODE_XML) {
-                ret = cli_quota_object_xml_output (local, path, sl_str, sl_val,
-                                                   limits, used_space, avail,
-                                                   sl ? "Yes" : "No",
-                                                   hl ? "Yes" : "No",
-                                                   limit_set);
-                if (ret) {
-                        gf_log ("cli", GF_LOG_ERROR, "Failed to "
-                                "output in xml format for quota "
-                                "list command");
-                }
-                goto out;
+    if (global_state->mode & GLUSTER_MODE_XML) {
+        ret = cli_quota_object_xml_output(local, path, sl_str, sl_val, limits,
+                                          used_space, avail, sl ? "Yes" : "No",
+                                          hl ? "Yes" : "No", limit_set);
+        if (ret) {
+            gf_log("cli", GF_LOG_ERROR,
+                   "Failed to output in xml format for quota list command");
         }
+        goto out;
+    }
 
-        if (limit_set) {
-                cli_out ("%-40s %9"PRIu64" %9s(%"PRId64") %10"PRIu64""
-                         "%10"PRIu64" %11"PRIu64" %15s %20s",
-                         path, limits->hl, sl_str, sl_val,
-                         used_space->file_count, used_space->dir_count,
-                         avail, sl ? "Yes" : "No", hl ? "Yes" : "No");
-        } else {
-                cli_out ("%-40s %9s %9s %10"PRIu64" %10"PRIu64" %11s %15s %20s",
-                         path, "N/A", "N/A", used_space->file_count,
-                         used_space->dir_count, "N/A", "N/A", "N/A");
-        }
-        ret = 0;
+    if (limit_set) {
+        cli_out("%-40s %9" PRIu64 " %9s(%" PRId64 ") %10" PRIu64
+                ""
+                "%10" PRIu64 " %11" PRIu64 " %15s %20s",
+                path, limits->hl, sl_str, sl_val, used_space->file_count,
+                used_space->dir_count, avail, sl ? "Yes" : "No",
+                hl ? "Yes" : "No");
+    } else {
+        cli_out("%-40s %9s %9s %10" PRIu64 " %10" PRIu64 " %11s %15s %20s",
+                path, "N/A", "N/A", used_space->file_count,
+                used_space->dir_count, "N/A", "N/A", "N/A");
+    }
+    ret = 0;
 
 out:
 
-        return ret;
+    return ret;
 }
 
 static int
-print_quota_list_output (cli_local_t *local, char *path, char *default_sl,
-                         quota_limits_t *limits, quota_meta_t *used_space,
-                         int type, gf_boolean_t limit_set)
-{
-        int64_t         avail            = 0;
-        char            percent_str[20]  = {0};
-        char           *sl_final         = NULL;
-        int             ret              = -1;
-        double          sl_num           = 0;
-        gf_boolean_t    sl               = _gf_false;
-        gf_boolean_t    hl               = _gf_false;
-        int64_t         used_size        = 0;
-
-        GF_ASSERT (local);
-        GF_ASSERT (path);
-
-        if (limit_set) {
-                if (limits->sl < 0) {
-                        ret = gf_string2percent (default_sl, &sl_num);
-                        sl_num = (sl_num * limits->hl) / 100;
-                        sl_final = default_sl;
-                } else {
-                        sl_num = (limits->sl * limits->hl) / 100;
-                        snprintf (percent_str, sizeof (percent_str), "%"PRIu64"%%",
-                                        limits->sl);
-                        sl_final = percent_str;
-                }
-                if (type == GF_QUOTA_OPTION_TYPE_LIST)
-                        used_size = used_space->size;
-                else
-                        used_size = used_space->file_count + used_space->dir_count;
-
-                if (limits->hl > used_size) {
-                        avail = limits->hl - used_size;
-                        hl = _gf_false;
-                        if (used_size > sl_num)
-                                sl = _gf_true;
-                        else
-                                sl = _gf_false;
-                } else {
-                        avail = 0;
-                        hl = sl = _gf_true;
-                }
+print_quota_list_output(cli_local_t *local, char *path, char *default_sl,
+                        quota_limits_t *limits, quota_meta_t *used_space,
+                        int type, gf_boolean_t limit_set)
+{
+    int64_t avail = 0;
+    char percent_str[20] = {0};
+    char *sl_final = NULL;
+    int ret = -1;
+    double sl_num = 0;
+    gf_boolean_t sl = _gf_false;
+    gf_boolean_t hl = _gf_false;
+    int64_t used_size = 0;
+
+    GF_ASSERT(local);
+    GF_ASSERT(path);
+
+    if (limit_set) {
+        if (limits->sl < 0) {
+            ret = gf_string2percent(default_sl, &sl_num);
+            if (ret) {
+                gf_log("cli", GF_LOG_ERROR,
+                       "could not convert default soft limit to percent");
+                goto out;
+            }
+            sl_num = (sl_num * limits->hl) / 100;
+            sl_final = default_sl;
+        } else {
+            sl_num = (limits->sl * limits->hl) / 100;
+            ret = snprintf(percent_str, sizeof(percent_str), "%" PRIu64 "%%",
+                           limits->sl);
+            if (ret < 0)
+                goto out;
+            sl_final = percent_str;
         }
-
         if (type == GF_QUOTA_OPTION_TYPE_LIST)
-                ret = print_quota_list_usage_output (local, path, avail,
-                                                     sl_final, limits,
-                                                     used_space, sl, hl,
-                                                     sl_num, limit_set);
+            used_size = used_space->size;
         else
-                ret = print_quota_list_object_output (local, path, avail,
-                                                      sl_final, limits,
-                                                      used_space, sl, hl,
-                                                      sl_num, limit_set);
+            used_size = used_space->file_count + used_space->dir_count;
+
+        if (limits->hl > used_size) {
+            avail = limits->hl - used_size;
+            hl = _gf_false;
+            if (used_size > sl_num)
+                sl = _gf_true;
+            else
+                sl = _gf_false;
+        } else {
+            avail = 0;
+            hl = sl = _gf_true;
+        }
+    }
 
-        return ret;
+    if (type == GF_QUOTA_OPTION_TYPE_LIST)
+        ret = print_quota_list_usage_output(local, path, avail, sl_final,
+                                            limits, used_space, sl, hl, sl_num,
+                                            limit_set);
+    else
+        ret = print_quota_list_object_output(local, path, avail, sl_final,
+                                             limits, used_space, sl, hl, sl_num,
+                                             limit_set);
+out:
+    return ret;
 }
 
 static int
-print_quota_list_from_mountdir (cli_local_t *local, char *mountdir,
-                                char *default_sl, char *path, int type)
-{
-        int             ret              = -1;
-        ssize_t         xattr_size       = 0;
-        quota_limits_t  limits           = {0,};
-        quota_meta_t    used_space       = {0,};
-        char           *key              = NULL;
-        gf_boolean_t    limit_set        = _gf_true;
-
-        GF_ASSERT (local);
-        GF_ASSERT (mountdir);
-        GF_ASSERT (path);
-
-        if (type == GF_QUOTA_OPTION_TYPE_LIST)
-                key = QUOTA_LIMIT_KEY;
-        else
-                key = QUOTA_LIMIT_OBJECTS_KEY;
-
-
-        ret = sys_lgetxattr (mountdir, key, (void *)&limits, sizeof (limits));
-        if (ret < 0) {
-                gf_log ("cli", GF_LOG_ERROR, "Failed to get the xattr %s "
-                        "on %s. Reason : %s", key, mountdir, strerror (errno));
-
-                switch (errno) {
+print_quota_list_from_mountdir(cli_local_t *local, char *mountdir,
+                               char *default_sl, char *path, int type)
+{
+    int ret = -1;
+    ssize_t xattr_size = 0;
+    quota_limits_t limits = {
+        0,
+    };
+    quota_meta_t used_space = {
+        0,
+    };
+    char *key = NULL;
+    gf_boolean_t limit_set = _gf_true;
+
+    GF_ASSERT(local);
+    GF_ASSERT(mountdir);
+    GF_ASSERT(path);
+
+    if (type == GF_QUOTA_OPTION_TYPE_LIST)
+        key = QUOTA_LIMIT_KEY;
+    else
+        key = QUOTA_LIMIT_OBJECTS_KEY;
+
+    ret = sys_lgetxattr(mountdir, key, (void *)&limits, sizeof(limits));
+    if (ret < 0) {
+        gf_log("cli", GF_LOG_ERROR,
+               "Failed to get the xattr %s on %s. Reason : %s", key, mountdir,
+               strerror(errno));
+
+        switch (errno) {
 #if defined(ENODATA)
-                case ENODATA:
+            case ENODATA:
 #endif
 #if defined(ENOATTR) && (ENOATTR != ENODATA)
-                case ENOATTR:
+            case ENOATTR:
 #endif
-                        /* If it's an ENOATTR, quota/inode-quota is
-                         * configured(limit is set atleast for one directory).
-                         * The user is trying to issue 'list/list-objects'
-                         * command for a directory on which quota limit is
-                         * not set and we are showing the used-space in case
-                         * of list-usage and showing (dir_count, file_count)
-                         * in case of list-objects. Other labels are
-                         * shown "N/A".
-                         */
-
-                        limit_set = _gf_false;
-                        goto enoattr;
-                        break;
-
-                default:
-                        cli_err ("%-40s %s", path, strerror (errno));
-                        break;
-                }
-
-                goto out;
-        }
-
-        limits.hl = ntoh64 (limits.hl);
-        limits.sl = ntoh64 (limits.sl);
-
-enoattr:
-        xattr_size = sys_lgetxattr (mountdir, QUOTA_SIZE_KEY, NULL, 0);
-        if (xattr_size < (sizeof (int64_t) * 2) &&
-            type == GF_QUOTA_OPTION_TYPE_LIST_OBJECTS) {
-                ret = -1;
-
-                /* This can happen when glusterfs is upgraded from 3.6 to 3.7
-                 * and the xattr healing is not completed.
-                 */
-        } else if (xattr_size > (sizeof (int64_t) * 2)) {
-                ret = sys_lgetxattr (mountdir, QUOTA_SIZE_KEY,
-                                     &used_space, sizeof (used_space));
-        } else if (xattr_size > 0) {
-                /* This is for compatibility.
-                 * Older version had only file usage
+                /* If it's an ENOATTR, quota/inode-quota is
+                 * configured(limit is set at least for one directory).
+                 * The user is trying to issue 'list/list-objects'
+                 * command for a directory on which quota limit is
+                 * not set and we are showing the used-space in case
+                 * of list-usage and showing (dir_count, file_count)
+                 * in case of list-objects. Other labels are
+                 * shown "N/A".
                  */
-                ret = sys_lgetxattr (mountdir, QUOTA_SIZE_KEY,
-                             &(used_space.size), sizeof (used_space.size));
-                used_space.file_count = 0;
-                used_space.dir_count = 0;
-        } else {
-                ret = -1;
-        }
 
-        if (ret < 0) {
-                gf_log ("cli", GF_LOG_ERROR, "Failed to get quota size "
-                        "on path %s: %s", mountdir, strerror (errno));
-                print_quota_list_empty (path, type);
-                goto out;
-        }
+                limit_set = _gf_false;
+                goto enoattr;
+                break;
 
-        used_space.size = ntoh64 (used_space.size);
-        used_space.file_count = ntoh64 (used_space.file_count);
-        used_space.dir_count = ntoh64 (used_space.dir_count);
+            default:
+                cli_err("%-40s %s", path, strerror(errno));
+                break;
+        }
 
-        ret = print_quota_list_output (local, path, default_sl, &limits,
-                                       &used_space, type, limit_set);
-out:
-        return ret;
-}
+        goto out;
+    }
 
-int
-gf_cli_print_limit_list_from_dict (cli_local_t *local, char *volname,
-                                   dict_t *dict, char *default_sl, int count,
-                                   int op_ret, int op_errno, char *op_errstr)
-{
-        int             ret                     = -1;
-        int             i                       = 0;
-        char            key[1024]               = {0,};
-        char            mountdir[PATH_MAX]      = {0,};
-        char            *path                   = NULL;
-        gf_boolean_t    xml_err_flag            = _gf_false;
-        char            err_str[NAME_MAX]       = {0,};
-        int             type                    = -1;
-
-        if (!dict|| count <= 0)
-                goto out;
+    limits.hl = ntoh64(limits.hl);
+    limits.sl = ntoh64(limits.sl);
 
-        ret = dict_get_int32 (dict, "type", &type);
-        if (ret) {
-                gf_log ("cli", GF_LOG_ERROR, "Failed to get quota type");
-                goto out;
-        }
+enoattr:
+    xattr_size = sys_lgetxattr(mountdir, QUOTA_SIZE_KEY, NULL, 0);
+    if (xattr_size < (sizeof(int64_t) * 2) &&
+        type == GF_QUOTA_OPTION_TYPE_LIST_OBJECTS) {
+        ret = -1;
 
-        /* Need to check if any quota limits are set on the volume before trying
-         * to list them
+        /* This can happen when glusterfs is upgraded from 3.6 to 3.7
+         * and the xattr healing is not completed.
          */
-        if (!_limits_set_on_volume (volname, type)) {
-                snprintf (err_str, sizeof (err_str), "No%s quota configured on "
-                          "volume %s",
-                          (type == GF_QUOTA_OPTION_TYPE_LIST) ? "" : " inode",
-                          volname);
-                if (global_state->mode & GLUSTER_MODE_XML) {
-                        xml_err_flag = _gf_true;
-                } else {
-                        cli_out ("quota: %s", err_str);
-                }
-                ret = 0;
-                goto out;
-        }
-
-        /* Check if the mount is online before doing any listing */
-        if (!_quota_aux_mount_online (volname)) {
-                ret = -1;
-                goto out;
-        }
-
-        if (global_state->mode & GLUSTER_MODE_XML) {
-                ret = cli_xml_output_vol_quota_limit_list_begin
-                                (local, op_ret, op_errno, op_errstr);
-                if (ret) {
-                        gf_log ("cli", GF_LOG_ERROR,
-                                "Error outputting xml begin");
-                        goto out;
-                }
-        } else {
-                print_quota_list_header (type);
-        }
+    } else if (xattr_size > (sizeof(int64_t) * 2)) {
+        ret = sys_lgetxattr(mountdir, QUOTA_SIZE_KEY, &used_space,
+                            sizeof(used_space));
+    } else if (xattr_size > 0) {
+        /* This is for compatibility.
+         * Older version had only file usage
+         */
+        ret = sys_lgetxattr(mountdir, QUOTA_SIZE_KEY, &(used_space.size),
+                            sizeof(used_space.size));
+        used_space.file_count = 0;
+        used_space.dir_count = 0;
+    } else {
+        ret = -1;
+    }
 
-        while (count--) {
-                snprintf (key, sizeof (key), "path%d", i++);
+    if (ret < 0) {
+        gf_log("cli", GF_LOG_ERROR, "Failed to get quota size on path %s: %s",
+               mountdir, strerror(errno));
+        print_quota_list_empty(path, type);
+        goto out;
+    }
 
-                ret = dict_get_str (dict, key, &path);
-                if (ret < 0) {
-                        gf_log ("cli", GF_LOG_DEBUG, "Path not present in limit"
-                                " list");
-                        continue;
-                }
-
-                ret = gf_canonicalize_path (path);
-                if (ret)
-                        goto out;
-                GLUSTERD_GET_QUOTA_AUX_MOUNT_PATH (mountdir, volname, path);
-                ret = print_quota_list_from_mountdir (local, mountdir,
-                                                      default_sl, path, type);
-        }
+    used_space.size = ntoh64(used_space.size);
+    used_space.file_count = ntoh64(used_space.file_count);
+    used_space.dir_count = ntoh64(used_space.dir_count);
 
+    ret = print_quota_list_output(local, path, default_sl, &limits, &used_space,
+                                  type, limit_set);
 out:
-        if (xml_err_flag) {
-                ret = cli_xml_output_str ("volQuota", NULL, -1, 0, err_str);
-                if (ret) {
-                        gf_log ("cli", GF_LOG_ERROR, "Error outputting in xml "
-                                "format");
-                }
-        }
-        return ret;
+    return ret;
 }
 
-int
-print_quota_list_from_quotad (call_frame_t *frame, dict_t *rsp_dict)
+static int
+gluster_remove_auxiliary_mount(char *volname)
 {
-        char             *path          = NULL;
-        char             *default_sl    = NULL;
-        int              ret            = -1;
-        cli_local_t     *local          = NULL;
-        dict_t          *gd_rsp_dict    = NULL;
-        quota_meta_t     used_space     = {0, };
-        quota_limits_t   limits         = {0, };
-        quota_limits_t  *size_limits    = NULL;
-        int32_t          type           = 0;
-        int32_t          success_count  = 0;
-
-        GF_ASSERT (frame);
+    int ret = -1;
+    char mountdir[PATH_MAX] = {
+        0,
+    };
+    xlator_t *this = NULL;
 
-        local = frame->local;
-        gd_rsp_dict = local->dict;
+    this = THIS;
+    GF_ASSERT(this);
 
-        ret = dict_get_int32 (rsp_dict, "type", &type);
-        if (ret) {
-                gf_log ("cli", GF_LOG_ERROR, "Failed to get type");
-                goto out;
-        }
+    GLUSTERD_GET_QUOTA_LIST_MOUNT_PATH(mountdir, volname, "/");
+    ret = gf_umount_lazy(this->name, mountdir, 1);
+    if (ret) {
+        gf_log("cli", GF_LOG_ERROR, "umount on %s failed, reason : %s",
+               mountdir, strerror(errno));
+    }
 
-        ret = dict_get_str (rsp_dict, GET_ANCESTRY_PATH_KEY, &path);
-        if (ret) {
-                gf_log ("cli", GF_LOG_WARNING, "path key is not present "
-                        "in dict");
-                goto out;
-        }
+    return ret;
+}
 
-        ret = dict_get_str (gd_rsp_dict, "default-soft-limit", &default_sl);
+static int
+gf_cli_print_limit_list_from_dict(cli_local_t *local, char *volname,
+                                  dict_t *dict, char *default_sl, int count,
+                                  int op_ret, int op_errno, char *op_errstr)
+{
+    int ret = -1;
+    int i = 0;
+    char key[32] = {
+        0,
+    };
+    int keylen;
+    char mountdir[PATH_MAX] = {
+        0,
+    };
+    char *path = NULL;
+    int type = -1;
+
+    if (!dict || count <= 0)
+        goto out;
+
+    ret = dict_get_int32_sizen(dict, "type", &type);
+    if (ret) {
+        gf_log("cli", GF_LOG_ERROR, "Failed to get quota type");
+        goto out;
+    }
+
+    if (global_state->mode & GLUSTER_MODE_XML) {
+        ret = cli_xml_output_vol_quota_limit_list_begin(local, op_ret, op_errno,
+                                                        op_errstr);
         if (ret) {
-                gf_log (frame->this->name, GF_LOG_ERROR, "failed to "
-                        "get default soft limit");
-                goto out;
+            gf_log("cli", GF_LOG_ERROR, XML_ERROR);
+            goto out;
         }
+    } else {
+        print_quota_list_header(type);
+    }
 
-        if (type == GF_QUOTA_OPTION_TYPE_LIST) {
-                ret = dict_get_bin (rsp_dict, QUOTA_LIMIT_KEY,
-                                    (void **)&size_limits);
-                if (ret) {
-                        gf_log ("cli", GF_LOG_WARNING,
-                                "limit key not present in dict on %s",
-                                path);
-                        goto out;
-                }
-        } else {
-                ret = dict_get_bin (rsp_dict, QUOTA_LIMIT_OBJECTS_KEY,
-                                   (void **)&size_limits);
-                if (ret) {
-                        gf_log ("cli", GF_LOG_WARNING,
-                                "object limit key not present in dict on %s",
-                                path);
-                        goto out;
-                }
-        }
-
-        limits.hl = ntoh64 (size_limits->hl);
-        limits.sl = ntoh64 (size_limits->sl);
-
-        if (type == GF_QUOTA_OPTION_TYPE_LIST)
-                ret = quota_dict_get_meta (rsp_dict, QUOTA_SIZE_KEY,
-                                           &used_space);
-        else
-                ret = quota_dict_get_inode_meta (rsp_dict, QUOTA_SIZE_KEY,
-                                                 &used_space);
+    while (count--) {
+        keylen = snprintf(key, sizeof(key), "path%d", i++);
 
+        ret = dict_get_strn(dict, key, keylen, &path);
         if (ret < 0) {
-                gf_log ("cli", GF_LOG_WARNING,
-                        "size key not present in dict");
-                print_quota_list_empty (path, type);
-                goto out;
+            gf_log("cli", GF_LOG_DEBUG, "Path not present in limit list");
+            continue;
         }
 
-        LOCK (&local->lock);
-        {
-                ret = dict_get_int32 (gd_rsp_dict, "quota-list-success-count",
-                                      &success_count);
-                if (ret)
-                        success_count = 0;
+        ret = gf_canonicalize_path(path);
+        if (ret)
+            goto out;
+        GLUSTERD_GET_QUOTA_LIST_MOUNT_PATH(mountdir, volname, path);
+        ret = print_quota_list_from_mountdir(local, mountdir, default_sl, path,
+                                             type);
+    }
 
-                ret = dict_set_int32 (gd_rsp_dict,
-                                      "quota-list-success-count",
-                                      success_count + 1);
+out:
+    return ret;
+}
+
+static int
+print_quota_list_from_quotad(call_frame_t *frame, dict_t *rsp_dict)
+{
+    char *path = NULL;
+    char *default_sl = NULL;
+    int ret = -1;
+    cli_local_t *local = NULL;
+    dict_t *gd_rsp_dict = NULL;
+    quota_meta_t used_space = {
+        0,
+    };
+    quota_limits_t limits = {
+        0,
+    };
+    quota_limits_t *size_limits = NULL;
+    int32_t type = 0;
+    int32_t success_count = 0;
+
+    GF_ASSERT(frame);
+
+    local = frame->local;
+    gd_rsp_dict = local->dict;
+
+    ret = dict_get_int32_sizen(rsp_dict, "type", &type);
+    if (ret) {
+        gf_log("cli", GF_LOG_ERROR, "Failed to get type");
+        goto out;
+    }
+
+    ret = dict_get_str_sizen(rsp_dict, GET_ANCESTRY_PATH_KEY, &path);
+    if (ret) {
+        gf_log("cli", GF_LOG_WARNING, "path key is not present in dict");
+        goto out;
+    }
+
+    ret = dict_get_str_sizen(gd_rsp_dict, "default-soft-limit", &default_sl);
+    if (ret) {
+        gf_log(frame->this->name, GF_LOG_ERROR,
+               "failed to get default soft limit");
+        goto out;
+    }
+
+    if (type == GF_QUOTA_OPTION_TYPE_LIST) {
+        ret = dict_get_bin(rsp_dict, QUOTA_LIMIT_KEY, (void **)&size_limits);
+        if (ret) {
+            gf_log("cli", GF_LOG_WARNING, "limit key not present in dict on %s",
+                   path);
+            goto out;
         }
-        UNLOCK (&local->lock);
+    } else {
+        ret = dict_get_bin(rsp_dict, QUOTA_LIMIT_OBJECTS_KEY,
+                           (void **)&size_limits);
         if (ret) {
-                gf_log ("cli", GF_LOG_ERROR, "Failed to set "
-                        "quota-list-success-count in dict");
+            gf_log("cli", GF_LOG_WARNING,
+                   "object limit key not present in dict on %s", path);
+            goto out;
+        }
+    }
+
+    limits.hl = ntoh64(size_limits->hl);
+    limits.sl = ntoh64(size_limits->sl);
+
+    if (type == GF_QUOTA_OPTION_TYPE_LIST)
+        ret = quota_dict_get_meta(rsp_dict, QUOTA_SIZE_KEY,
+                                  SLEN(QUOTA_SIZE_KEY), &used_space);
+    else
+        ret = quota_dict_get_inode_meta(rsp_dict, QUOTA_SIZE_KEY,
+                                        SLEN(QUOTA_SIZE_KEY), &used_space);
+
+    if (ret < 0) {
+        gf_log("cli", GF_LOG_WARNING, "size key not present in dict");
+        print_quota_list_empty(path, type);
+        goto out;
+    }
+
+    LOCK(&local->lock);
+    {
+        ret = dict_get_int32_sizen(gd_rsp_dict, "quota-list-success-count",
+                                   &success_count);
+        if (ret)
+            success_count = 0;
+
+        ret = dict_set_int32_sizen(gd_rsp_dict, "quota-list-success-count",
+                                   success_count + 1);
+    }
+    UNLOCK(&local->lock);
+    if (ret) {
+        gf_log("cli", GF_LOG_ERROR,
+               "Failed to set quota-list-success-count in dict");
+        goto out;
+    }
+
+    if (success_count == 0) {
+        if (!(global_state->mode & GLUSTER_MODE_XML)) {
+            print_quota_list_header(type);
+        } else {
+            ret = cli_xml_output_vol_quota_limit_list_begin(local, 0, 0, NULL);
+            if (ret) {
+                gf_log("cli", GF_LOG_ERROR, XML_ERROR);
                 goto out;
+            }
         }
+    }
 
-        if (success_count == 0) {
-                if (!(global_state->mode & GLUSTER_MODE_XML)) {
-                        print_quota_list_header (type);
-                } else {
-                        ret = cli_xml_output_vol_quota_limit_list_begin
-                                (local, 0, 0, NULL);
-                        if (ret) {
-                                gf_log ("cli", GF_LOG_ERROR, "Error in "
-                                        "printing xml output");
-                                goto out;
-                        }
-                }
-        }
-
-        ret = print_quota_list_output (local, path, default_sl, &limits,
-                                       &used_space, type, _gf_true);
+    ret = print_quota_list_output(local, path, default_sl, &limits, &used_space,
+                                  type, _gf_true);
 out:
-        return ret;
+    return ret;
 }
 
-void*
-cli_cmd_broadcast_response_detached (void *opaque)
+static void *
+cli_cmd_broadcast_response_detached(void *opaque)
 {
-        int32_t ret = 0;
+    int32_t ret = 0;
 
-        ret = (intptr_t) opaque;
-        cli_cmd_broadcast_response (ret);
+    ret = (intptr_t)opaque;
+    cli_cmd_broadcast_response(ret);
 
-        return NULL;
+    return NULL;
 }
 
-int
-cli_quotad_getlimit_cbk (struct rpc_req *req, struct iovec *iov,
-                          int count, void *myframe)
+static int32_t
+cli_quota_compare_path(struct list_head *list1, struct list_head *list2)
 {
-    /*TODO: we need to gather the path, hard-limit, soft-limit and used space*/
-        gf_cli_rsp         rsp         = {0,};
-        int                ret         = -1;
-        dict_t            *dict        = NULL;
-        call_frame_t      *frame       = NULL;
-        cli_local_t       *local       = NULL;
-        dict_t            *gd_rsp_dict = NULL;
-        int32_t            list_count  = 0;
-        pthread_t          th_id       = {0, };
-
-        frame = myframe;
-        GF_ASSERT (frame);
-
-        local = frame->local;
-        gd_rsp_dict = local->dict;
-
-        LOCK (&local->lock);
-        {
-                ret = dict_get_int32 (gd_rsp_dict, "quota-list-count",
-                                      &list_count);
-                if (ret)
-                        list_count = 0;
-                ret = dict_set_int32 (gd_rsp_dict, "quota-list-count",
-                                      list_count + 1);
-        }
-        UNLOCK (&local->lock);
+    struct list_node *node1 = NULL;
+    struct list_node *node2 = NULL;
+    dict_t *dict1 = NULL;
+    dict_t *dict2 = NULL;
+    char *path1 = NULL;
+    char *path2 = NULL;
+    int ret = 0;
 
-        if (ret) {
-                gf_log ("cli", GF_LOG_ERROR, "Failed to set "
-                        "quota-list-count in dict");
-                goto out;
-        }
+    node1 = list_entry(list1, struct list_node, list);
+    node2 = list_entry(list2, struct list_node, list);
 
-        if (-1 == req->rpc_status) {
-                if (list_count == 0)
-                        cli_err ("Connection failed. Please check if quota "
-                                 "daemon is operational.");
-                ret = -1;
-                goto out;
-        }
+    dict1 = node1->ptr;
+    dict2 = node2->ptr;
 
-        ret = xdr_to_generic (*iov, &rsp, (xdrproc_t)xdr_gf_cli_rsp);
-        if (ret < 0) {
-                gf_log (frame->this->name, GF_LOG_ERROR,
-                        "Failed to decode xdr response");
-                goto out;
-        }
+    ret = dict_get_str_sizen(dict1, GET_ANCESTRY_PATH_KEY, &path1);
+    if (ret < 0)
+        return 0;
 
-        if (rsp.op_ret) {
-                ret = -1;
-                if (strcmp (rsp.op_errstr, ""))
-                        cli_err ("quota command failed : %s", rsp.op_errstr);
-                else
-                        cli_err ("quota command : failed");
-                goto out;
-        }
+    ret = dict_get_str_sizen(dict2, GET_ANCESTRY_PATH_KEY, &path2);
+    if (ret < 0)
+        return 0;
 
-        if (rsp.dict.dict_len) {
-                /* Unserialize the dictionary */
-                dict  = dict_new ();
+    return strcmp(path1, path2);
+}
 
-                ret = dict_unserialize (rsp.dict.dict_val,
-                                        rsp.dict.dict_len,
-                                        &dict);
-                if (ret < 0) {
-                        gf_log ("cli", GF_LOG_ERROR,
-                                "failed to "
-                                "unserialize req-buffer to dictionary");
-                        goto out;
-                }
+static int
+cli_quotad_getlimit_cbk(struct rpc_req *req, struct iovec *iov, int count,
+                        void *myframe)
+{
+    /*TODO: we need to gather the path, hard-limit, soft-limit and used space*/
+    gf_cli_rsp rsp = {
+        0,
+    };
+    int ret = -1;
+    dict_t *dict = NULL;
+    struct list_node *node = NULL;
+    struct list_node *tmpnode = NULL;
+    call_frame_t *frame = NULL;
+    cli_local_t *local = NULL;
+    int32_t list_count = 0;
+    pthread_t th_id = {
+        0,
+    };
+    int32_t max_count = 0;
+
+    GF_ASSERT(myframe);
+
+    frame = myframe;
+
+    GF_ASSERT(frame->local);
+
+    local = frame->local;
+
+    LOCK(&local->lock);
+    {
+        ret = dict_get_int32_sizen(local->dict, "quota-list-count",
+                                   &list_count);
+        if (ret)
+            list_count = 0;
+
+        list_count++;
+        ret = dict_set_int32_sizen(local->dict, "quota-list-count", list_count);
+    }
+    UNLOCK(&local->lock);
+
+    if (ret) {
+        gf_log("cli", GF_LOG_ERROR, "Failed to set quota-list-count in dict");
+        goto out;
+    }
+
+    if (-1 == req->rpc_status) {
+        if (list_count == 0)
+            cli_err(
+                "Connection failed. Please check if quota "
+                "daemon is operational.");
+        ret = -1;
+        goto out;
+    }
+
+    ret = xdr_to_generic(*iov, &rsp, (xdrproc_t)xdr_gf_cli_rsp);
+    if (ret < 0) {
+        gf_log(frame->this->name, GF_LOG_ERROR, XDR_DECODE_FAIL);
+        goto out;
+    }
+
+    if (rsp.op_ret) {
+        ret = -1;
+        if (strcmp(rsp.op_errstr, ""))
+            cli_err("quota command failed : %s", rsp.op_errstr);
+        else
+            cli_err("quota command : failed");
+        goto out;
+    }
+
+    if (rsp.dict.dict_len) {
+        /* Unserialize the dictionary */
+        dict = dict_new();
 
-                print_quota_list_from_quotad (frame, dict);
+        ret = dict_unserialize(rsp.dict.dict_val, rsp.dict.dict_len, &dict);
+        if (ret < 0) {
+            gf_log("cli", GF_LOG_ERROR, DICT_UNSERIALIZE_FAIL);
+            goto out;
         }
 
-out:
-        /* Bad Fix: CLI holds the lock to process a command.
-         * When processing quota list command, below sequence of steps executed
-         * in the same thread and causing deadlock
-         *
-         * 1) CLI holds the lock
-         * 2) Send rpc_clnt_submit request to quotad for quota usage
-         * 3) If quotad is down, rpc_clnt_submit invokes cbk function with error
-         * 4) cbk function cli_quotad_getlimit_cbk invokes
-         *    cli_cmd_broadcast_response which tries to hold lock to broadcast
-         *    the results and hangs, because same thread has already holding
-         *    the lock
-         *
-         * Broadcasting response in a seperate thread which is not a
-         * good fix. This needs to be re-visted with better solution
-         */
-        if (ret == -1) {
-                ret = pthread_create (&th_id, NULL,
-                                cli_cmd_broadcast_response_detached,
-                                (void *)-1);
-                if (ret)
-                        gf_log ("cli", GF_LOG_ERROR, "pthread_create failed: "
-                                "%s", strerror (errno));
-        } else {
-                cli_cmd_broadcast_response (ret);
+        node = list_node_add_order(dict, &local->dict_list,
+                                   cli_quota_compare_path);
+        if (node == NULL) {
+            gf_log("cli", GF_LOG_ERROR, "failed to add node to the list");
+            dict_unref(dict);
+            ret = -1;
+            goto out;
         }
+    }
 
-        if (dict)
-                dict_unref (dict);
+    ret = dict_get_int32_sizen(local->dict, "max_count", &max_count);
+    if (ret < 0) {
+        gf_log("cli", GF_LOG_ERROR, "failed to get max_count");
+        goto out;
+    }
 
-        free (rsp.dict.dict_val);
-        return ret;
+    if (list_count == max_count) {
+        list_for_each_entry_safe(node, tmpnode, &local->dict_list, list)
+        {
+            dict = node->ptr;
+            print_quota_list_from_quotad(frame, dict);
+            list_node_del(node);
+            dict_unref(dict);
+        }
+    }
+
+out:
+    /* Bad Fix: CLI holds the lock to process a command.
+     * When processing quota list command, below sequence of steps executed
+     * in the same thread and causing deadlock
+     *
+     * 1) CLI holds the lock
+     * 2) Send rpc_clnt_submit request to quotad for quota usage
+     * 3) If quotad is down, rpc_clnt_submit invokes cbk function with error
+     * 4) cbk function cli_quotad_getlimit_cbk invokes
+     *    cli_cmd_broadcast_response which tries to hold lock to broadcast
+     *    the results and hangs, because same thread has already holding
+     *    the lock
+     *
+     * Broadcasting response in a separate thread which is not a
+     * good fix. This needs to be re-visted with better solution
+     */
+    if (ret == -1) {
+        ret = pthread_create(&th_id, NULL, cli_cmd_broadcast_response_detached,
+                             (void *)-1);
+        if (ret)
+            gf_log("cli", GF_LOG_ERROR, "pthread_create failed: %s",
+                   strerror(errno));
+    } else {
+        cli_cmd_broadcast_response(ret);
+    }
+    gf_free_xdr_cli_rsp(rsp);
+
+    return ret;
 }
 
-int
-cli_quotad_getlimit (call_frame_t *frame, xlator_t *this, void *data)
+static int
+cli_quotad_getlimit(call_frame_t *frame, xlator_t *this, void *data)
 {
-        gf_cli_req          req = {{0,}};
-        int                 ret = 0;
-        dict_t             *dict = NULL;
+    gf_cli_req req = {{
+        0,
+    }};
+    int ret = 0;
+    dict_t *dict = NULL;
 
-        if (!frame || !this ||  !data) {
-                ret = -1;
-                goto out;
-        }
+    if (!frame || !this || !data) {
+        ret = -1;
+        goto out;
+    }
 
-        dict = data;
-        ret = dict_allocate_and_serialize (dict, &req.dict.dict_val,
-                                           &req.dict.dict_len);
-        if (ret < 0) {
-                gf_log (this->name, GF_LOG_ERROR,
-                        "failed to serialize the data");
+    dict = data;
+    ret = add_cli_cmd_timeout_to_dict(dict);
 
-                goto out;
-        }
+    ret = dict_allocate_and_serialize(dict, &req.dict.dict_val,
+                                      &req.dict.dict_len);
+    if (ret < 0) {
+        gf_log(this->name, GF_LOG_ERROR, DICT_SERIALIZE_FAIL);
+
+        goto out;
+    }
 
-        ret = cli_cmd_submit (global_quotad_rpc, &req, frame, &cli_quotad_clnt,
-                              GF_AGGREGATOR_GETLIMIT, NULL,
-                              this, cli_quotad_getlimit_cbk,
-                              (xdrproc_t) xdr_gf_cli_req);
+    ret = cli_cmd_submit(global_quotad_rpc, &req, frame, &cli_quotad_clnt,
+                         GF_AGGREGATOR_GETLIMIT, NULL, this,
+                         cli_quotad_getlimit_cbk, (xdrproc_t)xdr_gf_cli_req);
 
 out:
-        gf_log ("cli", GF_LOG_DEBUG, "Returning %d", ret);
-        return ret;
+    GF_FREE(req.dict.dict_val);
+    gf_log("cli", GF_LOG_DEBUG, RETURNING, ret);
+    return ret;
 }
 
-void
-gf_cli_quota_list (cli_local_t *local, char *volname, dict_t *dict,
-                   char *default_sl, int count, int op_ret,
-                   int op_errno, char *op_errstr)
+static void
+gf_cli_quota_list(cli_local_t *local, char *volname, dict_t *dict,
+                  char *default_sl, int count, int op_ret, int op_errno,
+                  char *op_errstr)
 {
-        GF_VALIDATE_OR_GOTO ("cli", volname, out);
+    if (!cli_cmd_connected())
+        goto out;
 
-        if (!connected)
-                goto out;
+    if (count > 0) {
+        GF_VALIDATE_OR_GOTO("cli", volname, out);
 
-        if (count > 0)
-                gf_cli_print_limit_list_from_dict (local, volname, dict,
-                                                   default_sl, count, op_ret,
-                                                   op_errno, op_errstr);
+        gf_cli_print_limit_list_from_dict(local, volname, dict, default_sl,
+                                          count, op_ret, op_errno, op_errstr);
+    }
 out:
-        return;
+    return;
 }
 
-int
-gf_cli_quota_cbk (struct rpc_req *req, struct iovec *iov,
-                     int count, void *myframe)
-{
-        gf_cli_rsp         rsp         = {0,};
-        int                ret         = -1;
-        dict_t            *dict        = NULL;
-        char              *volname     = NULL;
-        int32_t            type        = 0;
-        call_frame_t      *frame       = NULL;
-        char              *default_sl  = NULL;
-        char              *limit_list  = NULL;
-        cli_local_t       *local       = NULL;
-        dict_t            *aggr        = NULL;
-        char              *default_sl_dup  = NULL;
-        int32_t            entry_count      = 0;
-        if (-1 == req->rpc_status) {
-                goto out;
+static int
+gf_cli_quota_cbk(struct rpc_req *req, struct iovec *iov, int count,
+                 void *myframe)
+{
+    gf_cli_rsp rsp = {
+        0,
+    };
+    int ret = -1;
+    dict_t *dict = NULL;
+    char *volname = NULL;
+    int32_t type = 0;
+    call_frame_t *frame = NULL;
+    char *default_sl = NULL;
+    cli_local_t *local = NULL;
+    char *default_sl_dup = NULL;
+    int32_t entry_count = 0;
+
+    GF_ASSERT(myframe);
+
+    if (-1 == req->rpc_status) {
+        goto out;
+    }
+
+    frame = myframe;
+
+    GF_ASSERT(frame->local);
+
+    local = frame->local;
+
+    ret = xdr_to_generic(*iov, &rsp, (xdrproc_t)xdr_gf_cli_rsp);
+    if (ret < 0) {
+        gf_log(frame->this->name, GF_LOG_ERROR, XDR_DECODE_FAIL);
+        goto out;
+    }
+
+    if (rsp.op_ret) {
+        ret = -1;
+        if (global_state->mode & GLUSTER_MODE_XML)
+            goto xml_output;
+
+        if (strcmp(rsp.op_errstr, "")) {
+            cli_err("quota command failed : %s", rsp.op_errstr);
+            if (rsp.op_ret == -ENOENT)
+                cli_err("please enter the path relative to the volume");
+        } else {
+            cli_err("quota command : failed");
         }
 
-        frame = myframe;
-        local = frame->local;
-        aggr  = local->dict;
+        goto out;
+    }
+
+    if (rsp.dict.dict_len) {
+        /* Unserialize the dictionary */
+        dict = dict_new();
 
-        ret = xdr_to_generic (*iov, &rsp, (xdrproc_t)xdr_gf_cli_rsp);
+        ret = dict_unserialize(rsp.dict.dict_val, rsp.dict.dict_len, &dict);
         if (ret < 0) {
-                gf_log (frame->this->name, GF_LOG_ERROR,
-                        "Failed to decode xdr response");
-                goto out;
+            gf_log("cli", GF_LOG_ERROR, DICT_UNSERIALIZE_FAIL);
+            goto out;
         }
+    }
 
-        if (rsp.op_ret) {
-                ret = -1;
-                if (global_state->mode & GLUSTER_MODE_XML)
-                        goto xml_output;
-
-                if (strcmp (rsp.op_errstr, "")) {
-                        cli_err ("quota command failed : %s", rsp.op_errstr);
-                        if (rsp.op_ret == -ENOENT)
-                                cli_err ("please enter the path relative to "
-                                         "the volume");
-                } else {
-                        cli_err ("quota command : failed");
-                }
-
-                goto out;
-        }
+    gf_log("cli", GF_LOG_DEBUG, "Received resp to quota command");
 
-        if (rsp.dict.dict_len) {
-                /* Unserialize the dictionary */
-                dict  = dict_new ();
+    ret = dict_get_str_sizen(dict, "default-soft-limit", &default_sl);
+    if (ret)
+        gf_log(frame->this->name, GF_LOG_TRACE,
+               "failed to get default soft limit");
 
-                ret = dict_unserialize (rsp.dict.dict_val,
-                                        rsp.dict.dict_len,
-                                        &dict);
-                if (ret < 0) {
-                        gf_log ("cli", GF_LOG_ERROR,
-                                "failed to "
-                                "unserialize req-buffer to dictionary");
-                        goto out;
-                }
+    // default-soft-limit is part of rsp_dict only iff we sent
+    // GLUSTER_CLI_QUOTA with type being GF_QUOTA_OPTION_TYPE_LIST
+    if (default_sl) {
+        default_sl_dup = gf_strdup(default_sl);
+        if (!default_sl_dup) {
+            ret = -1;
+            goto out;
+        }
+        ret = dict_set_dynstr_sizen(local->dict, "default-soft-limit",
+                                    default_sl_dup);
+        if (ret) {
+            gf_log(frame->this->name, GF_LOG_TRACE,
+                   "failed to set default soft limit");
+            GF_FREE(default_sl_dup);
         }
+    }
 
-        gf_log ("cli", GF_LOG_DEBUG, "Received resp to quota command");
+    ret = dict_get_str_sizen(dict, "volname", &volname);
+    if (ret)
+        gf_log(frame->this->name, GF_LOG_ERROR, "failed to get volname");
 
-        ret = dict_get_str (dict, "volname", &volname);
-        if (ret)
-                gf_log (frame->this->name, GF_LOG_ERROR,
-                        "failed to get volname");
+    ret = dict_get_int32_sizen(dict, "type", &type);
+    if (ret)
+        gf_log(frame->this->name, GF_LOG_TRACE, "failed to get type");
 
-        ret = dict_get_str (dict, "default-soft-limit", &default_sl);
-        if (ret)
-                gf_log (frame->this->name, GF_LOG_TRACE, "failed to get "
-                        "default soft limit");
+    ret = dict_get_int32_sizen(dict, "count", &entry_count);
+    if (ret)
+        gf_log(frame->this->name, GF_LOG_TRACE, "failed to get count");
 
-        // default-soft-limit is part of rsp_dict only iff we sent
-        // GLUSTER_CLI_QUOTA with type being GF_QUOTA_OPTION_TYPE_LIST
-        if (default_sl) {
-                default_sl_dup = gf_strdup (default_sl);
-                if (!default_sl_dup) {
-                        ret = -1;
-                        goto out;
-                }
-                ret = dict_set_dynstr (aggr, "default-soft-limit",
-                                       default_sl_dup);
-                if (ret) {
-                        gf_log (frame->this->name, GF_LOG_TRACE,
-                                "failed to set default soft limit");
-                        GF_FREE (default_sl_dup);
-                }
+    if ((type == GF_QUOTA_OPTION_TYPE_LIST) ||
+        (type == GF_QUOTA_OPTION_TYPE_LIST_OBJECTS)) {
+        gf_cli_quota_list(local, volname, dict, default_sl, entry_count,
+                          rsp.op_ret, rsp.op_errno, rsp.op_errstr);
+
+        if (global_state->mode & GLUSTER_MODE_XML) {
+            ret = cli_xml_output_vol_quota_limit_list_end(local);
+            if (ret < 0) {
+                ret = -1;
+                gf_log("cli", GF_LOG_ERROR, XML_ERROR);
+            }
+            goto out;
         }
+    }
 
-        ret = dict_get_int32 (dict, "type", &type);
-        if (ret)
-                gf_log (frame->this->name, GF_LOG_TRACE,
-                        "failed to get type");
+xml_output:
 
-        ret = dict_get_int32 (dict, "count", &entry_count);
+    if (global_state->mode & GLUSTER_MODE_XML) {
+        ret = cli_xml_output_str("volQuota", NULL, rsp.op_ret, rsp.op_errno,
+                                 rsp.op_errstr);
         if (ret)
-                gf_log (frame->this->name, GF_LOG_TRACE, "failed to get count");
-
-        if ((type == GF_QUOTA_OPTION_TYPE_LIST)
-            || (type == GF_QUOTA_OPTION_TYPE_LIST_OBJECTS)) {
-                gf_cli_quota_list (local, volname, dict, default_sl,
-                                   entry_count, rsp.op_ret,
-                                   rsp.op_errno, rsp.op_errstr);
+            gf_log("cli", GF_LOG_ERROR, XML_ERROR);
+        goto out;
+    }
 
-                if (global_state->mode & GLUSTER_MODE_XML) {
-                        ret = cli_xml_output_vol_quota_limit_list_end (local);
-                        if (ret < 0) {
-                                ret = -1;
-                                gf_log ("cli", GF_LOG_ERROR, "Error in printing"
-                                       " xml output");
-                        }
-                        goto out;
-                }
-        }
+    if (!rsp.op_ret && type != GF_QUOTA_OPTION_TYPE_LIST &&
+        type != GF_QUOTA_OPTION_TYPE_LIST_OBJECTS)
+        cli_out("volume quota : success");
 
-xml_output:
-        if (global_state->mode & GLUSTER_MODE_XML) {
-                ret = cli_xml_output_str ("volQuota", NULL, rsp.op_ret,
-                                          rsp.op_errno, rsp.op_errstr);
-                if (ret)
-                        gf_log ("cli", GF_LOG_ERROR,
-                                "Error outputting to xml");
-                goto out;
-        }
+    ret = rsp.op_ret;
+out:
 
-        if (!rsp.op_ret && type != GF_QUOTA_OPTION_TYPE_LIST
-                        && type != GF_QUOTA_OPTION_TYPE_LIST_OBJECTS)
-                cli_out ("volume quota : success");
+    if ((type == GF_QUOTA_OPTION_TYPE_LIST) ||
+        (type == GF_QUOTA_OPTION_TYPE_LIST_OBJECTS)) {
+        gluster_remove_auxiliary_mount(volname);
+    }
 
-        ret = rsp.op_ret;
-out:
-        cli_cmd_broadcast_response (ret);
-        if (dict)
-                dict_unref (dict);
+    cli_cmd_broadcast_response(ret);
+    if (dict)
+        dict_unref(dict);
 
-        free (rsp.dict.dict_val);
+    gf_free_xdr_cli_rsp(rsp);
 
-        return ret;
+    return ret;
 }
 
-int
-gf_cli_getspec_cbk (struct rpc_req *req, struct iovec *iov,
-                       int count, void *myframe)
+static int
+gf_cli_getspec_cbk(struct rpc_req *req, struct iovec *iov, int count,
+                   void *myframe)
 {
-        gf_getspec_rsp          rsp   = {0,};
-        int                     ret   = -1;
-        char                   *spec  = NULL;
-        call_frame_t           *frame = NULL;
+    gf_getspec_rsp rsp = {
+        0,
+    };
+    int ret = -1;
+    char *spec = NULL;
 
-        if (-1 == req->rpc_status) {
-                goto out;
-        }
+    GF_ASSERT(myframe);
 
-        frame = myframe;
+    if (-1 == req->rpc_status) {
+        goto out;
+    }
 
-        ret = xdr_to_generic (*iov, &rsp, (xdrproc_t)xdr_gf_getspec_rsp);
-        if (ret < 0) {
-                gf_log (frame->this->name, GF_LOG_ERROR,
-                        "Failed to decode xdr response");
-                goto out;
-        }
+    ret = xdr_to_generic(*iov, &rsp, (xdrproc_t)xdr_gf_getspec_rsp);
+    if (ret < 0) {
+        gf_log(((call_frame_t *)myframe)->this->name, GF_LOG_ERROR,
+               XDR_DECODE_FAIL);
+        goto out;
+    }
 
-        if (rsp.op_ret == -1) {
-                gf_log (frame->this->name, GF_LOG_ERROR,
-                        "getspec failed");
-                goto out;
-        }
+    if (rsp.op_ret == -1) {
+        gf_log(((call_frame_t *)myframe)->this->name, GF_LOG_ERROR,
+               "getspec failed");
+        ret = -1;
+        goto out;
+    }
 
-        gf_log ("cli", GF_LOG_INFO, "Received resp to getspec");
+    gf_log("cli", GF_LOG_INFO, "Received resp to getspec");
 
-        spec = GF_MALLOC (rsp.op_ret + 1, cli_mt_char);
-        if (!spec) {
-                gf_log("", GF_LOG_ERROR, "out of memory");
-                goto out;
-        }
-        memcpy (spec, rsp.spec, rsp.op_ret);
-        spec[rsp.op_ret] = '\0';
-        cli_out ("%s", spec);
-        GF_FREE (spec);
+    spec = GF_MALLOC(rsp.op_ret + 1, cli_mt_char);
+    if (!spec) {
+        gf_log("", GF_LOG_ERROR, "out of memory");
+        ret = -1;
+        goto out;
+    }
+    memcpy(spec, rsp.spec, rsp.op_ret);
+    spec[rsp.op_ret] = '\0';
+    cli_out("%s", spec);
+    GF_FREE(spec);
 
-        ret = 0;
+    ret = 0;
 
 out:
-        cli_cmd_broadcast_response (ret);
-        return ret;
+    cli_cmd_broadcast_response(ret);
+    gf_free_xdr_getspec_rsp(rsp);
+    return ret;
 }
 
-int
-gf_cli_pmap_b2p_cbk (struct rpc_req *req, struct iovec *iov,
-                        int count, void *myframe)
+static int
+gf_cli_pmap_b2p_cbk(struct rpc_req *req, struct iovec *iov, int count,
+                    void *myframe)
 {
-        pmap_port_by_brick_rsp rsp = {0,};
-        int                     ret   = -1;
-        char                   *spec  = NULL;
-        call_frame_t           *frame = NULL;
+    pmap_port_by_brick_rsp rsp = {
+        0,
+    };
+    int ret = -1;
+    char *spec = NULL;
 
-        if (-1 == req->rpc_status) {
-                goto out;
-        }
+    GF_ASSERT(myframe);
 
-        frame = myframe;
+    if (-1 == req->rpc_status) {
+        goto out;
+    }
 
-        ret = xdr_to_generic (*iov, &rsp, (xdrproc_t)xdr_pmap_port_by_brick_rsp);
-        if (ret < 0) {
-                gf_log (frame->this->name, GF_LOG_ERROR,
-                        "Failed to decode xdr response");
-                goto out;
-        }
+    ret = xdr_to_generic(*iov, &rsp, (xdrproc_t)xdr_pmap_port_by_brick_rsp);
+    if (ret < 0) {
+        gf_log(((call_frame_t *)myframe)->this->name, GF_LOG_ERROR,
+               XDR_DECODE_FAIL);
+        goto out;
+    }
 
-        if (rsp.op_ret == -1) {
-                gf_log (frame->this->name, GF_LOG_ERROR,
-                        "pump_b2p failed");
-                goto out;
-        }
+    if (rsp.op_ret == -1) {
+        gf_log(((call_frame_t *)myframe)->this->name, GF_LOG_ERROR,
+               "pump_b2p failed");
+        ret = -1;
+        goto out;
+    }
 
-        gf_log ("cli", GF_LOG_INFO, "Received resp to pmap b2p");
+    gf_log("cli", GF_LOG_INFO, "Received resp to pmap b2p");
 
-        cli_out ("%d", rsp.port);
-        GF_FREE (spec);
+    cli_out("%d", rsp.port);
+    GF_FREE(spec);
 
-        ret = rsp.op_ret;
+    ret = rsp.op_ret;
 
 out:
-        cli_cmd_broadcast_response (ret);
-        return ret;
+    cli_cmd_broadcast_response(ret);
+    return ret;
 }
 
-
-int32_t
-gf_cli_probe (call_frame_t *frame, xlator_t *this,
-                 void *data)
+static int32_t
+gf_cli_probe(call_frame_t *frame, xlator_t *this, void *data)
 {
-        gf_cli_req         req      = {{0,},};
-        int                ret      = 0;
-        dict_t            *dict     = NULL;
-        int                port     = 0;
-
-        if (!frame || !this ||  !data) {
-                ret = -1;
-                goto out;
-        }
-
-        dict = data;
-
-        ret = dict_get_int32 (dict, "port", &port);
-        if (ret) {
-                ret = dict_set_int32 (dict, "port", CLI_GLUSTERD_PORT);
-                if (ret)
-                        goto out;
-        }
+    gf_cli_req req = {
+        {
+            0,
+        },
+    };
+    int ret = 0;
+    dict_t *dict = NULL;
+    int port = 0;
+
+    if (!frame || !this || !data) {
+        ret = -1;
+        goto out;
+    }
+
+    dict = data;
+
+    ret = dict_get_int32_sizen(dict, "port", &port);
+    if (ret) {
+        ret = dict_set_int32_sizen(dict, "port", CLI_GLUSTERD_PORT);
+        if (ret)
+            goto out;
+    }
 
-        ret = cli_to_glusterd (&req, frame, gf_cli_probe_cbk,
-                               (xdrproc_t) xdr_gf_cli_req, dict,
-                               GLUSTER_CLI_PROBE, this, cli_rpc_prog, NULL);
+    ret = cli_to_glusterd(&req, frame, gf_cli_probe_cbk,
+                          (xdrproc_t)xdr_gf_cli_req, dict, GLUSTER_CLI_PROBE,
+                          this, cli_rpc_prog, NULL);
 
 out:
-        GF_FREE (req.dict.dict_val);
-        gf_log ("cli", GF_LOG_DEBUG, "Returning %d", ret);
+    GF_FREE(req.dict.dict_val);
+    gf_log("cli", GF_LOG_DEBUG, RETURNING, ret);
 
-        return ret;
+    return ret;
 }
 
-int32_t
-gf_cli_deprobe (call_frame_t *frame, xlator_t *this,
-                   void *data)
+static int32_t
+gf_cli_deprobe(call_frame_t *frame, xlator_t *this, void *data)
 {
-        gf_cli_req           req      = {{0,},};
-        int                  ret      = 0;
-        dict_t              *dict     = NULL;
-        int                  port     = 0;
-        int                  flags    = 0;
-
-        if (!frame || !this ||  !data) {
-                ret = -1;
-                goto out;
-        }
-
-        dict = data;
-        ret = dict_get_int32 (dict, "port", &port);
-        if (ret) {
-                ret = dict_set_int32 (dict, "port", CLI_GLUSTERD_PORT);
-                if (ret)
-                        goto out;
-        }
+    gf_cli_req req = {
+        {
+            0,
+        },
+    };
+    int ret = 0;
+    dict_t *dict = NULL;
+    int port = 0;
+    int flags = 0;
+
+    if (!frame || !this || !data) {
+        ret = -1;
+        goto out;
+    }
+
+    dict = data;
+    ret = dict_get_int32_sizen(dict, "port", &port);
+    if (ret) {
+        ret = dict_set_int32_sizen(dict, "port", CLI_GLUSTERD_PORT);
+        if (ret)
+            goto out;
+    }
 
-        ret = dict_get_int32 (dict, "flags", &flags);
-        if (ret) {
-                ret = dict_set_int32 (dict, "flags", 0);
-                if (ret)
-                        goto out;
-        }
+    ret = dict_get_int32_sizen(dict, "flags", &flags);
+    if (ret) {
+        ret = dict_set_int32_sizen(dict, "flags", 0);
+        if (ret)
+            goto out;
+    }
 
-        ret = cli_to_glusterd (&req, frame, gf_cli_deprobe_cbk,
-                               (xdrproc_t)xdr_gf_cli_req, dict,
-                              GLUSTER_CLI_DEPROBE, this, cli_rpc_prog, NULL);
+    ret = cli_to_glusterd(&req, frame, gf_cli_deprobe_cbk,
+                          (xdrproc_t)xdr_gf_cli_req, dict, GLUSTER_CLI_DEPROBE,
+                          this, cli_rpc_prog, NULL);
 
 out:
-        GF_FREE (req.dict.dict_val);
-        gf_log ("cli", GF_LOG_DEBUG, "Returning %d", ret);
+    GF_FREE(req.dict.dict_val);
+    gf_log("cli", GF_LOG_DEBUG, RETURNING, ret);
 
-        return ret;
+    return ret;
 }
 
-int32_t
-gf_cli_list_friends (call_frame_t *frame, xlator_t *this,
-                     void *data)
+static int32_t
+gf_cli_list_friends(call_frame_t *frame, xlator_t *this, void *data)
 {
-        gf1_cli_peer_list_req   req = {0,};
-        int                     ret = 0;
-        unsigned long           flags = 0;
+    gf1_cli_peer_list_req req = {
+        0,
+    };
+    int ret = 0;
+    unsigned long flags = 0;
 
-        if (!frame || !this) {
-                ret = -1;
-                goto out;
-        }
+    if (!frame || !this) {
+        ret = -1;
+        goto out;
+    }
 
-        GF_ASSERT (frame->local == NULL);
+    GF_ASSERT(frame->local == NULL);
 
-        flags = (long)data;
-        req.flags = flags;
-        frame->local = (void*)flags;
-        ret = cli_cmd_submit (NULL, &req, frame, cli_rpc_prog,
-                              GLUSTER_CLI_LIST_FRIENDS, NULL,
-                              this, gf_cli_list_friends_cbk,
-                              (xdrproc_t) xdr_gf1_cli_peer_list_req);
+    flags = (long)data;
+    req.flags = flags;
+    frame->local = (void *)flags;
+    ret = cli_cmd_submit(
+        NULL, &req, frame, cli_rpc_prog, GLUSTER_CLI_LIST_FRIENDS, NULL, this,
+        gf_cli_list_friends_cbk, (xdrproc_t)xdr_gf1_cli_peer_list_req);
 
 out:
-        if (ret) {
-                /*
-                 * If everything goes fine, gf_cli_list_friends_cbk()
-                 * [invoked through cli_cmd_submit()]resets the
-                 * frame->local to NULL. In case cli_cmd_submit()
-                 * fails in between, RESET frame->local here.
-                 */
-                frame->local = NULL;
-        }
-        gf_log ("cli", GF_LOG_DEBUG, "Returning %d", ret);
-        return ret;
+    if (ret && frame) {
+        /*
+         * If everything goes fine, gf_cli_list_friends_cbk()
+         * [invoked through cli_cmd_submit()]resets the
+         * frame->local to NULL. In case cli_cmd_submit()
+         * fails in between, RESET frame->local here.
+         */
+        frame->local = NULL;
+    }
+    gf_log("cli", GF_LOG_DEBUG, RETURNING, ret);
+    return ret;
 }
 
-int32_t
-gf_cli_get_next_volume (call_frame_t *frame, xlator_t *this,
-                           void *data)
+static int32_t
+gf_cli_get_state(call_frame_t *frame, xlator_t *this, void *data)
 {
+    gf_cli_req req = {
+        {
+            0,
+        },
+    };
+    int ret = 0;
+    dict_t *dict = NULL;
 
-        int                             ret = 0;
-        cli_cmd_volume_get_ctx_t        *ctx = NULL;
-        cli_local_t                     *local = NULL;
+    dict = data;
 
-        if (!frame || !this || !data) {
-                ret = -1;
-                goto out;
-        }
-
-        ctx = data;
-        local = frame->local;
+    ret = cli_to_glusterd(&req, frame, gf_cli_get_state_cbk,
+                          (xdrproc_t)xdr_gf_cli_req, dict,
+                          GLUSTER_CLI_GET_STATE, this, cli_rpc_prog, NULL);
+    gf_log("cli", GF_LOG_DEBUG, RETURNING, ret);
+    GF_FREE(req.dict.dict_val);
 
-        if (global_state->mode & GLUSTER_MODE_XML) {
-                ret = cli_xml_output_vol_info_begin (local, 0, 0, "");
-                if (ret) {
-                        gf_log ("cli", GF_LOG_ERROR, "Error outputting to xml");
-                        goto out;
-                }
-        }
+    return ret;
+}
 
-        ret = gf_cli_get_volume (frame, this, data);
+static int32_t
+gf_cli_get_next_volume(call_frame_t *frame, xlator_t *this, void *data)
+{
+    int ret = 0;
+    cli_cmd_volume_get_ctx_t *ctx = NULL;
+    cli_local_t *local = NULL;
 
+    if (!frame || !this || !data) {
+        ret = -1;
+        goto out;
+    }
 
-        if (!local || !local->get_vol.volname) {
-                if ((global_state->mode & GLUSTER_MODE_XML))
-                        goto end_xml;
+    ctx = data;
+    local = frame->local;
 
-                cli_err ("No volumes present");
-                goto out;
+    if (global_state->mode & GLUSTER_MODE_XML) {
+        ret = cli_xml_output_vol_info_begin(local, 0, 0, "");
+        if (ret) {
+            gf_log("cli", GF_LOG_ERROR, XML_ERROR);
+            goto out;
         }
+    }
 
+    ret = gf_cli_get_volume(frame, this, data);
 
-        ctx->volname = local->get_vol.volname;
+    if (!local || !local->get_vol.volname) {
+        if ((global_state->mode & GLUSTER_MODE_XML))
+            goto end_xml;
 
-        while (ctx->volname) {
-                ret = gf_cli_get_volume (frame, this, ctx);
-                if (ret)
-                        goto out;
-                ctx->volname = local->get_vol.volname;
-        }
+        cli_err("No volumes present");
+        goto out;
+    }
+
+    ctx->volname = local->get_vol.volname;
+
+    while (ctx->volname) {
+        ret = gf_cli_get_volume(frame, this, ctx);
+        if (ret)
+            goto out;
+        ctx->volname = local->get_vol.volname;
+    }
 
 end_xml:
-        if (global_state->mode & GLUSTER_MODE_XML) {
-                ret = cli_xml_output_vol_info_end (local);
-                if (ret)
-                        gf_log ("cli", GF_LOG_ERROR, "Error outputting to xml");
-        }
+    if (global_state->mode & GLUSTER_MODE_XML) {
+        ret = cli_xml_output_vol_info_end(local);
+        if (ret)
+            gf_log("cli", GF_LOG_ERROR, XML_ERROR);
+    }
 
 out:
-        gf_log ("cli", GF_LOG_DEBUG, "Returning %d", ret);
-        return ret;
+    gf_log("cli", GF_LOG_DEBUG, RETURNING, ret);
+    return ret;
 }
 
-int32_t
-gf_cli_get_volume (call_frame_t *frame, xlator_t *this,
-                      void *data)
-{
-        gf_cli_req                      req = {{0,}};
-        int                             ret = 0;
-        cli_cmd_volume_get_ctx_t        *ctx = NULL;
-        dict_t                          *dict = NULL;
-        int32_t                         flags = 0;
-
-        if (!frame || !this || !data) {
-                ret = -1;
-                goto out;
-        }
+static int32_t
+gf_cli_get_volume(call_frame_t *frame, xlator_t *this, void *data)
+{
+    gf_cli_req req = {{
+        0,
+    }};
+    int ret = 0;
+    cli_cmd_volume_get_ctx_t *ctx = NULL;
+    dict_t *dict = NULL;
+    int32_t flags = 0;
+
+    if (!this || !data) {
+        ret = -1;
+        goto out;
+    }
+
+    ctx = data;
+
+    dict = dict_new();
+    if (!dict) {
+        gf_log(THIS->name, GF_LOG_ERROR, "Failed to create the dict");
+        ret = -1;
+        goto out;
+    }
+
+    if (ctx->volname) {
+        ret = dict_set_str_sizen(dict, "volname", ctx->volname);
+        if (ret)
+            goto out;
+    }
 
-        ctx = data;
+    flags = ctx->flags;
+    ret = dict_set_int32_sizen(dict, "flags", flags);
+    if (ret) {
+        gf_log(frame->this->name, GF_LOG_ERROR, "failed to set flags");
+        goto out;
+    }
 
-        dict = dict_new ();
-        if (!dict)
-                goto out;
+    ret = dict_allocate_and_serialize(dict, &req.dict.dict_val,
+                                      &req.dict.dict_len);
+    if (ret) {
+        gf_log(frame->this->name, GF_LOG_ERROR, DICT_SERIALIZE_FAIL);
+        goto out;
+    }
 
-        if (ctx->volname) {
-                ret = dict_set_str (dict, "volname", ctx->volname);
-                if (ret)
-                        goto out;
-        }
-
-        flags = ctx->flags;
-        ret = dict_set_int32 (dict, "flags", flags);
-        if (ret) {
-                gf_log (frame->this->name, GF_LOG_ERROR, "failed to set flags");
-                goto out;
-        }
-
-        ret = dict_allocate_and_serialize (dict, &req.dict.dict_val,
-                                           &req.dict.dict_len);
-
-        ret = cli_cmd_submit (NULL, &req, frame, cli_rpc_prog,
-                              GLUSTER_CLI_GET_VOLUME, NULL,
-                              this, gf_cli_get_volume_cbk,
-                              (xdrproc_t) xdr_gf_cli_req);
+    ret = cli_cmd_submit(NULL, &req, frame, cli_rpc_prog,
+                         GLUSTER_CLI_GET_VOLUME, NULL, this,
+                         gf_cli_get_volume_cbk, (xdrproc_t)xdr_gf_cli_req);
 
 out:
-        if (dict)
-                dict_unref (dict);
+    if (dict)
+        dict_unref(dict);
 
-        GF_FREE (req.dict.dict_val);
+    GF_FREE(req.dict.dict_val);
 
-        gf_log ("cli", GF_LOG_DEBUG, "Returning %d", ret);
-        return ret;
+    gf_log("cli", GF_LOG_DEBUG, RETURNING, ret);
+    return ret;
 }
 
-int32_t
-gf_cli3_1_uuid_get (call_frame_t *frame, xlator_t *this,
-                      void *data)
+static int32_t
+gf_cli3_1_uuid_get(call_frame_t *frame, xlator_t *this, void *data)
 {
-        gf_cli_req                      req = {{0,}};
-        int                             ret = 0;
-        dict_t                          *dict = NULL;
-
-        if (!frame || !this || !data) {
-                ret = -1;
-                goto out;
-        }
+    gf_cli_req req = {{
+        0,
+    }};
+    int ret = 0;
+    dict_t *dict = NULL;
 
-        dict = data;
-        ret = cli_to_glusterd (&req, frame, gf_cli3_1_uuid_get_cbk,
-                               (xdrproc_t)xdr_gf_cli_req, dict,
-                               GLUSTER_CLI_UUID_GET, this, cli_rpc_prog,
-                               NULL);
-out:
-        gf_log ("cli", GF_LOG_DEBUG, "Returning %d", ret);
-        return ret;
+    dict = data;
+    ret = cli_to_glusterd(&req, frame, gf_cli3_1_uuid_get_cbk,
+                          (xdrproc_t)xdr_gf_cli_req, dict, GLUSTER_CLI_UUID_GET,
+                          this, cli_rpc_prog, NULL);
+    gf_log("cli", GF_LOG_DEBUG, RETURNING, ret);
+    GF_FREE(req.dict.dict_val);
+    return ret;
 }
 
-int32_t
-gf_cli3_1_uuid_reset (call_frame_t *frame, xlator_t *this,
-                      void *data)
+static int32_t
+gf_cli3_1_uuid_reset(call_frame_t *frame, xlator_t *this, void *data)
 {
-        gf_cli_req                      req = {{0,}};
-        int                             ret = 0;
-        dict_t                          *dict = NULL;
+    gf_cli_req req = {{
+        0,
+    }};
+    int ret = 0;
+    dict_t *dict = NULL;
 
-        if (!frame || !this || !data) {
-                ret = -1;
-                goto out;
-        }
-
-        dict = data;
-        ret = cli_to_glusterd (&req, frame, gf_cli3_1_uuid_reset_cbk,
-                               (xdrproc_t)xdr_gf_cli_req, dict,
-                               GLUSTER_CLI_UUID_RESET, this, cli_rpc_prog,
-                               NULL);
-out:
-        gf_log ("cli", GF_LOG_DEBUG, "Returning %d", ret);
-        return ret;
+    dict = data;
+    ret = cli_to_glusterd(&req, frame, gf_cli3_1_uuid_reset_cbk,
+                          (xdrproc_t)xdr_gf_cli_req, dict,
+                          GLUSTER_CLI_UUID_RESET, this, cli_rpc_prog, NULL);
+    gf_log("cli", GF_LOG_DEBUG, RETURNING, ret);
+    GF_FREE(req.dict.dict_val);
+    return ret;
 }
 
-int32_t
-gf_cli_create_volume (call_frame_t *frame, xlator_t *this,
-                         void *data)
+static int32_t
+gf_cli_create_volume(call_frame_t *frame, xlator_t *this, void *data)
 {
-        gf_cli_req              req = {{0,}};
-        int                     ret = 0;
-        dict_t                  *dict = NULL;
-
-        if (!frame || !this ||  !data) {
-                ret = -1;
-                goto out;
-        }
-
-        dict = data;
+    gf_cli_req req = {{
+        0,
+    }};
+    int ret = 0;
+    dict_t *dict = NULL;
 
-        ret = cli_to_glusterd (&req, frame, gf_cli_create_volume_cbk,
-                               (xdrproc_t) xdr_gf_cli_req, dict,
-                               GLUSTER_CLI_CREATE_VOLUME, this, cli_rpc_prog,
-                               NULL);
+    dict = data;
 
-out:
-        gf_log ("cli", GF_LOG_DEBUG, "Returning %d", ret);
+    ret = cli_to_glusterd(&req, frame, gf_cli_create_volume_cbk,
+                          (xdrproc_t)xdr_gf_cli_req, dict,
+                          GLUSTER_CLI_CREATE_VOLUME, this, cli_rpc_prog, NULL);
+    gf_log("cli", GF_LOG_DEBUG, RETURNING, ret);
 
-        GF_FREE (req.dict.dict_val);
+    GF_FREE(req.dict.dict_val);
 
-        return ret;
+    return ret;
 }
 
-int32_t
-gf_cli_delete_volume (call_frame_t *frame, xlator_t *this,
-                         void *data)
+static int32_t
+gf_cli_delete_volume(call_frame_t *frame, xlator_t *this, void *data)
 {
-        gf_cli_req              req = {{0,}};
-        int                     ret = 0;
-        dict_t                  *dict = NULL;
-
-        if (!frame || !this ||  !data) {
-                ret = -1;
-                goto out;
-        }
+    gf_cli_req req = {{
+        0,
+    }};
+    int ret = 0;
+    dict_t *dict = NULL;
 
-        dict = data;
+    dict = data;
 
-        ret = cli_to_glusterd (&req, frame, gf_cli_delete_volume_cbk,
-                               (xdrproc_t) xdr_gf_cli_req, dict,
-                               GLUSTER_CLI_DELETE_VOLUME, this, cli_rpc_prog,
-                               NULL);
+    ret = cli_to_glusterd(&req, frame, gf_cli_delete_volume_cbk,
+                          (xdrproc_t)xdr_gf_cli_req, dict,
+                          GLUSTER_CLI_DELETE_VOLUME, this, cli_rpc_prog, NULL);
+    GF_FREE(req.dict.dict_val);
+    gf_log("cli", GF_LOG_DEBUG, RETURNING, ret);
 
-out:
-        GF_FREE (req.dict.dict_val);
-        gf_log ("cli", GF_LOG_DEBUG, "Returning %d", ret);
-
-        return ret;
+    return ret;
 }
 
-int32_t
-gf_cli_start_volume (call_frame_t *frame, xlator_t *this,
-                         void *data)
+static int32_t
+gf_cli_start_volume(call_frame_t *frame, xlator_t *this, void *data)
 {
-        gf_cli_req              req = {{0,}};
-        int                     ret = 0;
-        dict_t                  *dict = NULL;
-
-        if (!frame || !this ||  !data) {
-                ret = -1;
-                goto out;
-        }
+    gf_cli_req req = {{
+        0,
+    }};
+    int ret = 0;
+    dict_t *dict = NULL;
 
-        dict = data;
+    dict = data;
 
-        ret = cli_to_glusterd (&req, frame, gf_cli_start_volume_cbk,
-                               (xdrproc_t) xdr_gf_cli_req, dict,
-                               GLUSTER_CLI_START_VOLUME, this, cli_rpc_prog,
-                               NULL);
+    ret = cli_to_glusterd(&req, frame, gf_cli_start_volume_cbk,
+                          (xdrproc_t)xdr_gf_cli_req, dict,
+                          GLUSTER_CLI_START_VOLUME, this, cli_rpc_prog, NULL);
 
-out:
-        gf_log ("cli", GF_LOG_DEBUG, "Returning %d", ret);
+    gf_log("cli", GF_LOG_DEBUG, RETURNING, ret);
+    GF_FREE(req.dict.dict_val);
 
-        return ret;
+    return ret;
 }
 
-int32_t
-gf_cli_stop_volume (call_frame_t *frame, xlator_t *this,
-                         void *data)
+static int32_t
+gf_cli_stop_volume(call_frame_t *frame, xlator_t *this, void *data)
 {
-        gf_cli_req             req = {{0,}};
-        int                    ret = 0;
-        dict_t                 *dict = data;
+    gf_cli_req req = {{
+        0,
+    }};
+    int ret = 0;
+    dict_t *dict = data;
 
-        if (!frame || !this ||  !data) {
-                ret = -1;
-                goto out;
-        }
-
-        dict = data;
-
-        ret = cli_to_glusterd (&req, frame, gf_cli_stop_volume_cbk,
-                               (xdrproc_t) xdr_gf_cli_req, dict,
-                               GLUSTER_CLI_STOP_VOLUME, this, cli_rpc_prog,
-                               NULL);
+    dict = data;
 
-out:
-        gf_log ("cli", GF_LOG_DEBUG, "Returning %d", ret);
+    ret = cli_to_glusterd(&req, frame, gf_cli_stop_volume_cbk,
+                          (xdrproc_t)xdr_gf_cli_req, dict,
+                          GLUSTER_CLI_STOP_VOLUME, this, cli_rpc_prog, NULL);
+    gf_log("cli", GF_LOG_DEBUG, RETURNING, ret);
+    GF_FREE(req.dict.dict_val);
 
-        return ret;
+    return ret;
 }
 
-int32_t
-gf_cli_defrag_volume (call_frame_t *frame, xlator_t *this,
-                         void *data)
+static int32_t
+gf_cli_defrag_volume(call_frame_t *frame, xlator_t *this, void *data)
 {
-        gf_cli_req              req     =  {{0,}};
-        int                     ret     = 0;
-        dict_t                 *dict    = NULL;
+    gf_cli_req req = {{
+        0,
+    }};
+    int ret = 0;
+    dict_t *dict = NULL;
 
-        if (!frame || !this ||  !data) {
-                ret = -1;
-                goto out;
-        }
-
-        dict = data;
+    dict = data;
 
-        ret = cli_to_glusterd (&req, frame, gf_cli_defrag_volume_cbk,
-                               (xdrproc_t) xdr_gf_cli_req, dict,
-                               GLUSTER_CLI_DEFRAG_VOLUME, this, cli_rpc_prog,
-                               NULL);
+    ret = cli_to_glusterd(&req, frame, gf_cli_defrag_volume_cbk,
+                          (xdrproc_t)xdr_gf_cli_req, dict,
+                          GLUSTER_CLI_DEFRAG_VOLUME, this, cli_rpc_prog, NULL);
+    gf_log("cli", GF_LOG_DEBUG, RETURNING, ret);
+    GF_FREE(req.dict.dict_val);
 
-out:
-        gf_log ("cli", GF_LOG_DEBUG, "Returning %d", ret);
-
-        return ret;
+    return ret;
 }
 
-int32_t
-gf_cli_rename_volume (call_frame_t *frame, xlator_t *this,
-                         void *data)
+static int32_t
+gf_cli_rename_volume(call_frame_t *frame, xlator_t *this, void *data)
 {
-        gf_cli_req              req = {{0,}};
-        int                     ret = 0;
-        dict_t                  *dict = NULL;
-
-        if (!frame || !this ||  !data) {
-                ret = -1;
-                goto out;
-        }
+    gf_cli_req req = {{
+        0,
+    }};
+    int ret = 0;
+    dict_t *dict = NULL;
 
-        dict = data;
+    if (!frame || !this || !data) {
+        ret = -1;
+        goto out;
+    }
 
-        ret = dict_allocate_and_serialize (dict, &req.dict.dict_val,
-                                           &req.dict.dict_len);
-        if (ret < 0) {
-                gf_log (this->name, GF_LOG_ERROR,
-                        "failed to serialize the data");
+    dict = data;
 
-                goto out;
-        }
+    ret = dict_allocate_and_serialize(dict, &req.dict.dict_val,
+                                      &req.dict.dict_len);
+    if (ret < 0) {
+        gf_log(this->name, GF_LOG_ERROR, DICT_SERIALIZE_FAIL);
 
+        goto out;
+    }
 
-        ret = cli_cmd_submit (NULL, &req, frame, cli_rpc_prog,
-                              GLUSTER_CLI_RENAME_VOLUME, NULL,
-                              this, gf_cli_rename_volume_cbk,
-                              (xdrproc_t) xdr_gf_cli_req);
+    ret = cli_cmd_submit(NULL, &req, frame, cli_rpc_prog,
+                         GLUSTER_CLI_RENAME_VOLUME, NULL, this,
+                         gf_cli_rename_volume_cbk, (xdrproc_t)xdr_gf_cli_req);
 
 out:
-        gf_log ("cli", GF_LOG_DEBUG, "Returning %d", ret);
+    GF_FREE(req.dict.dict_val);
+    gf_log("cli", GF_LOG_DEBUG, RETURNING, ret);
 
-        return ret;
+    return ret;
 }
 
-int32_t
-gf_cli_reset_volume (call_frame_t *frame, xlator_t *this,
-                        void *data)
+static int32_t
+gf_cli_reset_volume(call_frame_t *frame, xlator_t *this, void *data)
 {
-        gf_cli_req              req =  {{0,} };
-        int                     ret = 0;
-        dict_t                  *dict = NULL;
+    gf_cli_req req = {{
+        0,
+    }};
+    int ret = 0;
+    dict_t *dict = NULL;
 
-        if (!frame || !this ||  !data) {
-                ret = -1;
-                goto out;
-        }
+    dict = data;
 
-        dict = data;
-
-        ret = cli_to_glusterd (&req, frame, gf_cli_reset_volume_cbk,
-                               (xdrproc_t) xdr_gf_cli_req, dict,
-                               GLUSTER_CLI_RESET_VOLUME, this, cli_rpc_prog,
-                               NULL);
-
-out:
-        gf_log ("cli", GF_LOG_DEBUG, "Returning %d", ret);
-        return ret;
+    ret = cli_to_glusterd(&req, frame, gf_cli_reset_volume_cbk,
+                          (xdrproc_t)xdr_gf_cli_req, dict,
+                          GLUSTER_CLI_RESET_VOLUME, this, cli_rpc_prog, NULL);
+    gf_log("cli", GF_LOG_DEBUG, RETURNING, ret);
+    GF_FREE(req.dict.dict_val);
+    return ret;
 }
 
-int32_t
-gf_cli_ganesha (call_frame_t *frame, xlator_t *this, void *data)
+static int32_t
+gf_cli_ganesha(call_frame_t *frame, xlator_t *this, void *data)
 {
-        gf_cli_req              req =  { {0,} } ;
-        int                     ret = 0;
-        dict_t                  *dict = NULL;
-
-        if (!frame || !this ||  !data) {
-                ret = -1;
-                goto out;
-        }
+    gf_cli_req req = {{
+        0,
+    }};
+    int ret = 0;
+    dict_t *dict = NULL;
 
-        dict = data;
+    dict = data;
 
-        ret = cli_to_glusterd (&req, frame, gf_cli_ganesha_cbk,
-                               (xdrproc_t) xdr_gf_cli_req, dict,
-                               GLUSTER_CLI_GANESHA, this, cli_rpc_prog,
-                               NULL);
-out:
-        gf_log ("cli", GF_LOG_DEBUG, "Returning %d", ret);
+    ret = cli_to_glusterd(&req, frame, gf_cli_ganesha_cbk,
+                          (xdrproc_t)xdr_gf_cli_req, dict, GLUSTER_CLI_GANESHA,
+                          this, cli_rpc_prog, NULL);
+    gf_log("cli", GF_LOG_DEBUG, RETURNING, ret);
 
-        return ret;
+    return ret;
 }
 
-int32_t
-gf_cli_set_volume (call_frame_t *frame, xlator_t *this,
-                         void *data)
+static int32_t
+gf_cli_set_volume(call_frame_t *frame, xlator_t *this, void *data)
 {
-        gf_cli_req              req =  {{0,} };
-        int                     ret = 0;
-        dict_t                  *dict = NULL;
+    gf_cli_req req = {{
+        0,
+    }};
+    int ret = 0;
+    dict_t *dict = NULL;
 
-        if (!frame || !this ||  !data) {
-                ret = -1;
-                goto out;
-        }
-
-        dict = data;
+    dict = data;
 
-        ret = cli_to_glusterd (&req, frame, gf_cli_set_volume_cbk,
-                               (xdrproc_t) xdr_gf_cli_req, dict,
-                               GLUSTER_CLI_SET_VOLUME, this, cli_rpc_prog,
-                               NULL);
+    ret = cli_to_glusterd(&req, frame, gf_cli_set_volume_cbk,
+                          (xdrproc_t)xdr_gf_cli_req, dict,
+                          GLUSTER_CLI_SET_VOLUME, this, cli_rpc_prog, NULL);
+    gf_log("cli", GF_LOG_DEBUG, RETURNING, ret);
+    GF_FREE(req.dict.dict_val);
 
-out:
-        gf_log ("cli", GF_LOG_DEBUG, "Returning %d", ret);
-
-        return ret;
+    return ret;
 }
 
 int32_t
-gf_cli_add_brick (call_frame_t *frame, xlator_t *this,
-                         void *data)
+gf_cli_add_brick(call_frame_t *frame, xlator_t *this, void *data)
 {
-        gf_cli_req              req =  {{0,} };
-        int                     ret = 0;
-        dict_t                  *dict = NULL;
-        char                    *volname = NULL;
-        int32_t                 count = 0;
-
-        if (!frame || !this ||  !data) {
-                ret = -1;
-                goto out;
-        }
+    gf_cli_req req = {{
+        0,
+    }};
+    int ret = 0;
+    dict_t *dict = NULL;
 
-        dict = data;
+    dict = data;
 
-        ret = dict_get_str (dict, "volname", &volname);
+    ret = cli_to_glusterd(&req, frame, gf_cli_add_brick_cbk,
+                          (xdrproc_t)xdr_gf_cli_req, dict,
+                          GLUSTER_CLI_ADD_BRICK, this, cli_rpc_prog, NULL);
 
-        if (ret)
-                goto out;
+    gf_log("cli", GF_LOG_DEBUG, RETURNING, ret);
 
-        ret = dict_get_int32 (dict, "count", &count);
-        if (ret)
-                goto out;
+    GF_FREE(req.dict.dict_val);
 
-        ret = cli_to_glusterd (&req, frame, gf_cli_add_brick_cbk,
-                               (xdrproc_t) xdr_gf_cli_req, dict,
-                               GLUSTER_CLI_ADD_BRICK, this, cli_rpc_prog, NULL);
-
-out:
-        gf_log ("cli", GF_LOG_DEBUG, "Returning %d", ret);
-
-        GF_FREE (req.dict.dict_val);
-
-        return ret;
+    return ret;
 }
 
-int32_t
-gf_cli_attach_tier (call_frame_t *frame, xlator_t *this,
-                    void *data)
-{
-        gf_cli_req              req             =  {{0,} };
-        int                     ret             = 0;
-        dict_t                  *dict           = NULL;
-        dict_t                  *newdict        = NULL;
-        char                    *tierwords[]    = {"volume", "rebalance", "",
-                                                   "tier", "start", NULL};
-        const char              **words         = (const char **)tierwords;
-        int                     wordcount       = 5;
-        char                    *volname        = NULL;
-        cli_local_t             *local          = NULL;
-        cli_local_t             *oldlocal       = NULL;
-
-        if (!frame || !this ||  !data) {
-                ret = -1;
-                goto out;
-        }
-
-        dict = data;
-
-        if (ret)
-                goto out;
-
-        ret = cli_to_glusterd (&req, frame, gf_cli_attach_tier_cbk,
-                               (xdrproc_t) xdr_gf_cli_req, dict,
-                               GLUSTER_CLI_ATTACH_TIER, this,
-                               cli_rpc_prog, NULL);
-        if (ret)
-                goto out;
-        ret = dict_get_str (dict, "volname", &volname);
-        if (ret) {
-                gf_log ("cli", GF_LOG_ERROR, "Failed to get volume name");
-                goto notify_cli;
-        }
-
-        words[2] = volname;
-        ret = cli_cmd_volume_defrag_parse ((const char **)words,
-                                           wordcount, &newdict);
-        if (ret) {
-                gf_log ("cli", GF_LOG_ERROR, "Failed to parse tier start "
-                        "command");
-                goto notify_cli;
-        }
-
-        gf_log ("cli", GF_LOG_DEBUG, "Sending tier start");
-
-        oldlocal = frame->local;
-        CLI_LOCAL_INIT (local, words, frame, newdict);
-        ret = gf_cli_defrag_volume (frame, this, newdict);
-        frame->local = oldlocal;
-        cli_local_wipe (local);
+static int32_t
+gf_cli_remove_brick(call_frame_t *frame, xlator_t *this, void *data)
+{
+    gf_cli_req req = {{
+        0,
+    }};
+    ;
+    gf_cli_req status_req = {{
+        0,
+    }};
+    ;
+    int ret = 0;
+    dict_t *dict = NULL;
+    int32_t command = 0;
+    int32_t cmd = 0;
+
+    if (!frame || !this) {
+        ret = -1;
+        goto out;
+    }
+
+    dict = data;
+
+    ret = dict_get_int32_sizen(dict, "command", &command);
+    if (ret)
+        goto out;
+
+    if ((command != GF_OP_CMD_STATUS) && (command != GF_OP_CMD_STOP)) {
+        ret = cli_to_glusterd(
+            &req, frame, gf_cli_remove_brick_cbk, (xdrproc_t)xdr_gf_cli_req,
+            dict, GLUSTER_CLI_REMOVE_BRICK, this, cli_rpc_prog, NULL);
+    } else {
+        /* Need rebalance status to be sent :-) */
+        if (command == GF_OP_CMD_STATUS)
+            cmd |= GF_DEFRAG_CMD_STATUS;
+        else
+            cmd |= GF_DEFRAG_CMD_STOP;
 
-notify_cli:
+        ret = dict_set_int32_sizen(dict, "rebalance-command", (int32_t)cmd);
         if (ret) {
-                cli_out ("Failed to run tier start. Please execute tier start "
-                         "command explictly");
-                cli_out ("Usage : gluster volume rebalance <volname> tier "
-                         "start");
-        }
-
-out:
-        gf_log ("cli", GF_LOG_DEBUG, "Returning %d", ret);
-
-        GF_FREE (req.dict.dict_val);
-        return ret;
-}
-
-int32_t
-gf_cli_tier (call_frame_t *frame, xlator_t *this,
-             void *data)
-{
-        int                       ret = 0;
-        int32_t                   command = 0;
-        int32_t                   cmd = 0;
-        gf_cli_req                req =  { {0,} };
-        gf_cli_req                status_req = { {0,} };
-        dict_t                    *dict = NULL;
-        char                     *volname = NULL;
-
-        if (!frame || !this ||  !data) {
-                ret = -1;
-                goto out;
+            gf_log(this->name, GF_LOG_ERROR, "Failed to set dict");
+            goto out;
         }
-        dict = data;
 
-        ret = cli_to_glusterd (&req, frame, gf_cli_defrag_volume_cbk,
-                               (xdrproc_t) xdr_gf_cli_req, dict,
-                               GLUSTER_CLI_DEFRAG_VOLUME, this, cli_rpc_prog,
-                               NULL);
+        ret = cli_to_glusterd(
+            &status_req, frame, gf_cli3_remove_brick_status_cbk,
+            (xdrproc_t)xdr_gf_cli_req, dict, GLUSTER_CLI_DEFRAG_VOLUME, this,
+            cli_rpc_prog, NULL);
+    }
 
 out:
-        gf_log ("cli", GF_LOG_DEBUG, "Returning %d", ret);
+    gf_log("cli", GF_LOG_DEBUG, RETURNING, ret);
 
-        GF_FREE (req.dict.dict_val);
+    GF_FREE(req.dict.dict_val);
 
-        GF_FREE (status_req.dict.dict_val);
+    GF_FREE(status_req.dict.dict_val);
 
-        return ret;
+    return ret;
 }
 
-int32_t
-gf_cli_detach_tier (call_frame_t *frame, xlator_t *this,
-                    void *data)
-{
-        gf_cli_req                req =  { {0,} };
-        gf_cli_req                status_req = { {0,} };
-        int                       ret = 0;
-        dict_t                   *dict = NULL;
-        int32_t                   command = 0;
-        char                     *volname = NULL;
-        int32_t                   cmd = 0;
-
-        if (!frame || !this ||  !data) {
-                ret = -1;
-                goto out;
+static int32_t
+gf_cli_reset_brick(call_frame_t *frame, xlator_t *this, void *data)
+{
+    gf_cli_req req = {{
+        0,
+    }};
+    int ret = 0;
+    dict_t *dict = NULL;
+    char *dst_brick = NULL;
+    char *op = NULL;
+
+    if (!frame || !this || !data) {
+        ret = -1;
+        goto out;
+    }
+
+    dict = data;
+
+    ret = dict_get_str_sizen(dict, "operation", &op);
+    if (ret) {
+        gf_log(this->name, GF_LOG_DEBUG, "dict_get on operation failed");
+        goto out;
+    }
+
+    if (!strcmp(op, "GF_RESET_OP_COMMIT") ||
+        !strcmp(op, "GF_RESET_OP_COMMIT_FORCE")) {
+        ret = dict_get_str_sizen(dict, "dst-brick", &dst_brick);
+        if (ret) {
+            gf_log(this->name, GF_LOG_DEBUG, "dict_get on dst-brick failed");
+            goto out;
         }
+    }
 
-        dict = data;
-
-        ret = dict_get_str (dict, "volname", &volname);
-        if (ret)
-                goto out;
-
-        ret = dict_get_int32 (dict, "command", &command);
-        if (ret)
-                goto out;
-
-        if ((command != GF_OP_CMD_STATUS) &&
-            (command != GF_OP_CMD_STOP_DETACH_TIER)) {
-
-
-                ret = cli_to_glusterd (&req, frame, gf_cli_detach_tier_cbk,
-                                       (xdrproc_t) xdr_gf_cli_req, dict,
-                                       GLUSTER_CLI_REMOVE_BRICK, this,
-                                       cli_rpc_prog, NULL);
-        } else {
-                /* Need rebalance status to be sent :-) */
-                if (command == GF_OP_CMD_STATUS)
-                        cmd |= GF_DEFRAG_CMD_DETACH_STATUS;
-                else
-                        cmd |= GF_DEFRAG_CMD_STOP_DETACH_TIER;
-
-                ret = dict_set_int32 (dict, "rebalance-command", (int32_t) cmd);
-                if (ret) {
-                        gf_log (this->name, GF_LOG_ERROR,
-                                "Failed to set dict");
-                        goto out;
-                }
-
-                ret = cli_to_glusterd (&status_req, frame,
-                                       gf_cli_detach_tier_status_cbk,
-                                       (xdrproc_t) xdr_gf_cli_req, dict,
-                                       GLUSTER_CLI_DEFRAG_VOLUME, this,
-                                       cli_rpc_prog, NULL);
-
-                }
+    ret = cli_to_glusterd(&req, frame, gf_cli_reset_brick_cbk,
+                          (xdrproc_t)xdr_gf_cli_req, dict,
+                          GLUSTER_CLI_RESET_BRICK, this, cli_rpc_prog, NULL);
 
 out:
-        gf_log ("cli", GF_LOG_DEBUG, "Returning %d", ret);
+    gf_log("cli", GF_LOG_DEBUG, RETURNING, ret);
 
-        GF_FREE (req.dict.dict_val);
+    GF_FREE(req.dict.dict_val);
 
-        GF_FREE (status_req.dict.dict_val);
-
-        return ret;
+    return ret;
 }
 
+static int32_t
+gf_cli_replace_brick(call_frame_t *frame, xlator_t *this, void *data)
+{
+    gf_cli_req req = {{
+        0,
+    }};
+    int ret = 0;
+    dict_t *dict = NULL;
+    int32_t op = 0;
 
+    if (!frame || !this || !data) {
+        ret = -1;
+        goto out;
+    }
 
-int32_t
-gf_cli_remove_brick (call_frame_t *frame, xlator_t *this,
-                     void *data)
-{
-        gf_cli_req                req =  {{0,}};;
-        gf_cli_req                status_req =  {{0,}};;
-        int                       ret = 0;
-        dict_t                   *dict = NULL;
-        int32_t                   command = 0;
-        char                     *volname = NULL;
-        int32_t                   cmd = 0;
-
-        if (!frame || !this ||  !data) {
-                ret = -1;
-                goto out;
-        }
-
-        dict = data;
-
-        ret = dict_get_str (dict, "volname", &volname);
-        if (ret)
-                goto out;
-
-        ret = dict_get_int32 (dict, "command", &command);
-        if (ret)
-                goto out;
-
-        if ((command != GF_OP_CMD_STATUS) &&
-            (command != GF_OP_CMD_STOP)) {
-
-
-                ret = cli_to_glusterd (&req, frame, gf_cli_remove_brick_cbk,
-                                       (xdrproc_t) xdr_gf_cli_req, dict,
-                                       GLUSTER_CLI_REMOVE_BRICK, this,
-                                       cli_rpc_prog, NULL);
-        } else {
-                /* Need rebalance status to be sent :-) */
-                if (command == GF_OP_CMD_STATUS)
-                        cmd |= GF_DEFRAG_CMD_STATUS;
-                else
-                        cmd |= GF_DEFRAG_CMD_STOP;
-
-                ret = dict_set_int32 (dict, "rebalance-command", (int32_t) cmd);
-                if (ret) {
-                        gf_log (this->name, GF_LOG_ERROR,
-                                "Failed to set dict");
-                        goto out;
-                }
-
-                ret = cli_to_glusterd (&status_req, frame,
-                                       gf_cli3_remove_brick_status_cbk,
-                                       (xdrproc_t) xdr_gf_cli_req, dict,
-                                       GLUSTER_CLI_DEFRAG_VOLUME, this,
-                                       cli_rpc_prog, NULL);
-
-                }
-
-out:
-        gf_log ("cli", GF_LOG_DEBUG, "Returning %d", ret);
-
-        GF_FREE (req.dict.dict_val);
-
-        GF_FREE (status_req.dict.dict_val);
-
-        return ret;
-}
-
-int32_t
-gf_cli_replace_brick (call_frame_t *frame, xlator_t *this,
-                         void *data)
-{
-        gf_cli_req                  req        =  {{0,}};
-        int                         ret        = 0;
-        dict_t                     *dict       = NULL;
-        char                       *src_brick  = NULL;
-        char                       *dst_brick  = NULL;
-        char                       *volname    = NULL;
-        int32_t                     op         = 0;
-
-        if (!frame || !this ||  !data) {
-                ret = -1;
-                goto out;
-        }
-
-        dict = data;
-
-        ret = dict_get_int32 (dict, "operation", &op);
-        if (ret) {
-                gf_log (this->name, GF_LOG_DEBUG,
-                        "dict_get on operation failed");
-                goto out;
-        }
-        ret = dict_get_str (dict, "volname", &volname);
-        if (ret) {
-                gf_log (this->name, GF_LOG_DEBUG,
-                        "dict_get on volname failed");
-                goto out;
-        }
-
-        ret = dict_get_str (dict, "src-brick", &src_brick);
-        if (ret) {
-                gf_log (this->name, GF_LOG_DEBUG,
-                        "dict_get on src-brick failed");
-                goto out;
-        }
-
-        ret = dict_get_str (dict, "dst-brick", &dst_brick);
-        if (ret) {
-                gf_log (this->name, GF_LOG_DEBUG,
-                        "dict_get on dst-brick failed");
-                goto out;
-        }
+    dict = data;
 
-        gf_log (this->name, GF_LOG_DEBUG,
-                "Received command replace-brick %s with "
-                "%s with operation=%d", src_brick,
-                dst_brick, op);
+    ret = dict_get_int32_sizen(dict, "operation", &op);
+    if (ret) {
+        gf_log(this->name, GF_LOG_DEBUG, "dict_get on operation failed");
+        goto out;
+    }
 
-        ret = cli_to_glusterd (&req, frame, gf_cli_replace_brick_cbk,
-                               (xdrproc_t) xdr_gf_cli_req, dict,
-                               GLUSTER_CLI_REPLACE_BRICK, this, cli_rpc_prog,
-                               NULL);
+    ret = cli_to_glusterd(&req, frame, gf_cli_replace_brick_cbk,
+                          (xdrproc_t)xdr_gf_cli_req, dict,
+                          GLUSTER_CLI_REPLACE_BRICK, this, cli_rpc_prog, NULL);
 
 out:
-        gf_log ("cli", GF_LOG_DEBUG, "Returning %d", ret);
+    gf_log("cli", GF_LOG_DEBUG, RETURNING, ret);
 
-        GF_FREE (req.dict.dict_val);
+    GF_FREE(req.dict.dict_val);
 
-        return ret;
+    return ret;
 }
 
-
-int32_t
-gf_cli_log_rotate (call_frame_t *frame, xlator_t *this,
-                      void *data)
+static int32_t
+gf_cli_log_rotate(call_frame_t *frame, xlator_t *this, void *data)
 {
-        gf_cli_req                req = {{0,}};
-        int                       ret = 0;
-        dict_t                   *dict = NULL;
+    gf_cli_req req = {{
+        0,
+    }};
+    int ret = 0;
+    dict_t *dict = NULL;
 
-        if (!frame || !this ||  !data) {
-                ret = -1;
-                goto out;
-        }
+    dict = data;
 
-        dict = data;
-
-        ret = cli_to_glusterd (&req, frame, gf_cli_log_rotate_cbk,
-                               (xdrproc_t) xdr_gf_cli_req, dict,
-                               GLUSTER_CLI_LOG_ROTATE, this, cli_rpc_prog,
-                               NULL);
-
-out:
-        gf_log ("cli", GF_LOG_DEBUG, "Returning %d", ret);
+    ret = cli_to_glusterd(&req, frame, gf_cli_log_rotate_cbk,
+                          (xdrproc_t)xdr_gf_cli_req, dict,
+                          GLUSTER_CLI_LOG_ROTATE, this, cli_rpc_prog, NULL);
+    gf_log("cli", GF_LOG_DEBUG, RETURNING, ret);
 
-        GF_FREE (req.dict.dict_val);
-        return ret;
+    GF_FREE(req.dict.dict_val);
+    return ret;
 }
 
-int32_t
-gf_cli_sync_volume (call_frame_t *frame, xlator_t *this,
-                       void *data)
+static int32_t
+gf_cli_sync_volume(call_frame_t *frame, xlator_t *this, void *data)
 {
-        int               ret = 0;
-        gf_cli_req        req = {{0,}};
-        dict_t            *dict = NULL;
+    int ret = 0;
+    gf_cli_req req = {{
+        0,
+    }};
+    dict_t *dict = NULL;
 
-        if (!frame || !this || !data) {
-                ret = -1;
-                goto out;
-        }
+    dict = data;
 
-        dict = data;
+    ret = cli_to_glusterd(&req, frame, gf_cli_sync_volume_cbk,
+                          (xdrproc_t)xdr_gf_cli_req, dict,
+                          GLUSTER_CLI_SYNC_VOLUME, this, cli_rpc_prog, NULL);
+    gf_log("cli", GF_LOG_DEBUG, RETURNING, ret);
+    GF_FREE(req.dict.dict_val);
 
-        ret = cli_to_glusterd (&req, frame, gf_cli_sync_volume_cbk,
-                               (xdrproc_t) xdr_gf_cli_req, dict,
-                               GLUSTER_CLI_SYNC_VOLUME, this, cli_rpc_prog,
-                               NULL);
-
-out:
-        gf_log ("cli", GF_LOG_DEBUG, "Returning %d", ret);
-        GF_FREE (req.dict.dict_val);
-
-        return ret;
+    return ret;
 }
 
-int32_t
-gf_cli_getspec (call_frame_t *frame, xlator_t *this,
-                         void *data)
-{
-        gf_getspec_req          req = {0,};
-        int                     ret = 0;
-        dict_t                  *dict = NULL;
-        dict_t                  *op_dict = NULL;
-
-        if (!frame || !this ||  !data) {
-                ret = -1;
-                goto out;
-        }
-
-        dict = data;
-
-        ret = dict_get_str (dict, "volid", &req.key);
-        if (ret)
-                goto out;
-
-        op_dict = dict_new ();
-        if (!op_dict) {
-                ret = -1;
-                goto out;
-        }
-
-        // Set the supported min and max op-versions, so glusterd can make a
-        // decision
-        ret = dict_set_int32 (op_dict, "min-op-version", GD_OP_VERSION_MIN);
-        if (ret) {
-                gf_log (THIS->name, GF_LOG_ERROR, "Failed to set min-op-version"
-                        " in request dict");
-                goto out;
-        }
-
-        ret = dict_set_int32 (op_dict, "max-op-version", GD_OP_VERSION_MAX);
-        if (ret) {
-                gf_log (THIS->name, GF_LOG_ERROR, "Failed to set max-op-version"
-                        " in request dict");
-                goto out;
-        }
-
-        ret = dict_allocate_and_serialize (op_dict, &req.xdata.xdata_val,
-                                           &req.xdata.xdata_len);
-        if (ret < 0) {
-                gf_log (THIS->name, GF_LOG_ERROR,
-                        "Failed to serialize dictionary");
-                goto out;
-        }
-
-        ret = cli_cmd_submit (NULL, &req, frame, &cli_handshake_prog,
-                              GF_HNDSK_GETSPEC, NULL,
-                              this, gf_cli_getspec_cbk,
-                              (xdrproc_t) xdr_gf_getspec_req);
-
-out:
-        if (op_dict) {
-                dict_unref(op_dict);
-        }
-        gf_log ("cli", GF_LOG_DEBUG, "Returning %d", ret);
-
-        return ret;
+static int32_t
+gf_cli_getspec(call_frame_t *frame, xlator_t *this, void *data)
+{
+    gf_getspec_req req = {
+        0,
+    };
+    int ret = 0;
+    dict_t *dict = NULL;
+    dict_t *op_dict = NULL;
+
+    if (!frame || !this) {
+        ret = -1;
+        goto out;
+    }
+
+    dict = data;
+
+    ret = dict_get_str_sizen(dict, "volid", &req.key);
+    if (ret)
+        goto out;
+
+    op_dict = dict_new();
+    if (!op_dict) {
+        ret = -1;
+        goto out;
+    }
+
+    // Set the supported min and max op-versions, so glusterd can make a
+    // decision
+    ret = dict_set_int32_sizen(op_dict, "min-op-version", GD_OP_VERSION_MIN);
+    if (ret) {
+        gf_log(THIS->name, GF_LOG_ERROR,
+               "Failed to set min-op-version in request dict");
+        goto out;
+    }
+
+    ret = dict_set_int32_sizen(op_dict, "max-op-version", GD_OP_VERSION_MAX);
+    if (ret) {
+        gf_log(THIS->name, GF_LOG_ERROR,
+               "Failed to set max-op-version in request dict");
+        goto out;
+    }
+
+    ret = dict_allocate_and_serialize(op_dict, &req.xdata.xdata_val,
+                                      &req.xdata.xdata_len);
+    if (ret < 0) {
+        gf_log(THIS->name, GF_LOG_ERROR, DICT_SERIALIZE_FAIL);
+        goto out;
+    }
+
+    ret = cli_cmd_submit(NULL, &req, frame, &cli_handshake_prog,
+                         GF_HNDSK_GETSPEC, NULL, this, gf_cli_getspec_cbk,
+                         (xdrproc_t)xdr_gf_getspec_req);
+
+out:
+    if (op_dict) {
+        dict_unref(op_dict);
+    }
+    GF_FREE(req.xdata.xdata_val);
+    gf_log("cli", GF_LOG_DEBUG, RETURNING, ret);
+
+    return ret;
 }
 
-int32_t
-gf_cli_quota (call_frame_t *frame, xlator_t *this,
-                 void *data)
+static int32_t
+gf_cli_quota(call_frame_t *frame, xlator_t *this, void *data)
 {
-        gf_cli_req          req = {{0,}};
-        int                 ret = 0;
-        dict_t             *dict = NULL;
-
-        if (!frame || !this ||  !data) {
-                ret = -1;
-                goto out;
-        }
+    gf_cli_req req = {{
+        0,
+    }};
+    int ret = 0;
+    dict_t *dict = NULL;
 
-        dict = data;
+    dict = data;
 
-        ret = cli_to_glusterd (&req, frame, gf_cli_quota_cbk,
-                               (xdrproc_t) xdr_gf_cli_req, dict,
-                               GLUSTER_CLI_QUOTA, this, cli_rpc_prog, NULL);
-
-out:
-        GF_FREE (req.dict.dict_val);
-
-        return ret;
+    ret = cli_to_glusterd(&req, frame, gf_cli_quota_cbk,
+                          (xdrproc_t)xdr_gf_cli_req, dict, GLUSTER_CLI_QUOTA,
+                          this, cli_rpc_prog, NULL);
+    GF_FREE(req.dict.dict_val);
+    gf_log("cli", GF_LOG_DEBUG, RETURNING, ret);
+    return ret;
 }
 
-int32_t
-gf_cli_pmap_b2p (call_frame_t *frame, xlator_t *this, void *data)
+static int32_t
+gf_cli_pmap_b2p(call_frame_t *frame, xlator_t *this, void *data)
 {
-        pmap_port_by_brick_req  req = {0,};
-        int                     ret = 0;
-        dict_t                  *dict = NULL;
+    pmap_port_by_brick_req req = {
+        0,
+    };
+    int ret = 0;
+    dict_t *dict = NULL;
 
-        if (!frame || !this ||  !data) {
-                ret = -1;
-                goto out;
-        }
+    if (!frame || !this) {
+        ret = -1;
+        goto out;
+    }
 
-        dict = data;
+    dict = data;
 
-        ret = dict_get_str (dict, "brick", &req.brick);
-        if (ret)
-                goto out;
+    ret = dict_get_str_sizen(dict, "brick", &req.brick);
+    if (ret)
+        goto out;
 
-        ret = cli_cmd_submit (NULL, &req, frame, &cli_pmap_prog,
-                              GF_PMAP_PORTBYBRICK, NULL,
-                              this, gf_cli_pmap_b2p_cbk,
-                              (xdrproc_t) xdr_pmap_port_by_brick_req);
+    ret = cli_cmd_submit(NULL, &req, frame, &cli_pmap_prog, GF_PMAP_PORTBYBRICK,
+                         NULL, this, gf_cli_pmap_b2p_cbk,
+                         (xdrproc_t)xdr_pmap_port_by_brick_req);
 
 out:
-        gf_log ("cli", GF_LOG_DEBUG, "Returning %d", ret);
+    gf_log("cli", GF_LOG_DEBUG, RETURNING, ret);
 
-        return ret;
+    return ret;
 }
 
 static int
-gf_cli_fsm_log_cbk (struct rpc_req *req, struct iovec *iov,
-                       int count, void *myframe)
-{
-        gf1_cli_fsm_log_rsp        rsp   = {0,};
-        int                        ret   = -1;
-        dict_t                     *dict = NULL;
-        int                        tr_count = 0;
-        char                       key[256] = {0};
-        int                        i = 0;
-        char                       *old_state = NULL;
-        char                       *new_state = NULL;
-        char                       *event = NULL;
-        char                       *time = NULL;
-
-        if (-1 == req->rpc_status) {
-                goto out;
-        }
-
-        ret = xdr_to_generic (*iov, &rsp, (xdrproc_t)xdr_gf1_cli_fsm_log_rsp);
-        if (ret < 0) {
-                gf_log (((call_frame_t *) myframe)->this->name, GF_LOG_ERROR,
-                        "Failed to decode xdr response");
-                goto out;
-        }
-
-        if (rsp.op_ret) {
-                if (strcmp (rsp.op_errstr, ""))
-                        cli_err ("%s", rsp.op_errstr);
-                cli_err ("fsm log unsuccessful");
-                ret = rsp.op_ret;
-                goto out;
-        }
-
-        dict = dict_new ();
-        if (!dict) {
-                ret = -1;
-                goto out;
-        }
-
-        ret = dict_unserialize (rsp.fsm_log.fsm_log_val,
-                                rsp.fsm_log.fsm_log_len,
-                                &dict);
-
-        if (ret) {
-                cli_err ("bad response");
-                goto out;
-        }
-
-        ret = dict_get_int32 (dict, "count", &tr_count);
-        if (tr_count)
-                cli_out("number of transitions: %d", tr_count);
-        else
-                cli_err("No transitions");
-        for (i = 0; i < tr_count; i++) {
-                memset (key, 0, sizeof (key));
-                snprintf (key, sizeof (key), "log%d-old-state", i);
-                ret = dict_get_str (dict, key, &old_state);
-                if (ret)
-                        goto out;
+gf_cli_fsm_log_cbk(struct rpc_req *req, struct iovec *iov, int count,
+                   void *myframe)
+{
+    gf1_cli_fsm_log_rsp rsp = {
+        0,
+    };
+    int ret = -1;
+    dict_t *dict = NULL;
+    int tr_count = 0;
+    char key[64] = {0};
+    int keylen;
+    int i = 0;
+    char *old_state = NULL;
+    char *new_state = NULL;
+    char *event = NULL;
+    char *time = NULL;
+
+    GF_ASSERT(myframe);
+
+    if (-1 == req->rpc_status) {
+        goto out;
+    }
+
+    ret = xdr_to_generic(*iov, &rsp, (xdrproc_t)xdr_gf1_cli_fsm_log_rsp);
+    if (ret < 0) {
+        gf_log(((call_frame_t *)myframe)->this->name, GF_LOG_ERROR,
+               XDR_DECODE_FAIL);
+        goto out;
+    }
+
+    if (rsp.op_ret) {
+        if (strcmp(rsp.op_errstr, ""))
+            cli_err("%s", rsp.op_errstr);
+        cli_err("fsm log unsuccessful");
+        ret = rsp.op_ret;
+        goto out;
+    }
+
+    dict = dict_new();
+    if (!dict) {
+        ret = -1;
+        goto out;
+    }
+
+    ret = dict_unserialize(rsp.fsm_log.fsm_log_val, rsp.fsm_log.fsm_log_len,
+                           &dict);
+
+    if (ret) {
+        cli_err(DICT_UNSERIALIZE_FAIL);
+        goto out;
+    }
+
+    ret = dict_get_int32_sizen(dict, "count", &tr_count);
+    if (!ret && tr_count)
+        cli_out("number of transitions: %d", tr_count);
+    else
+        cli_err("No transitions");
+    for (i = 0; i < tr_count; i++) {
+        keylen = snprintf(key, sizeof(key), "log%d-old-state", i);
+        ret = dict_get_strn(dict, key, keylen, &old_state);
+        if (ret)
+            goto out;
 
-                memset (key, 0, sizeof (key));
-                snprintf (key, sizeof (key), "log%d-event", i);
-                ret = dict_get_str (dict, key, &event);
-                if (ret)
-                        goto out;
+        keylen = snprintf(key, sizeof(key), "log%d-event", i);
+        ret = dict_get_strn(dict, key, keylen, &event);
+        if (ret)
+            goto out;
 
-                memset (key, 0, sizeof (key));
-                snprintf (key, sizeof (key), "log%d-new-state", i);
-                ret = dict_get_str (dict, key, &new_state);
-                if (ret)
-                        goto out;
+        keylen = snprintf(key, sizeof(key), "log%d-new-state", i);
+        ret = dict_get_strn(dict, key, keylen, &new_state);
+        if (ret)
+            goto out;
 
-                memset (key, 0, sizeof (key));
-                snprintf (key, sizeof (key), "log%d-time", i);
-                ret = dict_get_str (dict, key, &time);
-                if (ret)
-                        goto out;
-                cli_out ("Old State: [%s]\n"
-                         "New State: [%s]\n"
-                         "Event    : [%s]\n"
-                         "timestamp: [%s]\n", old_state, new_state, event, time);
-        }
+        keylen = snprintf(key, sizeof(key), "log%d-time", i);
+        ret = dict_get_strn(dict, key, keylen, &time);
+        if (ret)
+            goto out;
+        cli_out(
+            "Old State: [%s]\n"
+            "New State: [%s]\n"
+            "Event    : [%s]\n"
+            "timestamp: [%s]\n",
+            old_state, new_state, event, time);
+    }
 
-        ret = rsp.op_ret;
+    ret = rsp.op_ret;
 
 out:
-        cli_cmd_broadcast_response (ret);
-        return ret;
+    cli_cmd_broadcast_response(ret);
+    if (dict) {
+        dict_unref(dict);
+    }
+    gf_free_xdr_fsm_log_rsp(rsp);
+
+    return ret;
 }
 
-int32_t
-gf_cli_fsm_log (call_frame_t *frame, xlator_t *this, void *data)
+static int32_t
+gf_cli_fsm_log(call_frame_t *frame, xlator_t *this, void *data)
 {
-        int                        ret = -1;
-        gf1_cli_fsm_log_req        req = {0,};
+    int ret = -1;
+    gf1_cli_fsm_log_req req = {
+        0,
+    };
 
-        GF_ASSERT (frame);
-        GF_ASSERT (this);
-        GF_ASSERT (data);
+    GF_ASSERT(frame);
+    GF_ASSERT(this);
+    GF_ASSERT(data);
 
-        if (!frame || !this || !data)
-                goto out;
-        req.name = data;
-        ret = cli_cmd_submit (NULL, &req, frame, cli_rpc_prog,
-                              GLUSTER_CLI_FSM_LOG, NULL,
-                              this, gf_cli_fsm_log_cbk,
-                              (xdrproc_t) xdr_gf1_cli_fsm_log_req);
+    if (!frame || !this || !data)
+        goto out;
+    req.name = data;
+    ret = cli_cmd_submit(NULL, &req, frame, cli_rpc_prog, GLUSTER_CLI_FSM_LOG,
+                         NULL, this, gf_cli_fsm_log_cbk,
+                         (xdrproc_t)xdr_gf1_cli_fsm_log_req);
 
 out:
-        gf_log ("cli", GF_LOG_DEBUG, "Returning %d", ret);
+    gf_log("cli", GF_LOG_DEBUG, RETURNING, ret);
 
-        return ret;
+    return ret;
 }
 
-int
-gf_cli_gsync_config_command (dict_t *dict)
-{
-        runner_t runner     = {0,};
-        char *subop         = NULL;
-        char *gwd           = NULL;
-        char *slave         = NULL;
-        char *confpath      = NULL;
-        char *master        = NULL;
-        char *op_name       = NULL;
-        int   ret           = -1;
-        char  conf_path[PATH_MAX] = "";
-
-        if (dict_get_str (dict, "subop", &subop) != 0)
-                return -1;
-
-        if (strcmp (subop, "get") != 0 && strcmp (subop, "get-all") != 0) {
-                cli_out (GEOREP" config updated successfully");
-                return 0;
-        }
-
-        if (dict_get_str (dict, "glusterd_workdir", &gwd) != 0 ||
-            dict_get_str (dict, "slave", &slave) != 0)
-                return -1;
-
-        if (dict_get_str (dict, "master", &master) != 0)
-                master = NULL;
-        if (dict_get_str (dict, "op_name", &op_name) != 0)
-                op_name = NULL;
-
-        ret = dict_get_str (dict, "conf_path", &confpath);
-        if (!confpath) {
-                ret = snprintf (conf_path, sizeof(conf_path) - 1,
-                                "%s/"GEOREP"/gsyncd_template.conf", gwd);
-                conf_path[ret] = '\0';
-                confpath = conf_path;
-        }
-
-        runinit (&runner);
-        runner_add_args (&runner, GSYNCD_PREFIX"/gsyncd", "-c", NULL);
-        runner_argprintf (&runner, "%s", confpath);
-        runner_argprintf (&runner, "--iprefix=%s", DATADIR);
-        if (master)
-                runner_argprintf (&runner, ":%s", master);
-        runner_add_arg (&runner, slave);
-        runner_argprintf (&runner, "--config-%s", subop);
-        if (op_name)
-                runner_add_arg (&runner, op_name);
-
-        return runner_run (&runner);
-}
-
-char*
-get_struct_variable (int mem_num, gf_gsync_status_t *sts_val)
-{
-        switch (mem_num) {
-        case 0:  return (sts_val->node);
-        case 1:  return (sts_val->master);
-        case 2:  return (sts_val->brick);
-        case 3:  return (sts_val->slave_user);
-        case 4:  return (sts_val->slave);
-        case 5:  return (sts_val->slave_node);
-        case 6:  return (sts_val->worker_status);
-        case 7:  return (sts_val->crawl_status);
-        case 8:  return (sts_val->last_synced);
-        case 9:  return (sts_val->entry);
-        case 10:  return (sts_val->data);
-        case 11:  return (sts_val->meta);
-        case 12: return (sts_val->failures);
-        case 13:  return (sts_val->checkpoint_time);
-        case 14:  return (sts_val->checkpoint_completed);
-        case 15:  return (sts_val->checkpoint_completion_time);
-        case 16: return (sts_val->brick_host_uuid);
-        case 17: return (sts_val->last_synced_utc);
-        case 18: return (sts_val->checkpoint_time_utc);
-        case 19: return (sts_val->checkpoint_completion_time_utc);
-        case 20: return (sts_val->slavekey);
-        case 21: return (sts_val->session_slave);
-        default:
-                 goto out;
-        }
-
-out:
-        return NULL;
+static int
+gf_cli_gsync_config_command(dict_t *dict)
+{
+    runner_t runner = {
+        0,
+    };
+    char *subop = NULL;
+    char *gwd = NULL;
+    char *slave = NULL;
+    char *confpath = NULL;
+    char *master = NULL;
+    char *op_name = NULL;
+    int ret = -1;
+    char conf_path[PATH_MAX] = "";
+
+    if (dict_get_str_sizen(dict, "subop", &subop) != 0)
+        return -1;
+
+    if (strcmp(subop, "get") != 0 && strcmp(subop, "get-all") != 0) {
+        cli_out(GEOREP " config updated successfully");
+        return 0;
+    }
+
+    if (dict_get_str_sizen(dict, "glusterd_workdir", &gwd) != 0 ||
+        dict_get_str_sizen(dict, "slave", &slave) != 0)
+        return -1;
+
+    if (dict_get_str_sizen(dict, "master", &master) != 0)
+        master = NULL;
+    if (dict_get_str_sizen(dict, "op_name", &op_name) != 0)
+        op_name = NULL;
+
+    ret = dict_get_str_sizen(dict, "conf_path", &confpath);
+    if (ret || !confpath) {
+        ret = snprintf(conf_path, sizeof(conf_path) - 1,
+                       "%s/" GEOREP "/gsyncd_template.conf", gwd);
+        conf_path[ret] = '\0';
+        confpath = conf_path;
+    }
+
+    runinit(&runner);
+    runner_add_args(&runner, GSYNCD_PREFIX "/gsyncd", "-c", NULL);
+    runner_argprintf(&runner, "%s", confpath);
+    runner_argprintf(&runner, "--iprefix=%s", DATADIR);
+    if (master)
+        runner_argprintf(&runner, ":%s", master);
+    runner_add_arg(&runner, slave);
+    runner_argprintf(&runner, "--config-%s", subop);
+    if (op_name)
+        runner_add_arg(&runner, op_name);
+
+    return runner_run(&runner);
 }
 
-int
-gf_cli_print_status (char **title_values,
-                     gf_gsync_status_t **sts_vals,
-                     int *spacing, int gsync_count,
-                     int number_of_fields, int is_detail)
-{
-        int     i                        = 0;
-        int     j                        = 0;
-        int     ret                      = 0;
-        int     status_fields            = 8; /* Indexed at 0 */
-        int     total_spacing            = 0;
-        char  **output_values            = NULL;
-        char   *tmp                      = NULL;
-        char   *hyphens                  = NULL;
-
-        /* calculating spacing for hyphens */
-        for (i = 0; i < number_of_fields; i++) {
-                /* Suppressing detail output for status */
-                if ((!is_detail) && (i > status_fields)) {
-                       /* Suppressing detailed output for
-                        * status */
-                        continue;
-                }
-                spacing[i] += 3; /* Adding extra space to
-                                    distinguish between fields */
-                total_spacing += spacing[i];
-        }
-        total_spacing += 4; /* For the spacing between the fields */
-
-        /* char pointers for each field */
-        output_values = GF_CALLOC (number_of_fields, sizeof (char *),
-                                   gf_common_mt_char);
-        if (!output_values) {
-                ret = -1;
-                goto out;
-        }
-        for (i = 0; i < number_of_fields; i++) {
-                output_values[i] = GF_CALLOC (spacing[i] + 1, sizeof (char),
-                                              gf_common_mt_char);
-                if (!output_values[i]) {
-                        ret = -1;
-                        goto out;
-                }
-        }
-
-        hyphens = GF_CALLOC (total_spacing + 1, sizeof (char),
-                             gf_common_mt_char);
-        if (!hyphens) {
+static int
+gf_cli_print_status(char **title_values, gf_gsync_status_t **sts_vals,
+                    int *spacing, int gsync_count, int number_of_fields,
+                    int is_detail)
+{
+    int i = 0;
+    int j = 0;
+    int ret = 0;
+    int status_fields = 8; /* Indexed at 0 */
+    int total_spacing = 0;
+    char **output_values = NULL;
+    char *tmp = NULL;
+    char *hyphens = NULL;
+
+    /* calculating spacing for hyphens */
+    for (i = 0; i < number_of_fields; i++) {
+        /* Suppressing detail output for status */
+        if ((!is_detail) && (i > status_fields)) {
+            /* Suppressing detailed output for
+             * status */
+            continue;
+        }
+        spacing[i] += 3; /* Adding extra space to
+                            distinguish between fields */
+        total_spacing += spacing[i];
+    }
+    total_spacing += 4; /* For the spacing between the fields */
+
+    /* char pointers for each field */
+    output_values = GF_MALLOC(number_of_fields * sizeof(char *),
+                              gf_common_mt_char);
+    if (!output_values) {
+        ret = -1;
+        goto out;
+    }
+    for (i = 0; i < number_of_fields; i++) {
+        output_values[i] = GF_CALLOC(spacing[i] + 1, sizeof(char),
+                                     gf_common_mt_char);
+        if (!output_values[i]) {
+            ret = -1;
+            goto out;
+        }
+    }
+
+    cli_out(" ");
+
+    /* setting the title "NODE", "MASTER", etc. from title_values[]
+       and printing the same */
+    for (j = 0; j < number_of_fields; j++) {
+        if ((!is_detail) && (j > status_fields)) {
+            /* Suppressing detailed output for
+             * status */
+            output_values[j][0] = '\0';
+            continue;
+        }
+        memset(output_values[j], ' ', spacing[j]);
+        memcpy(output_values[j], title_values[j], strlen(title_values[j]));
+        output_values[j][spacing[j]] = '\0';
+    }
+    cli_out("%s %s %s %s %s %s %s %s %s %s %s %s %s %s %s %s", output_values[0],
+            output_values[1], output_values[2], output_values[3],
+            output_values[4], output_values[5], output_values[6],
+            output_values[7], output_values[8], output_values[9],
+            output_values[10], output_values[11], output_values[12],
+            output_values[13], output_values[14], output_values[15]);
+
+    hyphens = GF_MALLOC((total_spacing + 1) * sizeof(char), gf_common_mt_char);
+    if (!hyphens) {
+        ret = -1;
+        goto out;
+    }
+
+    /* setting and printing the hyphens */
+    memset(hyphens, '-', total_spacing);
+    hyphens[total_spacing] = '\0';
+    cli_out("%s", hyphens);
+
+    for (i = 0; i < gsync_count; i++) {
+        for (j = 0; j < number_of_fields; j++) {
+            if ((!is_detail) && (j > status_fields)) {
+                /* Suppressing detailed output for
+                 * status */
+                output_values[j][0] = '\0';
+                continue;
+            }
+            tmp = get_struct_variable(j, sts_vals[i]);
+            if (!tmp) {
+                gf_log("", GF_LOG_ERROR, "struct member empty.");
                 ret = -1;
                 goto out;
+            }
+            memset(output_values[j], ' ', spacing[j]);
+            memcpy(output_values[j], tmp, strlen(tmp));
+            output_values[j][spacing[j]] = '\0';
         }
 
-        cli_out (" ");
-
-        /* setting the title "NODE", "MASTER", etc. from title_values[]
-           and printing the same */
-        for (j = 0; j < number_of_fields; j++) {
-                if ((!is_detail) && (j > status_fields)) {
-                       /* Suppressing detailed output for
-                        * status */
-                       output_values[j][0] = '\0';
-                       continue;
-                }
-                memset (output_values[j], ' ', spacing[j]);
-                memcpy (output_values[j], title_values[j],
-                        strlen(title_values[j]));
-                output_values[j][spacing[j]] = '\0';
-        }
-        cli_out ("%s %s %s %s %s %s %s %s %s %s %s %s %s %s %s %s",
-                 output_values[0], output_values[1],
-                 output_values[2], output_values[3],
-                 output_values[4], output_values[5],
-                 output_values[6], output_values[7],
-                 output_values[8], output_values[9],
-                 output_values[10], output_values[11],
-                 output_values[12], output_values[13],
-                 output_values[14], output_values[15]);
-
-        /* setting and printing the hyphens */
-        memset (hyphens, '-', total_spacing);
-        hyphens[total_spacing] = '\0';
-        cli_out ("%s", hyphens);
-
-        for (i = 0; i < gsync_count; i++) {
-                for (j = 0; j < number_of_fields; j++) {
-                        if ((!is_detail) && (j > status_fields)) {
-                                /* Suppressing detailed output for
-                                 * status */
-                                output_values[j][0] = '\0';
-                                continue;
-                        }
-                        tmp = get_struct_variable(j, sts_vals[i]);
-                        if (!tmp) {
-                                gf_log ("", GF_LOG_ERROR,
-                                        "struct member empty.");
-                                ret = -1;
-                                goto out;
-                        }
-                        memset (output_values[j], ' ', spacing[j]);
-                        memcpy (output_values[j], tmp, strlen (tmp));
-                        output_values[j][spacing[j]] = '\0';
-                }
-
-                cli_out ("%s %s %s %s %s %s %s %s %s %s %s %s %s %s %s %s",
-                         output_values[0], output_values[1],
-                         output_values[2], output_values[3],
-                         output_values[4], output_values[5],
-                         output_values[6], output_values[7],
-                         output_values[8], output_values[9],
-                         output_values[10], output_values[11],
-                         output_values[12], output_values[13],
-                         output_values[14], output_values[15]);
-        }
+        cli_out("%s %s %s %s %s %s %s %s %s %s %s %s %s %s %s %s",
+                output_values[0], output_values[1], output_values[2],
+                output_values[3], output_values[4], output_values[5],
+                output_values[6], output_values[7], output_values[8],
+                output_values[9], output_values[10], output_values[11],
+                output_values[12], output_values[13], output_values[14],
+                output_values[15]);
+    }
 
 out:
-        if (output_values) {
-                for (i = 0; i < number_of_fields; i++) {
-                        if (output_values[i])
-                                GF_FREE (output_values[i]);
-                }
-                GF_FREE (output_values);
+    if (output_values) {
+        for (i = 0; i < number_of_fields; i++) {
+            if (output_values[i])
+                GF_FREE(output_values[i]);
         }
+        GF_FREE(output_values);
+    }
 
-        if (hyphens)
-                GF_FREE (hyphens);
+    if (hyphens)
+        GF_FREE(hyphens);
 
-        return ret;
+    return ret;
 }
 
 int
-gf_gsync_status_t_comparator (const void *p, const void *q)
+gf_gsync_status_t_comparator(const void *p, const void *q)
 {
-        char *slavekey1 = NULL;
-        char *slavekey2 = NULL;
+    char *slavekey1 = NULL;
+    char *slavekey2 = NULL;
 
-        slavekey1 = get_struct_variable (20, (*(gf_gsync_status_t **)p));
-        slavekey2 = get_struct_variable (20, (*(gf_gsync_status_t **)q));
-        if (!slavekey1 || !slavekey2) {
-                gf_log ("cli", GF_LOG_ERROR,
-                        "struct member empty.");
-                return 0;
-        }
+    slavekey1 = get_struct_variable(20, (*(gf_gsync_status_t **)p));
+    slavekey2 = get_struct_variable(20, (*(gf_gsync_status_t **)q));
+    if (!slavekey1 || !slavekey2) {
+        gf_log("cli", GF_LOG_ERROR, "struct member empty.");
+        return 0;
+    }
 
-        return strcmp (slavekey1, slavekey2);
+    return strcmp(slavekey1, slavekey2);
 }
 
-int
-gf_cli_read_status_data (dict_t *dict,
-                         gf_gsync_status_t **sts_vals,
-                         int *spacing, int gsync_count,
-                         int number_of_fields)
-{
-        char   *tmp                    = NULL;
-        char    sts_val_name[PATH_MAX] = "";
-        int     ret                    = 0;
-        int     i                      = 0;
-        int     j                      = 0;
-
-        /* Storing per node status info in each object */
-        for (i = 0; i < gsync_count; i++) {
-                snprintf (sts_val_name, sizeof(sts_val_name), "status_value%d", i);
-
-                /* Fetching the values from dict, and calculating
-                   the max length for each field */
-                ret = dict_get_bin (dict, sts_val_name, (void **)&(sts_vals[i]));
-                if (ret)
-                        goto out;
-
-                for (j = 0; j < number_of_fields; j++) {
-                        tmp = get_struct_variable(j, sts_vals[i]);
-                        if (!tmp) {
-                                gf_log ("", GF_LOG_ERROR,
-                                        "struct member empty.");
-                                ret = -1;
-                                goto out;
-                        }
-                        if (strlen (tmp) > spacing[j])
-                                spacing[j] = strlen (tmp);
-                }
+static int
+gf_cli_read_status_data(dict_t *dict, gf_gsync_status_t **sts_vals,
+                        int *spacing, int gsync_count, int number_of_fields)
+{
+    char *tmp = NULL;
+    char sts_val_name[PATH_MAX] = "";
+    int ret = 0;
+    int i = 0;
+    int j = 0;
+
+    /* Storing per node status info in each object */
+    for (i = 0; i < gsync_count; i++) {
+        snprintf(sts_val_name, sizeof(sts_val_name), "status_value%d", i);
+
+        /* Fetching the values from dict, and calculating
+           the max length for each field */
+        ret = dict_get_bin(dict, sts_val_name, (void **)&(sts_vals[i]));
+        if (ret)
+            goto out;
+
+        for (j = 0; j < number_of_fields; j++) {
+            tmp = get_struct_variable(j, sts_vals[i]);
+            if (!tmp) {
+                gf_log("", GF_LOG_ERROR, "struct member empty.");
+                ret = -1;
+                goto out;
+            }
+            if (strlen(tmp) > spacing[j])
+                spacing[j] = strlen(tmp);
         }
+    }
 
-        /* Sort based on Session Slave */
-        qsort(sts_vals, gsync_count,
-              sizeof(gf_gsync_status_t *),
-              gf_gsync_status_t_comparator);
+    /* Sort based on Session Slave */
+    qsort(sts_vals, gsync_count, sizeof(gf_gsync_status_t *),
+          gf_gsync_status_t_comparator);
 
 out:
-        return ret;
+    return ret;
 }
 
-int
-gf_cli_gsync_status_output (dict_t *dict, gf_boolean_t is_detail)
-{
-        int                     gsync_count    = 0;
-        int                     i              = 0;
-        int                     ret            = 0;
-        int                     spacing[16]    = {0};
-        int                     num_of_fields  = 16;
-        char                    errmsg[1024]   = "";
-        char                   *master         = NULL;
-        char                   *slave          = NULL;
-        char                   *title_values[] = {"MASTER NODE", "MASTER VOL",
-                                                  "MASTER BRICK", "SLAVE USER",
-                                                  "SLAVE", "SLAVE NODE",
-                                                  "STATUS", "CRAWL STATUS",
-                                                  "LAST_SYNCED", "ENTRY",
-                                                  "DATA", "META", "FAILURES",
-                                                  "CHECKPOINT TIME",
-                                                  "CHECKPOINT COMPLETED",
-                                                  "CHECKPOINT COMPLETION TIME"};
-        gf_gsync_status_t     **sts_vals       = NULL;
-
-        /* Checks if any session is active or not */
-        ret = dict_get_int32 (dict, "gsync-count", &gsync_count);
-        if (ret) {
-                ret = dict_get_str (dict, "master", &master);
-
-                ret = dict_get_str (dict, "slave", &slave);
-
-                if (master) {
-                        if (slave)
-                                snprintf (errmsg, sizeof(errmsg), "No active "
-                                          "geo-replication sessions between %s"
-                                          " and %s", master, slave);
-                        else
-                                snprintf (errmsg, sizeof(errmsg), "No active "
-                                          "geo-replication sessions for %s",
-                                          master);
-                } else
-                        snprintf (errmsg, sizeof(errmsg), "No active "
-                                  "geo-replication sessions");
-
-                gf_log ("cli", GF_LOG_INFO, "%s", errmsg);
-                cli_out ("%s", errmsg);
-                ret = 0;
-                goto out;
-        }
+static int
+gf_cli_gsync_status_output(dict_t *dict, gf_boolean_t is_detail)
+{
+    int gsync_count = 0;
+    int i = 0;
+    int ret = 0;
+    int spacing[16] = {0};
+    int num_of_fields = 16;
+    char errmsg[1024] = "";
+    char *master = NULL;
+    char *slave = NULL;
+    static char *title_values[] = {"MASTER NODE",
+                                   "MASTER VOL",
+                                   "MASTER BRICK",
+                                   "SLAVE USER",
+                                   "SLAVE",
+                                   "SLAVE NODE",
+                                   "STATUS",
+                                   "CRAWL STATUS",
+                                   "LAST_SYNCED",
+                                   "ENTRY",
+                                   "DATA",
+                                   "META",
+                                   "FAILURES",
+                                   "CHECKPOINT TIME",
+                                   "CHECKPOINT COMPLETED",
+                                   "CHECKPOINT COMPLETION TIME"};
+    gf_gsync_status_t **sts_vals = NULL;
+
+    /* Checks if any session is active or not */
+    ret = dict_get_int32_sizen(dict, "gsync-count", &gsync_count);
+    if (ret) {
+        ret = dict_get_str_sizen(dict, "master", &master);
+
+        ret = dict_get_str_sizen(dict, "slave", &slave);
+
+        if (master) {
+            if (slave)
+                snprintf(errmsg, sizeof(errmsg),
+                         "No active geo-replication sessions between %s"
+                         " and %s",
+                         master, slave);
+            else
+                snprintf(errmsg, sizeof(errmsg),
+                         "No active geo-replication sessions for %s", master);
+        } else
+            snprintf(errmsg, sizeof(errmsg),
+                     "No active geo-replication sessions");
 
-        for (i = 0; i < num_of_fields; i++)
-                spacing[i] = strlen(title_values[i]);
+        gf_log("cli", GF_LOG_INFO, "%s", errmsg);
+        cli_out("%s", errmsg);
+        ret = -1;
+        goto out;
+    }
 
-        /* gsync_count = number of nodes reporting output.
-           each sts_val object will store output of each
-           node */
-        sts_vals = GF_CALLOC (gsync_count, sizeof (gf_gsync_status_t *),
-                              gf_common_mt_char);
-        if (!sts_vals) {
-                ret = -1;
-                goto out;
-        }
-        for (i = 0; i < gsync_count; i++) {
-                sts_vals[i] = GF_CALLOC (1, sizeof (gf_gsync_status_t),
-                                         gf_common_mt_char);
-                if (!sts_vals[i]) {
-                        ret = -1;
-                        goto out;
-                }
-        }
+    for (i = 0; i < num_of_fields; i++)
+        spacing[i] = strlen(title_values[i]);
 
-        ret = gf_cli_read_status_data (dict, sts_vals, spacing,
-                                       gsync_count, num_of_fields);
-        if (ret) {
-                gf_log ("", GF_LOG_ERROR, "Unable to read status data");
-                goto out;
-        }
+    /* gsync_count = number of nodes reporting output.
+       each sts_val object will store output of each
+       node */
+    sts_vals = GF_MALLOC(gsync_count * sizeof(gf_gsync_status_t *),
+                         gf_common_mt_char);
+    if (!sts_vals) {
+        ret = -1;
+        goto out;
+    }
 
-        ret = gf_cli_print_status (title_values, sts_vals, spacing, gsync_count,
-                                   num_of_fields, is_detail);
-        if (ret) {
-                gf_log ("", GF_LOG_ERROR, "Unable to print status output");
-                goto out;
-        }
+    ret = gf_cli_read_status_data(dict, sts_vals, spacing, gsync_count,
+                                  num_of_fields);
+    if (ret) {
+        gf_log("", GF_LOG_ERROR, "Unable to read status data");
+        goto out;
+    }
+
+    ret = gf_cli_print_status(title_values, sts_vals, spacing, gsync_count,
+                              num_of_fields, is_detail);
+    if (ret) {
+        gf_log("", GF_LOG_ERROR, "Unable to print status output");
+        goto out;
+    }
 
 out:
-        if (sts_vals)
-                GF_FREE (sts_vals);
+    if (sts_vals)
+        GF_FREE(sts_vals);
 
-        return ret;
+    return ret;
 }
 
 static int32_t
-write_contents_to_common_pem_file (dict_t *dict, int output_count)
-{
-        char            *workdir                   = NULL;
-        char             common_pem_file[PATH_MAX] = "";
-        char            *output                    = NULL;
-        char             output_name[PATH_MAX]     = "";
-        int              bytes_written             = 0;
-        int              fd                        = -1;
-        int              ret                       = -1;
-        int              i                         = -1;
-
-        ret = dict_get_str (dict, "glusterd_workdir", &workdir);
-        if (ret || !workdir) {
-                gf_log ("", GF_LOG_ERROR, "Unable to fetch workdir");
+write_contents_to_common_pem_file(dict_t *dict, int output_count)
+{
+    char *workdir = NULL;
+    char common_pem_file[PATH_MAX] = "";
+    char *output = NULL;
+    char output_name[32] = "";
+    int bytes_written = 0;
+    int fd = -1;
+    int ret = -1;
+    int i = -1;
+
+    ret = dict_get_str_sizen(dict, "glusterd_workdir", &workdir);
+    if (ret || !workdir) {
+        gf_log("", GF_LOG_ERROR, "Unable to fetch workdir");
+        ret = -1;
+        goto out;
+    }
+
+    snprintf(common_pem_file, sizeof(common_pem_file),
+             "%s/geo-replication/common_secret.pem.pub", workdir);
+
+    sys_unlink(common_pem_file);
+
+    fd = open(common_pem_file, O_WRONLY | O_CREAT, 0600);
+    if (fd == -1) {
+        gf_log("", GF_LOG_ERROR, "Failed to open %s Error : %s",
+               common_pem_file, strerror(errno));
+        ret = -1;
+        goto out;
+    }
+
+    for (i = 1; i <= output_count; i++) {
+        ret = snprintf(output_name, sizeof(output_name), "output_%d", i);
+        ret = dict_get_strn(dict, output_name, ret, &output);
+        if (ret) {
+            gf_log("", GF_LOG_ERROR, "Failed to get %s.", output_name);
+            cli_out("Unable to fetch output.");
+        }
+        if (output) {
+            bytes_written = sys_write(fd, output, strlen(output));
+            if (bytes_written != strlen(output)) {
+                gf_log("", GF_LOG_ERROR, "Failed to write to %s",
+                       common_pem_file);
                 ret = -1;
                 goto out;
-        }
-
-        snprintf (common_pem_file, sizeof(common_pem_file),
-                  "%s/geo-replication/common_secret.pem.pub",
-                  workdir);
-
-        sys_unlink (common_pem_file);
-
-        fd = open (common_pem_file, O_WRONLY | O_CREAT, 0600);
-        if (fd == -1) {
-                gf_log ("", GF_LOG_ERROR, "Failed to open %s"
-                        " Error : %s", common_pem_file,
-                        strerror (errno));
+            }
+            /* Adding the new line character */
+            bytes_written = sys_write(fd, "\n", 1);
+            if (bytes_written != 1) {
+                gf_log("", GF_LOG_ERROR, "Failed to add new line char");
                 ret = -1;
                 goto out;
+            }
+            output = NULL;
         }
+    }
 
-        for (i = 1; i <= output_count; i++) {
-                memset (output_name, '\0', sizeof (output_name));
-                snprintf (output_name, sizeof (output_name),
-                          "output_%d", i);
-                ret = dict_get_str (dict, output_name, &output);
-                if (ret) {
-                        gf_log ("", GF_LOG_ERROR, "Failed to get %s.",
-                                output_name);
-                        cli_out ("Unable to fetch output.");
-                }
-                if (output) {
-                        bytes_written = sys_write (fd, output, strlen(output));
-                        if (bytes_written != strlen(output)) {
-                                gf_log ("", GF_LOG_ERROR, "Failed to write "
-                                        "to %s", common_pem_file);
-                                ret = -1;
-                                goto out;
-                        }
-                        /* Adding the new line character */
-                        bytes_written = sys_write (fd, "\n", strlen("\n"));
-                        if (bytes_written != strlen("\n")) {
-                                gf_log ("", GF_LOG_ERROR,
-                                        "Failed to add new line char");
-                                ret = -1;
-                                goto out;
-                        }
-                        output = NULL;
-                }
-        }
-
-        cli_out ("Common secret pub file present at %s", common_pem_file);
-        ret = 0;
+    cli_out("Common secret pub file present at %s", common_pem_file);
+    ret = 0;
 out:
-        if (fd >= 0)
-                sys_close (fd);
+    if (fd >= 0)
+        sys_close(fd);
 
-        gf_log ("", GF_LOG_DEBUG, "Returning %d", ret);
-        return ret;
+    gf_log("", GF_LOG_DEBUG, RETURNING, ret);
+    return ret;
 }
 
-int
-gf_cli_sys_exec_cbk (struct rpc_req *req, struct iovec *iov,
-                     int count, void *myframe)
-{
-        int                     ret     = -1;
-        int                     output_count     = -1;
-        int                     i     = -1;
-        char                   *output  = NULL;
-        char                   *command = NULL;
-        char                    output_name[PATH_MAX] = "";
-        gf_cli_rsp              rsp     = {0, };
-        dict_t                  *dict   = NULL;
-        call_frame_t            *frame  = NULL;
-
-        if (req->rpc_status == -1) {
-                ret = -1;
-                goto out;
-        }
-
-        frame = myframe;
-
-        ret = xdr_to_generic (*iov, &rsp, (xdrproc_t)xdr_gf_cli_rsp);
-        if (ret < 0) {
-                gf_log (frame->this->name, GF_LOG_ERROR,
-                        "Failed to decode xdr response");
-                goto out;
-       }
-
-        dict = dict_new ();
-
-        if (!dict) {
-                ret = -1;
-                goto out;
-        }
+static int
+gf_cli_sys_exec_cbk(struct rpc_req *req, struct iovec *iov, int count,
+                    void *myframe)
+{
+    int ret = -1;
+    int output_count = -1;
+    int i = -1;
+    char *output = NULL;
+    char *command = NULL;
+    char output_name[32] = "";
+    gf_cli_rsp rsp = {
+        0,
+    };
+    dict_t *dict = NULL;
+
+    GF_ASSERT(myframe);
+
+    if (req->rpc_status == -1) {
+        goto out;
+    }
+
+    ret = xdr_to_generic(*iov, &rsp, (xdrproc_t)xdr_gf_cli_rsp);
+    if (ret < 0) {
+        gf_log(((call_frame_t *)myframe)->this->name, GF_LOG_ERROR,
+               XDR_DECODE_FAIL);
+        goto out;
+    }
+
+    dict = dict_new();
+
+    if (!dict) {
+        ret = -1;
+        goto out;
+    }
+
+    ret = dict_unserialize(rsp.dict.dict_val, rsp.dict.dict_len, &dict);
+
+    if (ret)
+        goto out;
+
+    if (rsp.op_ret) {
+        cli_err("%s", rsp.op_errstr ? rsp.op_errstr : "Command failed.");
+        ret = rsp.op_ret;
+        goto out;
+    }
 
-        ret = dict_unserialize (rsp.dict.dict_val, rsp.dict.dict_len, &dict);
+    ret = dict_get_int32_sizen(dict, "output_count", &output_count);
+    if (ret) {
+        cli_out("Command executed successfully.");
+        ret = 0;
+        goto out;
+    }
 
-        if (ret)
-                goto out;
+    ret = dict_get_str_sizen(dict, "command", &command);
+    if (ret) {
+        gf_log("", GF_LOG_ERROR, "Unable to get command from dict");
+        goto out;
+    }
 
-        if (rsp.op_ret) {
-                cli_err ("%s", rsp.op_errstr ? rsp.op_errstr :
-                         "Command failed.");
-                ret = rsp.op_ret;
-                goto out;
-        }
-
-        ret = dict_get_int32 (dict, "output_count", &output_count);
-        if (ret) {
-                cli_out ("Command executed successfully.");
-                ret = 0;
-                goto out;
-        }
+    if (!strcmp(command, "gsec_create")) {
+        ret = write_contents_to_common_pem_file(dict, output_count);
+        if (!ret)
+            goto out;
+    }
 
-        ret = dict_get_str (dict, "command", &command);
+    for (i = 1; i <= output_count; i++) {
+        ret = snprintf(output_name, sizeof(output_name), "output_%d", i);
+        ret = dict_get_strn(dict, output_name, ret, &output);
         if (ret) {
-                gf_log ("", GF_LOG_ERROR,
-                        "Unable to get command from dict");
-                goto out;
+            gf_log("", GF_LOG_ERROR, "Failed to get %s.", output_name);
+            cli_out("Unable to fetch output.");
         }
-
-        if (!strcmp (command, "gsec_create")) {
-                ret = write_contents_to_common_pem_file (dict, output_count);
-                if (!ret)
-                        goto out;
+        if (output) {
+            cli_out("%s", output);
+            output = NULL;
         }
+    }
 
-        for (i = 1; i <= output_count; i++) {
-                memset (output_name, '\0', sizeof (output_name));
-                snprintf (output_name, sizeof (output_name),
-                          "output_%d", i);
-                ret = dict_get_str (dict, output_name, &output);
-                if (ret) {
-                        gf_log ("", GF_LOG_ERROR, "Failed to get %s.",
-                                output_name);
-                        cli_out ("Unable to fetch output.");
-                }
-                if (output) {
-                        cli_out ("%s", output);
-                        output = NULL;
-                }
-        }
-
-        ret = 0;
+    ret = 0;
 out:
-        if (dict)
-                dict_unref (dict);
-        cli_cmd_broadcast_response (ret);
-
-        free (rsp.dict.dict_val);
-
-        return ret;
+    if (dict)
+        dict_unref(dict);
+    cli_cmd_broadcast_response(ret);
+    gf_free_xdr_cli_rsp(rsp);
+    return ret;
 }
 
-int
-gf_cli_copy_file_cbk (struct rpc_req *req, struct iovec *iov,
-                      int count, void *myframe)
+static int
+gf_cli_copy_file_cbk(struct rpc_req *req, struct iovec *iov, int count,
+                     void *myframe)
 {
-        int                     ret     = -1;
-        gf_cli_rsp              rsp     = {0, };
-        dict_t                  *dict   = NULL;
-        call_frame_t            *frame  = NULL;
+    int ret = -1;
+    gf_cli_rsp rsp = {
+        0,
+    };
+    dict_t *dict = NULL;
 
-        if (req->rpc_status == -1) {
-                ret = -1;
-                goto out;
-        }
+    GF_ASSERT(myframe);
 
-        frame = myframe;
+    if (req->rpc_status == -1) {
+        goto out;
+    }
 
-        ret = xdr_to_generic (*iov, &rsp, (xdrproc_t)xdr_gf_cli_rsp);
-        if (ret < 0) {
-                gf_log (frame->this->name, GF_LOG_ERROR,
-                        "Failed to decode xdr response");
-                goto out;
-       }
+    ret = xdr_to_generic(*iov, &rsp, (xdrproc_t)xdr_gf_cli_rsp);
+    if (ret < 0) {
+        gf_log(((call_frame_t *)myframe)->this->name, GF_LOG_ERROR,
+               XDR_DECODE_FAIL);
+        goto out;
+    }
 
-        dict = dict_new ();
+    dict = dict_new();
 
-        if (!dict) {
-                ret = -1;
-                goto out;
-        }
+    if (!dict) {
+        ret = -1;
+        goto out;
+    }
 
-        ret = dict_unserialize (rsp.dict.dict_val, rsp.dict.dict_len, &dict);
+    ret = dict_unserialize(rsp.dict.dict_val, rsp.dict.dict_len, &dict);
 
-       if (ret)
-                goto out;
+    if (ret)
+        goto out;
 
-        if (rsp.op_ret) {
-                cli_err ("%s", rsp.op_errstr ? rsp.op_errstr :
-                         "Copy unsuccessful");
-                ret = rsp.op_ret;
-                goto out;
-        }
+    if (rsp.op_ret) {
+        cli_err("%s", rsp.op_errstr ? rsp.op_errstr : "Copy unsuccessful");
+        ret = rsp.op_ret;
+        goto out;
+    }
 
-        cli_out ("Successfully copied file.");
+    cli_out("Successfully copied file.");
 
 out:
-        if (dict)
-                dict_unref (dict);
-        cli_cmd_broadcast_response (ret);
-
-        free (rsp.dict.dict_val);
-
-        return ret;
+    if (dict)
+        dict_unref(dict);
+    cli_cmd_broadcast_response(ret);
+    gf_free_xdr_cli_rsp(rsp);
+    return ret;
 }
 
-int
-gf_cli_gsync_set_cbk (struct rpc_req *req, struct iovec *iov,
-                         int count, void *myframe)
-{
-        int                     ret     = -1;
-        gf_cli_rsp              rsp     = {0, };
-        dict_t                  *dict   = NULL;
-        char                    *gsync_status = NULL;
-        char                    *master = NULL;
-        char                    *slave  = NULL;
-        int32_t                  type   = 0;
-        call_frame_t            *frame  = NULL;
-        gf_boolean_t             status_detail = _gf_false;
-
-
-        if (req->rpc_status == -1) {
-                ret = -1;
-                goto out;
-        }
-
-        frame = myframe;
-
-        ret = xdr_to_generic (*iov, &rsp, (xdrproc_t)xdr_gf_cli_rsp);
-        if (ret < 0) {
-                gf_log (frame->this->name, GF_LOG_ERROR,
-                        "Failed to decode xdr response");
-                goto out;
-        }
-
-        dict = dict_new ();
-
-        if (!dict) {
-                ret = -1;
-                goto out;
-        }
-
-        ret = dict_unserialize (rsp.dict.dict_val, rsp.dict.dict_len, &dict);
-
+static int
+gf_cli_gsync_set_cbk(struct rpc_req *req, struct iovec *iov, int count,
+                     void *myframe)
+{
+    int ret = -1;
+    gf_cli_rsp rsp = {
+        0,
+    };
+    dict_t *dict = NULL;
+    char *gsync_status = NULL;
+    char *master = NULL;
+    char *slave = NULL;
+    int32_t type = 0;
+    gf_boolean_t status_detail = _gf_false;
+
+    GF_ASSERT(myframe);
+
+    if (req->rpc_status == -1) {
+        goto out;
+    }
+
+    ret = xdr_to_generic(*iov, &rsp, (xdrproc_t)xdr_gf_cli_rsp);
+    if (ret < 0) {
+        gf_log(((call_frame_t *)myframe)->this->name, GF_LOG_ERROR,
+               XDR_DECODE_FAIL);
+        goto out;
+    }
+
+    dict = dict_new();
+
+    if (!dict) {
+        ret = -1;
+        goto out;
+    }
+
+    ret = dict_unserialize(rsp.dict.dict_val, rsp.dict.dict_len, &dict);
+
+    if (ret)
+        goto out;
+
+    if (global_state->mode & GLUSTER_MODE_XML) {
+        ret = cli_xml_output_vol_gsync(dict, rsp.op_ret, rsp.op_errno,
+                                       rsp.op_errstr);
         if (ret)
-                goto out;
-
-        if (global_state->mode & GLUSTER_MODE_XML) {
-                ret = cli_xml_output_vol_gsync (dict, rsp.op_ret, rsp.op_errno,
-                                                rsp.op_errstr);
-                if (ret)
-                        gf_log ("cli", GF_LOG_ERROR,
-                                "Error outputting to xml");
-                goto out;
-        }
-
-        if (rsp.op_ret) {
-                cli_err ("%s", rsp.op_errstr ? rsp.op_errstr :
-                         GEOREP" command unsuccessful");
-                ret = rsp.op_ret;
-                goto out;
-        }
-
-        ret = dict_get_str (dict, "gsync-status", &gsync_status);
-        if (!ret)
-                cli_out ("%s", gsync_status);
-        else
-                ret = 0;
-
-        ret = dict_get_int32 (dict, "type", &type);
-        if (ret) {
-                gf_log (frame->this->name, GF_LOG_ERROR, "failed to get type");
-                goto out;
-        }
-
-        switch (type) {
-                case GF_GSYNC_OPTION_TYPE_START:
-                case GF_GSYNC_OPTION_TYPE_STOP:
-                        if (dict_get_str (dict, "master", &master) != 0)
-                                master = "???";
-                        if (dict_get_str (dict, "slave", &slave) != 0)
-                                slave = "???";
-
-                        cli_out ("%s " GEOREP " session between %s & %s"
-                                 " has been successful",
-                                 type == GF_GSYNC_OPTION_TYPE_START ?
-                                  "Starting" : "Stopping",
-                                 master, slave);
-                break;
-
-                case GF_GSYNC_OPTION_TYPE_PAUSE:
-                case GF_GSYNC_OPTION_TYPE_RESUME:
-                        if (dict_get_str (dict, "master", &master) != 0)
-                                master = "???";
-                        if (dict_get_str (dict, "slave", &slave) != 0)
-                                slave = "???";
-
-                        cli_out ("%s " GEOREP " session between %s & %s"
-                                 " has been successful",
-                                 type == GF_GSYNC_OPTION_TYPE_PAUSE ?
-                                  "Pausing" : "Resuming",
-                                 master, slave);
-                break;
-
-                case GF_GSYNC_OPTION_TYPE_CONFIG:
-                        ret = gf_cli_gsync_config_command (dict);
-                break;
-
-                case GF_GSYNC_OPTION_TYPE_STATUS:
-                        status_detail = dict_get_str_boolean (dict,
-                                                              "status-detail",
-                                                              _gf_false);
-                        ret = gf_cli_gsync_status_output (dict, status_detail);
-                break;
-
-                case GF_GSYNC_OPTION_TYPE_DELETE:
-                        if (dict_get_str (dict, "master", &master) != 0)
-                                master = "???";
-                        if (dict_get_str (dict, "slave", &slave) != 0)
-                                slave = "???";
-                        cli_out ("Deleting " GEOREP " session between %s & %s"
-                                 " has been successful", master, slave);
-                break;
+            gf_log("cli", GF_LOG_ERROR, XML_ERROR);
+        goto out;
+    }
 
-                case GF_GSYNC_OPTION_TYPE_CREATE:
-                        if (dict_get_str (dict, "master", &master) != 0)
-                                master = "???";
-                        if (dict_get_str (dict, "slave", &slave) != 0)
-                                slave = "???";
-                        cli_out ("Creating " GEOREP " session between %s & %s"
-                                 " has been successful", master, slave);
-                break;
+    if (rsp.op_ret) {
+        cli_err("%s",
+                rsp.op_errstr ? rsp.op_errstr : GEOREP " command unsuccessful");
+        ret = rsp.op_ret;
+        goto out;
+    }
+
+    ret = dict_get_str_sizen(dict, "gsync-status", &gsync_status);
+    if (!ret)
+        cli_out("%s", gsync_status);
+
+    ret = dict_get_int32_sizen(dict, "type", &type);
+    if (ret) {
+        gf_log(((call_frame_t *)myframe)->this->name, GF_LOG_ERROR,
+               "failed to get type");
+        goto out;
+    }
+
+    switch (type) {
+        case GF_GSYNC_OPTION_TYPE_START:
+        case GF_GSYNC_OPTION_TYPE_STOP:
+            if (dict_get_str_sizen(dict, "master", &master) != 0)
+                master = "???";
+            if (dict_get_str_sizen(dict, "slave", &slave) != 0)
+                slave = "???";
+
+            cli_out(
+                "%s " GEOREP " session between %s & %s has been successful",
+                type == GF_GSYNC_OPTION_TYPE_START ? "Starting" : "Stopping",
+                master, slave);
+            break;
+
+        case GF_GSYNC_OPTION_TYPE_PAUSE:
+        case GF_GSYNC_OPTION_TYPE_RESUME:
+            if (dict_get_str_sizen(dict, "master", &master) != 0)
+                master = "???";
+            if (dict_get_str_sizen(dict, "slave", &slave) != 0)
+                slave = "???";
+
+            cli_out("%s " GEOREP " session between %s & %s has been successful",
+                    type == GF_GSYNC_OPTION_TYPE_PAUSE ? "Pausing" : "Resuming",
+                    master, slave);
+            break;
+
+        case GF_GSYNC_OPTION_TYPE_CONFIG:
+            ret = gf_cli_gsync_config_command(dict);
+            break;
+
+        case GF_GSYNC_OPTION_TYPE_STATUS:
+            status_detail = dict_get_str_boolean(dict, "status-detail",
+                                                 _gf_false);
+            ret = gf_cli_gsync_status_output(dict, status_detail);
+            break;
+
+        case GF_GSYNC_OPTION_TYPE_DELETE:
+            if (dict_get_str_sizen(dict, "master", &master) != 0)
+                master = "???";
+            if (dict_get_str_sizen(dict, "slave", &slave) != 0)
+                slave = "???";
+            cli_out("Deleting " GEOREP
+                    " session between %s & %s has been successful",
+                    master, slave);
+            break;
+
+        case GF_GSYNC_OPTION_TYPE_CREATE:
+            if (dict_get_str_sizen(dict, "master", &master) != 0)
+                master = "???";
+            if (dict_get_str_sizen(dict, "slave", &slave) != 0)
+                slave = "???";
+            cli_out("Creating " GEOREP
+                    " session between %s & %s has been successful",
+                    master, slave);
+            break;
 
-                default:
-                        cli_out (GEOREP" command executed successfully");
-        }
+        default:
+            cli_out(GEOREP " command executed successfully");
+    }
 
 out:
-        if (dict)
-                dict_unref (dict);
-        cli_cmd_broadcast_response (ret);
-
-        free (rsp.dict.dict_val);
-
-        return ret;
+    if (dict)
+        dict_unref(dict);
+    cli_cmd_broadcast_response(ret);
+    gf_free_xdr_cli_rsp(rsp);
+    return ret;
 }
 
-int32_t
-gf_cli_sys_exec (call_frame_t *frame, xlator_t *this, void *data)
+static int32_t
+gf_cli_sys_exec(call_frame_t *frame, xlator_t *this, void *data)
 {
-        int                      ret    = 0;
-        dict_t                  *dict   = NULL;
-        gf_cli_req               req = {{0,}};
+    int ret = 0;
+    dict_t *dict = NULL;
+    gf_cli_req req = {{
+        0,
+    }};
+
+    dict = data;
 
+    ret = cli_to_glusterd(&req, frame, gf_cli_sys_exec_cbk,
+                          (xdrproc_t)xdr_gf_cli_req, dict, GLUSTER_CLI_SYS_EXEC,
+                          this, cli_rpc_prog, NULL);
+    if (ret)
         if (!frame || !this || !data) {
-                ret = -1;
-                gf_log ("cli", GF_LOG_ERROR, "Invalid data");
-                goto out;
+            ret = -1;
+            gf_log("cli", GF_LOG_ERROR, "Invalid data");
         }
 
-        dict = data;
-
-        ret = cli_to_glusterd (&req, frame, gf_cli_sys_exec_cbk,
-                               (xdrproc_t) xdr_gf_cli_req, dict,
-                               GLUSTER_CLI_SYS_EXEC, this, cli_rpc_prog,
-                               NULL);
-out:
-        GF_FREE (req.dict.dict_val);
-        return ret;
+    GF_FREE(req.dict.dict_val);
+    return ret;
 }
 
-int32_t
-gf_cli_copy_file (call_frame_t *frame, xlator_t *this, void *data)
+static int32_t
+gf_cli_copy_file(call_frame_t *frame, xlator_t *this, void *data)
 {
-        int                      ret    = 0;
-        dict_t                  *dict   = NULL;
-        gf_cli_req               req = {{0,}};
+    int ret = 0;
+    dict_t *dict = NULL;
+    gf_cli_req req = {{
+        0,
+    }};
+
+    dict = data;
 
+    ret = cli_to_glusterd(&req, frame, gf_cli_copy_file_cbk,
+                          (xdrproc_t)xdr_gf_cli_req, dict,
+                          GLUSTER_CLI_COPY_FILE, this, cli_rpc_prog, NULL);
+    if (ret)
         if (!frame || !this || !data) {
-                ret = -1;
-                gf_log ("cli", GF_LOG_ERROR, "Invalid data");
-                goto out;
+            ret = -1;
+            gf_log("cli", GF_LOG_ERROR, "Invalid data");
         }
 
-        dict = data;
-
-        ret = cli_to_glusterd (&req, frame, gf_cli_copy_file_cbk,
-                               (xdrproc_t) xdr_gf_cli_req, dict,
-                               GLUSTER_CLI_COPY_FILE, this, cli_rpc_prog,
-                               NULL);
-out:
-        GF_FREE (req.dict.dict_val);
-        return ret;
+    GF_FREE(req.dict.dict_val);
+    return ret;
 }
 
-int32_t
-gf_cli_gsync_set (call_frame_t *frame, xlator_t *this,
-                     void *data)
+static int32_t
+gf_cli_gsync_set(call_frame_t *frame, xlator_t *this, void *data)
 {
-        int                      ret    = 0;
-        dict_t                  *dict   = NULL;
-        gf_cli_req               req = {{0,}};
-
-        if (!frame || !this || !data) {
-                ret = -1;
-                goto out;
-        }
+    int ret = 0;
+    dict_t *dict = NULL;
+    gf_cli_req req = {{
+        0,
+    }};
 
-        dict = data;
+    dict = data;
 
-        ret = cli_to_glusterd (&req, frame, gf_cli_gsync_set_cbk,
-                               (xdrproc_t) xdr_gf_cli_req, dict,
-                               GLUSTER_CLI_GSYNC_SET, this, cli_rpc_prog,
-                               NULL);
-
-out:
-        GF_FREE (req.dict.dict_val);
+    ret = cli_to_glusterd(&req, frame, gf_cli_gsync_set_cbk,
+                          (xdrproc_t)xdr_gf_cli_req, dict,
+                          GLUSTER_CLI_GSYNC_SET, this, cli_rpc_prog, NULL);
+    GF_FREE(req.dict.dict_val);
 
-        return ret;
+    return ret;
 }
 
-
 int
-cli_profile_info_percentage_cmp (void *a, void *b)
+cli_profile_info_percentage_cmp(void *a, void *b)
 {
-        cli_profile_info_t *ia = NULL;
-        cli_profile_info_t *ib = NULL;
-        int                ret = 0;
+    cli_profile_info_t *ia = NULL;
+    cli_profile_info_t *ib = NULL;
 
-        ia = a;
-        ib = b;
-        if (ia->percentage_avg_latency < ib->percentage_avg_latency)
-                ret = -1;
-        else if (ia->percentage_avg_latency > ib->percentage_avg_latency)
-                ret = 1;
-        else
-                ret = 0;
-        return ret;
-}
-
-
-void
-cmd_profile_volume_brick_out (dict_t *dict, int count, int interval)
-{
-        char                    key[256] = {0};
-        int                     i = 0;
-        uint64_t                sec = 0;
-        uint64_t                r_count = 0;
-        uint64_t                w_count = 0;
-        uint64_t                rb_counts[32] = {0};
-        uint64_t                wb_counts[32] = {0};
-        cli_profile_info_t      profile_info[GF_FOP_MAXVALUE] = {{0}};
-        char                    output[128] = {0};
-        int                     per_line = 0;
-        char                    read_blocks[128] = {0};
-        char                    write_blocks[128] = {0};
-        int                     index = 0;
-        int                     is_header_printed = 0;
-        int                     ret = 0;
-        double                  total_percentage_latency = 0;
-
-        for (i = 0; i < 32; i++) {
-                memset (key, 0, sizeof (key));
-                snprintf (key, sizeof (key), "%d-%d-read-%d", count,
-                          interval, (1 << i));
-                ret = dict_get_uint64 (dict, key, &rb_counts[i]);
-        }
-
-        for (i = 0; i < 32; i++) {
-                memset (key, 0, sizeof (key));
-                snprintf (key, sizeof (key), "%d-%d-write-%d", count, interval,
-                          (1<<i));
-                ret = dict_get_uint64 (dict, key, &wb_counts[i]);
-        }
+    ia = a;
+    ib = b;
+    if (ia->percentage_avg_latency < ib->percentage_avg_latency)
+        return -1;
+    else if (ia->percentage_avg_latency > ib->percentage_avg_latency)
+        return 1;
 
-        for (i = 0; i < GF_FOP_MAXVALUE; i++) {
-                memset (key, 0, sizeof (key));
-                snprintf (key, sizeof (key), "%d-%d-%d-hits", count,
-                          interval, i);
-                ret = dict_get_uint64 (dict, key, &profile_info[i].fop_hits);
-
-                memset (key, 0, sizeof (key));
-                snprintf (key, sizeof (key), "%d-%d-%d-avglatency", count,
-                          interval, i);
-                ret = dict_get_double (dict, key, &profile_info[i].avg_latency);
-
-                memset (key, 0, sizeof (key));
-                snprintf (key, sizeof (key), "%d-%d-%d-minlatency", count,
-                          interval, i);
-                ret = dict_get_double (dict, key, &profile_info[i].min_latency);
-
-                memset (key, 0, sizeof (key));
-                snprintf (key, sizeof (key), "%d-%d-%d-maxlatency", count,
-                          interval, i);
-                ret = dict_get_double (dict, key, &profile_info[i].max_latency);
-                profile_info[i].fop_name = (char *)gf_fop_list[i];
-
-                total_percentage_latency +=
-                       (profile_info[i].fop_hits * profile_info[i].avg_latency);
-        }
-        if (total_percentage_latency) {
-                for (i = 0; i < GF_FOP_MAXVALUE; i++) {
-                        profile_info[i].percentage_avg_latency = 100 * (
-                     (profile_info[i].avg_latency* profile_info[i].fop_hits) /
-                                total_percentage_latency);
-                }
-                gf_array_insertionsort (profile_info, 1, GF_FOP_MAXVALUE - 1,
-                                    sizeof (cli_profile_info_t),
-                                    cli_profile_info_percentage_cmp);
-        }
-        memset (key, 0, sizeof (key));
-        snprintf (key, sizeof (key), "%d-%d-duration", count, interval);
-        ret = dict_get_uint64 (dict, key, &sec);
-
-        memset (key, 0, sizeof (key));
-        snprintf (key, sizeof (key), "%d-%d-total-read", count, interval);
-        ret = dict_get_uint64 (dict, key, &r_count);
-
-        memset (key, 0, sizeof (key));
-        snprintf (key, sizeof (key), "%d-%d-total-write", count, interval);
-        ret = dict_get_uint64 (dict, key, &w_count);
+    return 0;
+}
 
-        if (ret == 0) {
+static void
+cmd_profile_volume_brick_out(dict_t *dict, int count, int interval)
+{
+    char key[256] = {0};
+    int i = 0;
+    uint64_t sec = 0;
+    uint64_t r_count = 0;
+    uint64_t w_count = 0;
+    uint64_t rb_counts[32] = {0};
+    uint64_t wb_counts[32] = {0};
+    cli_profile_info_t profile_info[GF_FOP_MAXVALUE] = {{0}};
+    cli_profile_info_t upcall_info[GF_UPCALL_FLAGS_MAXVALUE] = {
+        {0},
+    };
+    char output[128] = {0};
+    int per_line = 0;
+    char read_blocks[128] = {0};
+    char write_blocks[128] = {0};
+    int index = 0;
+    int is_header_printed = 0;
+    int ret = 0;
+    double total_percentage_latency = 0;
+
+    for (i = 0; i < 32; i++) {
+        snprintf(key, sizeof(key), "%d-%d-read-%" PRIu32, count, interval,
+                 (1U << i));
+        ret = dict_get_uint64(dict, key, &rb_counts[i]);
+        if (ret) {
+            gf_log("cli", GF_LOG_DEBUG, "failed to get %s from dict", key);
         }
+    }
 
-        if (interval == -1)
-                cli_out ("Cumulative Stats:");
-        else
-                cli_out ("Interval %d Stats:", interval);
-        snprintf (output, sizeof (output), "%14s", "Block Size:");
-        snprintf (read_blocks, sizeof (read_blocks), "%14s", "No. of Reads:");
-        snprintf (write_blocks, sizeof (write_blocks), "%14s", "No. of Writes:");
-        index = 14;
-        for (i = 0; i < 32; i++) {
-                if ((rb_counts[i] == 0) && (wb_counts[i] == 0))
-                        continue;
-                per_line++;
-                snprintf (output+index, sizeof (output)-index, "%19db+ ", (1<<i));
-                if (rb_counts[i]) {
-                        snprintf (read_blocks+index, sizeof (read_blocks)-index,
-                                  "%21"PRId64" ", rb_counts[i]);
-                } else {
-                        snprintf (read_blocks+index, sizeof (read_blocks)-index,
-                                  "%21s ", "0");
-                }
-                if (wb_counts[i]) {
-                        snprintf (write_blocks+index, sizeof (write_blocks)-index,
-                                  "%21"PRId64" ", wb_counts[i]);
-                } else {
-                        snprintf (write_blocks+index, sizeof (write_blocks)-index,
-                                  "%21s ", "0");
-                }
-                index += 22;
-                if (per_line == 3) {
-                        cli_out ("%s", output);
-                        cli_out ("%s", read_blocks);
-                        cli_out ("%s", write_blocks);
-                        cli_out (" ");
-                        per_line = 0;
-                        memset (output, 0, sizeof (output));
-                        memset (read_blocks, 0, sizeof (read_blocks));
-                        memset (write_blocks, 0, sizeof (write_blocks));
-                        snprintf (output, sizeof (output), "%14s", "Block Size:");
-                        snprintf (read_blocks, sizeof (read_blocks), "%14s",
-                                  "No. of Reads:");
-                        snprintf (write_blocks, sizeof (write_blocks), "%14s",
-                                  "No. of Writes:");
-                        index = 14;
-                }
+    for (i = 0; i < 32; i++) {
+        snprintf(key, sizeof(key), "%d-%d-write-%" PRIu32, count, interval,
+                 (1U << i));
+        ret = dict_get_uint64(dict, key, &wb_counts[i]);
+        if (ret) {
+            gf_log("cli", GF_LOG_DEBUG, "failed to get %s from dict", key);
         }
+    }
 
-        if (per_line != 0) {
-                cli_out ("%s", output);
-                cli_out ("%s", read_blocks);
-                cli_out ("%s", write_blocks);
-        }
-        for (i = 0; i < GF_FOP_MAXVALUE; i++) {
-                if (profile_info[i].fop_hits == 0)
-                        continue;
-                if (is_header_printed == 0) {
-                        cli_out ("%10s %13s %13s %13s %14s %11s", "%-latency",
-                                 "Avg-latency", "Min-Latency", "Max-Latency",
-                                 "No. of calls", "Fop");
-                        cli_out ("%10s %13s %13s %13s %14s %11s", "---------",
-                                 "-----------", "-----------", "-----------",
-                                 "------------", "----");
-                        is_header_printed = 1;
-                }
-                if (profile_info[i].fop_hits) {
-                        cli_out ("%10.2lf %10.2lf us %10.2lf us %10.2lf us"
-                                 " %14"PRId64" %11s",
-                                 profile_info[i].percentage_avg_latency,
-                                 profile_info[i].avg_latency,
-                                 profile_info[i].min_latency,
-                                 profile_info[i].max_latency,
-                                 profile_info[i].fop_hits,
-                                 profile_info[i].fop_name);
-                }
+    for (i = 0; i < GF_UPCALL_FLAGS_MAXVALUE; i++) {
+        snprintf(key, sizeof(key), "%d-%d-%d-upcall-hits", count, interval, i);
+        ret = dict_get_uint64(dict, key, &upcall_info[i].fop_hits);
+        if (ret) {
+            gf_log("cli", GF_LOG_DEBUG, "failed to get %s from dict", key);
         }
-        cli_out (" ");
-        cli_out ("%12s: %"PRId64" seconds", "Duration", sec);
-        cli_out ("%12s: %"PRId64" bytes", "Data Read", r_count);
-        cli_out ("%12s: %"PRId64" bytes", "Data Written", w_count);
-        cli_out (" ");
-}
+        upcall_info[i].fop_name = (char *)gf_upcall_list[i];
+    }
 
-int32_t
-gf_cli_profile_volume_cbk (struct rpc_req *req, struct iovec *iov,
-                              int count, void *myframe)
-{
-        gf_cli_rsp                        rsp   = {0,};
-        int                               ret   = -1;
-        dict_t                            *dict = NULL;
-        gf1_cli_stats_op                  op = GF_CLI_STATS_NONE;
-        char                              key[256] = {0};
-        int                               interval = 0;
-        int                               i = 1;
-        int32_t                           brick_count = 0;
-        char                              *volname = NULL;
-        char                              *brick = NULL;
-        char                              str[1024] = {0,};
-        int                               stats_cleared = 0;
-        gf1_cli_info_op                   info_op = GF_CLI_INFO_NONE;
-
-        if (-1 == req->rpc_status) {
-                goto out;
+    for (i = 0; i < GF_FOP_MAXVALUE; i++) {
+        snprintf(key, sizeof(key), "%d-%d-%d-hits", count, interval, i);
+        ret = dict_get_uint64(dict, key, &profile_info[i].fop_hits);
+        if (ret) {
+            gf_log("cli", GF_LOG_DEBUG, "failed to get %s from dict", key);
         }
 
-        gf_log ("cli", GF_LOG_DEBUG, "Received resp to profile");
-        ret = xdr_to_generic (*iov, &rsp, (xdrproc_t)xdr_gf_cli_rsp);
-        if (ret < 0) {
-                gf_log (((call_frame_t *) myframe)->this->name, GF_LOG_ERROR,
-                        "Failed to decode xdr response");
-                goto out;
+        snprintf(key, sizeof(key), "%d-%d-%d-avglatency", count, interval, i);
+        ret = dict_get_double(dict, key, &profile_info[i].avg_latency);
+        if (ret) {
+            gf_log("cli", GF_LOG_DEBUG, "failed to get %s from dict", key);
         }
 
-        dict = dict_new ();
-
-        if (!dict) {
-                ret = -1;
-                goto out;
+        snprintf(key, sizeof(key), "%d-%d-%d-minlatency", count, interval, i);
+        ret = dict_get_double(dict, key, &profile_info[i].min_latency);
+        if (ret) {
+            gf_log("cli", GF_LOG_DEBUG, "failed to get %s from dict", key);
         }
 
-        ret = dict_unserialize (rsp.dict.dict_val,
-                                rsp.dict.dict_len,
-                                &dict);
-
+        snprintf(key, sizeof(key), "%d-%d-%d-maxlatency", count, interval, i);
+        ret = dict_get_double(dict, key, &profile_info[i].max_latency);
         if (ret) {
-                gf_log ("", GF_LOG_ERROR,
-                                "Unable to allocate memory");
-                goto out;
-        } else {
-                dict->extra_stdfree = rsp.dict.dict_val;
+            gf_log("cli", GF_LOG_DEBUG, "failed to get %s from dict", key);
         }
+        profile_info[i].fop_name = (char *)gf_fop_list[i];
 
-        if (global_state->mode & GLUSTER_MODE_XML) {
-                ret = cli_xml_output_vol_profile (dict, rsp.op_ret,
-                                                  rsp.op_errno,
-                                                  rsp.op_errstr);
-                if (ret)
-                        gf_log ("cli", GF_LOG_ERROR,
-                                "Error outputting to xml");
-                goto out;
+        total_percentage_latency += (profile_info[i].fop_hits *
+                                     profile_info[i].avg_latency);
+    }
+    if (total_percentage_latency) {
+        for (i = 0; i < GF_FOP_MAXVALUE; i++) {
+            profile_info[i]
+                .percentage_avg_latency = 100 * ((profile_info[i].avg_latency *
+                                                  profile_info[i].fop_hits) /
+                                                 total_percentage_latency);
+        }
+        gf_array_insertionsort(profile_info, 1, GF_FOP_MAXVALUE - 1,
+                               sizeof(cli_profile_info_t),
+                               cli_profile_info_percentage_cmp);
+    }
+    snprintf(key, sizeof(key), "%d-%d-duration", count, interval);
+    ret = dict_get_uint64(dict, key, &sec);
+    if (ret) {
+        gf_log("cli", GF_LOG_DEBUG, "failed to get %s from dict", key);
+    }
+
+    snprintf(key, sizeof(key), "%d-%d-total-read", count, interval);
+    ret = dict_get_uint64(dict, key, &r_count);
+    if (ret) {
+        gf_log("cli", GF_LOG_DEBUG, "failed to get %s from dict", key);
+    }
+
+    snprintf(key, sizeof(key), "%d-%d-total-write", count, interval);
+    ret = dict_get_uint64(dict, key, &w_count);
+    if (ret) {
+        gf_log("cli", GF_LOG_DEBUG, "failed to get %s from dict", key);
+    }
+
+    if (interval == -1)
+        cli_out("Cumulative Stats:");
+    else
+        cli_out("Interval %d Stats:", interval);
+    snprintf(output, sizeof(output), "%14s", "Block Size:");
+    snprintf(read_blocks, sizeof(read_blocks), "%14s", "No. of Reads:");
+    snprintf(write_blocks, sizeof(write_blocks), "%14s", "No. of Writes:");
+    index = 14;
+    for (i = 0; i < 32; i++) {
+        if ((rb_counts[i] == 0) && (wb_counts[i] == 0))
+            continue;
+        per_line++;
+        snprintf(output + index, sizeof(output) - index, "%19" PRIu32 "b+ ",
+                 (1U << i));
+        if (rb_counts[i]) {
+            snprintf(read_blocks + index, sizeof(read_blocks) - index,
+                     "%21" PRId64 " ", rb_counts[i]);
+        } else {
+            snprintf(read_blocks + index, sizeof(read_blocks) - index, "%21s ",
+                     "0");
         }
+        if (wb_counts[i]) {
+            snprintf(write_blocks + index, sizeof(write_blocks) - index,
+                     "%21" PRId64 " ", wb_counts[i]);
+        } else {
+            snprintf(write_blocks + index, sizeof(write_blocks) - index,
+                     "%21s ", "0");
+        }
+        index += 22;
+        if (per_line == 3) {
+            cli_out("%s", output);
+            cli_out("%s", read_blocks);
+            cli_out("%s", write_blocks);
+            cli_out(" ");
+            per_line = 0;
+            snprintf(output, sizeof(output), "%14s", "Block Size:");
+            snprintf(read_blocks, sizeof(read_blocks), "%14s", "No. of Reads:");
+            snprintf(write_blocks, sizeof(write_blocks), "%14s",
+                     "No. of Writes:");
+            index = 14;
+        }
+    }
+
+    if (per_line != 0) {
+        cli_out("%s", output);
+        cli_out("%s", read_blocks);
+        cli_out("%s", write_blocks);
+    }
+    for (i = 0; i < GF_FOP_MAXVALUE; i++) {
+        if (profile_info[i].fop_hits == 0)
+            continue;
+        if (is_header_printed == 0) {
+            cli_out("%10s %13s %13s %13s %14s %11s", "%-latency", "Avg-latency",
+                    "Min-Latency", "Max-Latency", "No. of calls", "Fop");
+            cli_out("%10s %13s %13s %13s %14s %11s", "---------", "-----------",
+                    "-----------", "-----------", "------------", "----");
+            is_header_printed = 1;
+        }
+        if (profile_info[i].fop_hits) {
+            cli_out(
+                "%10.2lf %10.2lf us %10.2lf us %10.2lf us"
+                " %14" PRId64 " %11s",
+                profile_info[i].percentage_avg_latency,
+                profile_info[i].avg_latency, profile_info[i].min_latency,
+                profile_info[i].max_latency, profile_info[i].fop_hits,
+                profile_info[i].fop_name);
+        }
+    }
+
+    for (i = 0; i < GF_UPCALL_FLAGS_MAXVALUE; i++) {
+        if (upcall_info[i].fop_hits == 0)
+            continue;
+        if (upcall_info[i].fop_hits) {
+            cli_out(
+                "%10.2lf %10.2lf us %10.2lf us %10.2lf us"
+                " %14" PRId64 " %11s",
+                upcall_info[i].percentage_avg_latency,
+                upcall_info[i].avg_latency, upcall_info[i].min_latency,
+                upcall_info[i].max_latency, upcall_info[i].fop_hits,
+                upcall_info[i].fop_name);
+        }
+    }
+
+    cli_out(" ");
+    cli_out("%12s: %" PRId64 " seconds", "Duration", sec);
+    cli_out("%12s: %" PRId64 " bytes", "Data Read", r_count);
+    cli_out("%12s: %" PRId64 " bytes", "Data Written", w_count);
+    cli_out(" ");
+}
 
-        ret = dict_get_str (dict, "volname", &volname);
-        if (ret)
-                goto out;
-
-        ret = dict_get_int32 (dict, "op", (int32_t*)&op);
+static int32_t
+gf_cli_profile_volume_cbk(struct rpc_req *req, struct iovec *iov, int count,
+                          void *myframe)
+{
+    gf_cli_rsp rsp = {
+        0,
+    };
+    int ret = -1;
+    dict_t *dict = NULL;
+    gf1_cli_stats_op op = GF_CLI_STATS_NONE;
+    char key[64] = {0};
+    int len;
+    int interval = 0;
+    int i = 1;
+    int32_t brick_count = 0;
+    char *volname = NULL;
+    char *brick = NULL;
+    char str[1024] = {
+        0,
+    };
+    int stats_cleared = 0;
+    gf1_cli_info_op info_op = GF_CLI_INFO_NONE;
+
+    GF_ASSERT(myframe);
+
+    if (-1 == req->rpc_status) {
+        goto out;
+    }
+
+    gf_log("cli", GF_LOG_DEBUG, "Received resp to profile");
+    ret = xdr_to_generic(*iov, &rsp, (xdrproc_t)xdr_gf_cli_rsp);
+    if (ret < 0) {
+        gf_log(((call_frame_t *)myframe)->this->name, GF_LOG_ERROR,
+               XDR_DECODE_FAIL);
+        goto out;
+    }
+
+    dict = dict_new();
+
+    if (!dict) {
+        ret = -1;
+        goto out;
+    }
+
+    ret = dict_unserialize(rsp.dict.dict_val, rsp.dict.dict_len, &dict);
+
+    if (ret) {
+        gf_log("", GF_LOG_ERROR, DICT_UNSERIALIZE_FAIL);
+        goto out;
+    }
+
+    if (global_state->mode & GLUSTER_MODE_XML) {
+        ret = cli_xml_output_vol_profile(dict, rsp.op_ret, rsp.op_errno,
+                                         rsp.op_errstr);
         if (ret)
-                goto out;
-
-        if (rsp.op_ret && strcmp (rsp.op_errstr, "")) {
-                cli_err ("%s", rsp.op_errstr);
-        } else {
-                switch (op) {
-                case GF_CLI_STATS_START:
-                        cli_out ("Starting volume profile on %s has been %s ",
-                                 volname,
-                                 (rsp.op_ret) ? "unsuccessful": "successful");
-                        break;
-                case GF_CLI_STATS_STOP:
-                        cli_out ("Stopping volume profile on %s has been %s ",
-                                 volname,
-                                 (rsp.op_ret) ? "unsuccessful": "successful");
-                        break;
-                case GF_CLI_STATS_INFO:
-                        break;
-                default:
-                        cli_out ("volume profile on %s has been %s ",
-                                 volname,
-                                 (rsp.op_ret) ? "unsuccessful": "successful");
-                        break;
-                }
+            gf_log("cli", GF_LOG_ERROR, XML_ERROR);
+        goto out;
+    }
+
+    ret = dict_get_int32_sizen(dict, "op", (int32_t *)&op);
+    if (ret)
+        goto out;
+
+    ret = dict_get_str_sizen(dict, "volname", &volname);
+    if (ret)
+        goto out;
+
+    if (rsp.op_ret && strcmp(rsp.op_errstr, "")) {
+        cli_err("%s", rsp.op_errstr);
+    } else {
+        switch (op) {
+            case GF_CLI_STATS_START:
+                cli_out("Starting volume profile on %s has been %s ", volname,
+                        (rsp.op_ret) ? "unsuccessful" : "successful");
+                break;
+            case GF_CLI_STATS_STOP:
+                cli_out("Stopping volume profile on %s has been %s ", volname,
+                        (rsp.op_ret) ? "unsuccessful" : "successful");
+                break;
+            case GF_CLI_STATS_INFO:
+                break;
+            default:
+                cli_out("volume profile on %s has been %s ", volname,
+                        (rsp.op_ret) ? "unsuccessful" : "successful");
+                break;
         }
+    }
 
-        if (rsp.op_ret) {
-                ret = rsp.op_ret;
-                goto out;
-        }
+    if (rsp.op_ret) {
+        ret = rsp.op_ret;
+        goto out;
+    }
 
-        if (GF_CLI_STATS_INFO != op) {
-                ret = 0;
-                goto out;
-        }
+    if (GF_CLI_STATS_INFO != op) {
+        ret = 0;
+        goto out;
+    }
 
-        ret = dict_get_int32 (dict, "info-op", (int32_t*)&info_op);
-        if (ret)
-                goto out;
+    ret = dict_get_int32_sizen(dict, "count", &brick_count);
+    if (ret)
+        goto out;
 
-        ret = dict_get_int32 (dict, "count", &brick_count);
-        if (ret)
-                goto out;
+    if (!brick_count) {
+        cli_err("All bricks of volume %s are down.", volname);
+        goto out;
+    }
 
-        if (!brick_count) {
-                cli_err ("All bricks of volume %s are down.", volname);
-                goto out;
+    ret = dict_get_int32_sizen(dict, "info-op", (int32_t *)&info_op);
+    if (ret)
+        goto out;
+
+    while (i <= brick_count) {
+        len = snprintf(key, sizeof(key), "%d-brick", i);
+        ret = dict_get_strn(dict, key, len, &brick);
+        if (ret) {
+            gf_log("cli", GF_LOG_ERROR, "Couldn't get brick name");
+            goto out;
         }
 
-        while (i <= brick_count) {
-                memset (key, 0, sizeof (key));
-                snprintf (key, sizeof (key), "%d-brick", i);
-                ret = dict_get_str (dict, key, &brick);
-                if (ret) {
-                        gf_log ("cli", GF_LOG_ERROR, "Couldn't get brick name");
-                        goto out;
-                }
+        ret = dict_get_str_boolean(dict, "nfs", _gf_false);
 
-                ret = dict_get_str_boolean (dict, "nfs", _gf_false);
+        if (ret)
+            len = snprintf(str, sizeof(str), "NFS Server : %s", brick);
+        else
+            len = snprintf(str, sizeof(str), "Brick: %s", brick);
+        cli_out("%s", str);
+        memset(str, '-', len);
+        cli_out("%s", str);
+
+        if (GF_CLI_INFO_CLEAR == info_op) {
+            len = snprintf(key, sizeof(key), "%d-stats-cleared", i);
+            ret = dict_get_int32n(dict, key, len, &stats_cleared);
+            if (ret)
+                goto out;
+            cli_out(stats_cleared ? "Cleared stats."
+                                  : "Failed to clear stats.");
+        } else {
+            len = snprintf(key, sizeof(key), "%d-cumulative", i);
+            ret = dict_get_int32n(dict, key, len, &interval);
+            if (ret == 0)
+                cmd_profile_volume_brick_out(dict, i, interval);
 
-                if (ret)
-                        snprintf (str, sizeof (str), "NFS Server : %s", brick);
-                else
-                        snprintf (str, sizeof (str), "Brick: %s", brick);
-                cli_out ("%s", str);
-                memset (str, '-', strlen (str));
-                cli_out ("%s", str);
-
-                if (GF_CLI_INFO_CLEAR == info_op) {
-                        snprintf (key, sizeof (key), "%d-stats-cleared", i);
-                        ret = dict_get_int32 (dict, key, &stats_cleared);
-                        if (ret)
-                                goto out;
-                        cli_out (stats_cleared ? "Cleared stats." :
-                                                 "Failed to clear stats.");
-                } else {
-                        snprintf (key, sizeof (key), "%d-cumulative", i);
-                        ret = dict_get_int32 (dict, key, &interval);
-                        if (ret == 0)
-                                cmd_profile_volume_brick_out (dict, i,
-                                                              interval);
-
-                        snprintf (key, sizeof (key), "%d-interval", i);
-                        ret = dict_get_int32 (dict, key, &interval);
-                        if (ret == 0)
-                                cmd_profile_volume_brick_out (dict, i,
-                                                              interval);
-                }
-                i++;
+            len = snprintf(key, sizeof(key), "%d-interval", i);
+            ret = dict_get_int32n(dict, key, len, &interval);
+            if (ret == 0)
+                cmd_profile_volume_brick_out(dict, i, interval);
         }
-        ret = rsp.op_ret;
+        i++;
+    }
+    ret = rsp.op_ret;
 
 out:
-        if (dict)
-                dict_unref (dict);
-        free (rsp.op_errstr);
-        cli_cmd_broadcast_response (ret);
-        return ret;
+    if (dict)
+        dict_unref(dict);
+    cli_cmd_broadcast_response(ret);
+    gf_free_xdr_cli_rsp(rsp);
+    return ret;
 }
 
-int32_t
-gf_cli_profile_volume (call_frame_t *frame, xlator_t *this, void *data)
+static int32_t
+gf_cli_profile_volume(call_frame_t *frame, xlator_t *this, void *data)
 {
-        int                        ret   = -1;
-        gf_cli_req                 req   = {{0,}};
-        dict_t                     *dict = NULL;
-
-        GF_ASSERT (frame);
-        GF_ASSERT (this);
-        GF_ASSERT (data);
+    int ret = -1;
+    gf_cli_req req = {{
+        0,
+    }};
+    dict_t *dict = NULL;
 
-        if (!frame || !this || !data)
-                goto out;
-        dict = data;
+    GF_ASSERT(frame);
+    GF_ASSERT(this);
+    GF_ASSERT(data);
 
-        ret = cli_to_glusterd (&req, frame, gf_cli_profile_volume_cbk,
-                               (xdrproc_t) xdr_gf_cli_req, dict,
-                               GLUSTER_CLI_PROFILE_VOLUME, this, cli_rpc_prog,
-                               NULL);
+    dict = data;
 
-out:
-        gf_log ("cli", GF_LOG_DEBUG, "Returning %d", ret);
+    ret = cli_to_glusterd(&req, frame, gf_cli_profile_volume_cbk,
+                          (xdrproc_t)xdr_gf_cli_req, dict,
+                          GLUSTER_CLI_PROFILE_VOLUME, this, cli_rpc_prog, NULL);
+    gf_log("cli", GF_LOG_DEBUG, RETURNING, ret);
 
-        GF_FREE (req.dict.dict_val);
-        return ret;
+    GF_FREE(req.dict.dict_val);
+    return ret;
 }
 
-int32_t
-gf_cli_top_volume_cbk (struct rpc_req *req, struct iovec *iov,
-                              int count, void *myframe)
-{
-        gf_cli_rsp                        rsp   = {0,};
-        int                               ret   = -1;
-        dict_t                           *dict = NULL;
-        gf1_cli_stats_op                  op = GF_CLI_STATS_NONE;
-        char                              key[256] = {0};
-        int                               i = 0;
-        int32_t                           brick_count = 0;
-        char                              brick[1024];
-        int32_t                           members = 0;
-        char                             *filename;
-        char                             *bricks;
-        uint64_t                          value = 0;
-        int32_t                           j = 0;
-        gf1_cli_top_op                    top_op = GF_CLI_TOP_NONE;
-        uint64_t                          nr_open = 0;
-        uint64_t                          max_nr_open = 0;
-        double                            throughput = 0;
-        double                            time = 0;
-        int32_t                           time_sec = 0;
-        long int                          time_usec = 0;
-        char                              timestr[256] = {0, };
-        char                             *openfd_str = NULL;
-        gf_boolean_t                      nfs = _gf_false;
-        gf_boolean_t                      clear_stats = _gf_false;
-        int                               stats_cleared = 0;
-
-        if (-1 == req->rpc_status) {
-                goto out;
-        }
+static int32_t
+gf_cli_top_volume_cbk(struct rpc_req *req, struct iovec *iov, int count,
+                      void *myframe)
+{
+    gf_cli_rsp rsp = {
+        0,
+    };
+    int ret = -1;
+    dict_t *dict = NULL;
+    gf1_cli_stats_op op = GF_CLI_STATS_NONE;
+    char key[256] = {0};
+    int keylen;
+    int i = 0;
+    int32_t brick_count = 0;
+    char brick[1024];
+    int32_t members = 0;
+    char *filename;
+    char *bricks;
+    uint64_t value = 0;
+    int32_t j = 0;
+    gf1_cli_top_op top_op = GF_CLI_TOP_NONE;
+    uint64_t nr_open = 0;
+    uint64_t max_nr_open = 0;
+    double throughput = 0;
+    double time = 0;
+    int32_t time_sec = 0;
+    long int time_usec = 0;
+    char timestr[GF_TIMESTR_SIZE] = {
+        0,
+    };
+    char *openfd_str = NULL;
+    gf_boolean_t nfs = _gf_false;
+    gf_boolean_t clear_stats = _gf_false;
+    int stats_cleared = 0;
+
+    GF_ASSERT(myframe);
+
+    if (-1 == req->rpc_status) {
+        goto out;
+    }
+
+    gf_log("cli", GF_LOG_DEBUG, "Received resp to top");
+    ret = xdr_to_generic(*iov, &rsp, (xdrproc_t)xdr_gf_cli_rsp);
+    if (ret < 0) {
+        gf_log(((call_frame_t *)myframe)->this->name, GF_LOG_ERROR,
+               XDR_DECODE_FAIL);
+        goto out;
+    }
+
+    if (rsp.op_ret) {
+        if (strcmp(rsp.op_errstr, ""))
+            cli_err("%s", rsp.op_errstr);
+        cli_err("volume top unsuccessful");
+        ret = rsp.op_ret;
+        goto out;
+    }
 
-        gf_log ("cli", GF_LOG_DEBUG, "Received resp to top");
-        ret = xdr_to_generic (*iov, &rsp, (xdrproc_t)xdr_gf_cli_rsp);
-        if (ret < 0) {
-                gf_log (((call_frame_t *) myframe)->this->name, GF_LOG_ERROR,
-                        "Failed to decode xdr response");
-                goto out;
-        }
+    dict = dict_new();
 
-        if (rsp.op_ret) {
-                if (strcmp (rsp.op_errstr, ""))
-                        cli_err ("%s", rsp.op_errstr);
-                cli_err ("volume top unsuccessful");
-                ret = rsp.op_ret;
-                goto out;
-        }
+    if (!dict) {
+        ret = -1;
+        goto out;
+    }
 
-        dict = dict_new ();
+    ret = dict_unserialize(rsp.dict.dict_val, rsp.dict.dict_len, &dict);
 
-        if (!dict) {
-                ret = -1;
-                goto out;
-        }
+    if (ret) {
+        gf_log("", GF_LOG_ERROR, DICT_UNSERIALIZE_FAIL);
+        goto out;
+    }
+
+    ret = dict_get_int32_sizen(dict, "op", (int32_t *)&op);
 
-        ret = dict_unserialize (rsp.dict.dict_val,
-                                rsp.dict.dict_len,
-                                &dict);
+    if (op != GF_CLI_STATS_TOP) {
+        ret = 0;
+        goto out;
+    }
 
+    if (global_state->mode & GLUSTER_MODE_XML) {
+        ret = cli_xml_output_vol_top(dict, rsp.op_ret, rsp.op_errno,
+                                     rsp.op_errstr);
         if (ret) {
-                gf_log ("", GF_LOG_ERROR,
-                                "Unable to allocate memory");
-                goto out;
+            gf_log("cli", GF_LOG_ERROR, XML_ERROR);
         }
+        goto out;
+    }
 
-        ret = dict_get_int32 (dict, "op", (int32_t*)&op);
+    ret = dict_get_int32_sizen(dict, "count", &brick_count);
+    if (ret)
+        goto out;
+    keylen = snprintf(key, sizeof(key), "%d-top-op", 1);
+    ret = dict_get_int32n(dict, key, keylen, (int32_t *)&top_op);
+    if (ret)
+        goto out;
 
-        if (op != GF_CLI_STATS_TOP) {
-                ret = 0;
+    clear_stats = dict_get_str_boolean(dict, "clear-stats", _gf_false);
+
+    while (i < brick_count) {
+        i++;
+        keylen = snprintf(brick, sizeof(brick), "%d-brick", i);
+        ret = dict_get_strn(dict, brick, keylen, &bricks);
+        if (ret)
+            goto out;
+
+        nfs = dict_get_str_boolean(dict, "nfs", _gf_false);
+
+        if (clear_stats) {
+            keylen = snprintf(key, sizeof(key), "%d-stats-cleared", i);
+            ret = dict_get_int32n(dict, key, keylen, &stats_cleared);
+            if (ret)
                 goto out;
+            cli_out(stats_cleared ? "Cleared stats for %s %s"
+                                  : "Failed to clear stats for %s %s",
+                    nfs ? "NFS server on" : "brick", bricks);
+            continue;
         }
 
-        if (global_state->mode & GLUSTER_MODE_XML) {
-                ret = cli_xml_output_vol_top (dict, rsp.op_ret,
-                                              rsp.op_errno,
-                                              rsp.op_errstr);
-                if (ret) {
-                        gf_log ("cli", GF_LOG_ERROR,
-                                "Error outputting to xml");
+        if (nfs)
+            cli_out("NFS Server : %s", bricks);
+        else
+            cli_out("Brick: %s", bricks);
+
+        keylen = snprintf(key, sizeof(key), "%d-members", i);
+        ret = dict_get_int32n(dict, key, keylen, &members);
+
+        switch (top_op) {
+            case GF_CLI_TOP_OPEN:
+                snprintf(key, sizeof(key), "%d-current-open", i);
+                ret = dict_get_uint64(dict, key, &nr_open);
+                if (ret)
+                    break;
+                snprintf(key, sizeof(key), "%d-max-open", i);
+                ret = dict_get_uint64(dict, key, &max_nr_open);
+                if (ret)
+                    goto out;
+                keylen = snprintf(key, sizeof(key), "%d-max-openfd-time", i);
+                ret = dict_get_strn(dict, key, keylen, &openfd_str);
+                if (ret)
+                    goto out;
+                cli_out("Current open fds: %" PRIu64 ", Max open fds: %" PRIu64
+                        ", Max openfd time: %s",
+                        nr_open, max_nr_open, openfd_str);
+            case GF_CLI_TOP_READ:
+            case GF_CLI_TOP_WRITE:
+            case GF_CLI_TOP_OPENDIR:
+            case GF_CLI_TOP_READDIR:
+                if (!members) {
+                    continue;
+                }
+                cli_out("Count\t\tfilename\n=======================");
+                break;
+            case GF_CLI_TOP_READ_PERF:
+            case GF_CLI_TOP_WRITE_PERF:
+                snprintf(key, sizeof(key), "%d-throughput", i);
+                ret = dict_get_double(dict, key, &throughput);
+                if (!ret) {
+                    snprintf(key, sizeof(key), "%d-time", i);
+                    ret = dict_get_double(dict, key, &time);
                 }
+                if (!ret)
+                    cli_out("Throughput %.2f MBps time %.4f secs", throughput,
+                            time / 1e6);
+
+                if (!members) {
+                    continue;
+                }
+                cli_out("%*s %-*s %-*s", VOL_TOP_PERF_SPEED_WIDTH, "MBps",
+                        VOL_TOP_PERF_FILENAME_DEF_WIDTH, "Filename",
+                        VOL_TOP_PERF_TIME_WIDTH, "Time");
+                cli_out("%*s %-*s %-*s", VOL_TOP_PERF_SPEED_WIDTH,
+                        "====", VOL_TOP_PERF_FILENAME_DEF_WIDTH,
+                        "========", VOL_TOP_PERF_TIME_WIDTH, "====");
+                break;
+            default:
                 goto out;
         }
 
-        ret = dict_get_int32 (dict, "count", &brick_count);
-        if (ret)
-                goto out;
-        snprintf (key, sizeof (key), "%d-top-op", 1);
-        ret = dict_get_int32 (dict, key, (int32_t*)&top_op);
-        if (ret)
-                goto out;
-
-        clear_stats = dict_get_str_boolean (dict, "clear-stats", _gf_false);
-
-        while (i < brick_count) {
-                i++;
-                snprintf (brick, sizeof (brick), "%d-brick", i);
-                ret = dict_get_str (dict, brick, &bricks);
-                if (ret)
-                        goto out;
-
-                nfs = dict_get_str_boolean (dict, "nfs", _gf_false);
-
-                if (clear_stats) {
-                        memset (key, 0, sizeof (key));
-                        snprintf (key, sizeof (key), "%d-stats-cleared", i);
-                        ret = dict_get_int32 (dict, key, &stats_cleared);
-                        if (ret)
-                                goto out;
-                        cli_out (stats_cleared ? "Cleared stats for %s %s" :
-                                 "Failed to clear stats for %s %s",
-                                 nfs ? "NFS server on" : "brick", bricks);
-                        continue;
-                }
-
-                if (nfs)
-                        cli_out ("NFS Server : %s", bricks);
+        for (j = 1; j <= members; j++) {
+            keylen = snprintf(key, sizeof(key), "%d-filename-%d", i, j);
+            ret = dict_get_strn(dict, key, keylen, &filename);
+            if (ret)
+                break;
+            snprintf(key, sizeof(key), "%d-value-%d", i, j);
+            ret = dict_get_uint64(dict, key, &value);
+            if (ret)
+                goto out;
+            if (top_op == GF_CLI_TOP_READ_PERF ||
+                top_op == GF_CLI_TOP_WRITE_PERF) {
+                keylen = snprintf(key, sizeof(key), "%d-time-sec-%d", i, j);
+                ret = dict_get_int32n(dict, key, keylen, (int32_t *)&time_sec);
+                if (ret)
+                    goto out;
+                keylen = snprintf(key, sizeof(key), "%d-time-usec-%d", i, j);
+                ret = dict_get_int32n(dict, key, keylen, (int32_t *)&time_usec);
+                if (ret)
+                    goto out;
+                gf_time_fmt(timestr, sizeof timestr, time_sec, gf_timefmt_FT);
+                snprintf(timestr + strlen(timestr),
+                         sizeof timestr - strlen(timestr), ".%ld", time_usec);
+                if (strlen(filename) < VOL_TOP_PERF_FILENAME_DEF_WIDTH)
+                    cli_out("%*" PRIu64 " %-*s %-*s", VOL_TOP_PERF_SPEED_WIDTH,
+                            value, VOL_TOP_PERF_FILENAME_DEF_WIDTH, filename,
+                            VOL_TOP_PERF_TIME_WIDTH, timestr);
                 else
-                        cli_out ("Brick: %s", bricks);
-
-                snprintf(key, sizeof (key), "%d-members", i);
-                ret = dict_get_int32 (dict, key, &members);
-
-                switch (top_op) {
-                case GF_CLI_TOP_OPEN:
-                        snprintf (key, sizeof (key), "%d-current-open", i);
-                        ret = dict_get_uint64 (dict, key, &nr_open);
-                        if (ret)
-                                break;
-                        snprintf (key, sizeof (key), "%d-max-open", i);
-                        ret = dict_get_uint64 (dict, key, &max_nr_open);
-                        if (ret)
-                                goto out;
-                        snprintf (key, sizeof (key), "%d-max-openfd-time", i);
-                        ret = dict_get_str (dict, key, &openfd_str);
-                        if (ret)
-                                goto out;
-                        cli_out ("Current open fds: %"PRIu64", Max open"
-                                " fds: %"PRIu64", Max openfd time: %s", nr_open,
-                                 max_nr_open, openfd_str);
-                case GF_CLI_TOP_READ:
-                case GF_CLI_TOP_WRITE:
-                case GF_CLI_TOP_OPENDIR:
-                case GF_CLI_TOP_READDIR:
-                        if (!members) {
-                                continue;
-                        }
-                        cli_out ("Count\t\tfilename\n=======================");
-                        break;
-                case GF_CLI_TOP_READ_PERF:
-                case GF_CLI_TOP_WRITE_PERF:
-                        snprintf (key, sizeof (key), "%d-throughput", i);
-                        ret = dict_get_double (dict, key, &throughput);
-                        if (!ret) {
-                                snprintf (key, sizeof (key), "%d-time", i);
-                                ret = dict_get_double (dict, key, &time);
-                        }
-                        if (!ret)
-                                cli_out ("Throughput %.2f MBps time %.4f secs", throughput,
-                                          time / 1e6);
-
-                        if (!members) {
-                                continue;
-                        }
-                        cli_out ("%*s %-*s %-*s",
-                                 VOL_TOP_PERF_SPEED_WIDTH, "MBps",
-                                 VOL_TOP_PERF_FILENAME_DEF_WIDTH, "Filename",
-                                 VOL_TOP_PERF_TIME_WIDTH, "Time");
-                        cli_out ("%*s %-*s %-*s",
-                                 VOL_TOP_PERF_SPEED_WIDTH, "====",
-                                 VOL_TOP_PERF_FILENAME_DEF_WIDTH, "========",
-                                 VOL_TOP_PERF_TIME_WIDTH, "====");
-                        break;
-                default:
-                        goto out;
-                }
-
-                for (j = 1; j <= members; j++) {
-                        snprintf (key, sizeof (key), "%d-filename-%d", i, j);
-                        ret = dict_get_str (dict, key, &filename);
-                        if (ret)
-                                break;
-                        snprintf (key, sizeof (key), "%d-value-%d", i, j);
-                        ret = dict_get_uint64 (dict, key, &value);
-                        if (ret)
-                                goto out;
-                        if ( top_op == GF_CLI_TOP_READ_PERF ||
-                                top_op == GF_CLI_TOP_WRITE_PERF) {
-                                snprintf (key, sizeof (key), "%d-time-sec-%d", i, j);
-                                ret = dict_get_int32 (dict, key, (int32_t *)&time_sec);
-                                if (ret)
-                                        goto out;
-                                snprintf (key, sizeof (key), "%d-time-usec-%d", i, j);
-                                ret = dict_get_int32 (dict, key, (int32_t *)&time_usec);
-                                if (ret)
-                                        goto out;
-                                gf_time_fmt (timestr, sizeof timestr,
-                                             time_sec, gf_timefmt_FT);
-                                snprintf (timestr + strlen (timestr), sizeof timestr - strlen (timestr),
-                                  ".%ld", time_usec);
-                                if (strlen (filename) < VOL_TOP_PERF_FILENAME_DEF_WIDTH)
-                                        cli_out ("%*"PRIu64" %-*s %-*s",
-                                                 VOL_TOP_PERF_SPEED_WIDTH,
-                                                 value,
-                                                 VOL_TOP_PERF_FILENAME_DEF_WIDTH,
-                                                 filename,
-                                                 VOL_TOP_PERF_TIME_WIDTH,
-                                                 timestr);
-                                else
-                                        cli_out ("%*"PRIu64" ...%-*s %-*s",
-                                                 VOL_TOP_PERF_SPEED_WIDTH,
-                                                 value,
-                                                 VOL_TOP_PERF_FILENAME_ALT_WIDTH ,
-                                                 filename + strlen (filename) -
-                                                 VOL_TOP_PERF_FILENAME_ALT_WIDTH,
-                                                 VOL_TOP_PERF_TIME_WIDTH,
-                                                 timestr);
-                        } else {
-                                cli_out ("%"PRIu64"\t\t%s", value, filename);
-                        }
-                }
+                    cli_out("%*" PRIu64 " ...%-*s %-*s",
+                            VOL_TOP_PERF_SPEED_WIDTH, value,
+                            VOL_TOP_PERF_FILENAME_ALT_WIDTH,
+                            filename + strlen(filename) -
+                                VOL_TOP_PERF_FILENAME_ALT_WIDTH,
+                            VOL_TOP_PERF_TIME_WIDTH, timestr);
+            } else {
+                cli_out("%" PRIu64 "\t\t%s", value, filename);
+            }
         }
-        ret = rsp.op_ret;
+    }
+    ret = rsp.op_ret;
 
 out:
-        cli_cmd_broadcast_response (ret);
+    cli_cmd_broadcast_response(ret);
 
-        if (dict)
-                dict_unref (dict);
+    if (dict)
+        dict_unref(dict);
 
-        free (rsp.dict.dict_val);
-        return ret;
+    gf_free_xdr_cli_rsp(rsp);
+    return ret;
 }
 
-int32_t
-gf_cli_top_volume (call_frame_t *frame, xlator_t *this, void *data)
+static int32_t
+gf_cli_top_volume(call_frame_t *frame, xlator_t *this, void *data)
 {
-        int                        ret   = -1;
-        gf_cli_req                 req   = {{0,}};
-        dict_t                     *dict = NULL;
-
-        GF_ASSERT (frame);
-        GF_ASSERT (this);
-        GF_ASSERT (data);
+    int ret = -1;
+    gf_cli_req req = {{
+        0,
+    }};
+    dict_t *dict = NULL;
 
-        if (!frame || !this || !data)
-                goto out;
-        dict = data;
+    GF_ASSERT(frame);
+    GF_ASSERT(this);
+    GF_ASSERT(data);
 
-        ret = cli_to_glusterd (&req, frame, gf_cli_top_volume_cbk,
-                               (xdrproc_t) xdr_gf_cli_req, dict,
-                               GLUSTER_CLI_PROFILE_VOLUME, this, cli_rpc_prog,
-                               NULL);
+    dict = data;
 
-out:
-        gf_log ("cli", GF_LOG_DEBUG, "Returning %d", ret);
-        GF_FREE (req.dict.dict_val);
-        return ret;
+    ret = cli_to_glusterd(&req, frame, gf_cli_top_volume_cbk,
+                          (xdrproc_t)xdr_gf_cli_req, dict,
+                          GLUSTER_CLI_PROFILE_VOLUME, this, cli_rpc_prog, NULL);
+    gf_log("cli", GF_LOG_DEBUG, RETURNING, ret);
+    GF_FREE(req.dict.dict_val);
+    return ret;
 }
 
-
-int
-gf_cli_getwd_cbk (struct rpc_req *req, struct iovec *iov,
-                       int count, void *myframe)
+static int
+gf_cli_getwd_cbk(struct rpc_req *req, struct iovec *iov, int count,
+                 void *myframe)
 {
-        gf1_cli_getwd_rsp rsp   = {0,};
-        int               ret   = -1;
-
-        if (-1 == req->rpc_status) {
-                goto out;
-        }
-
-        ret = xdr_to_generic (*iov, &rsp, (xdrproc_t)xdr_gf1_cli_getwd_rsp);
-        if (ret < 0) {
-                gf_log (((call_frame_t *) myframe)->this->name, GF_LOG_ERROR,
-                        "Failed to decode xdr response");
-                goto out;
-        }
-
-        if (rsp.op_ret == -1) {
-                cli_err ("getwd failed");
-                ret = rsp.op_ret;
-                goto out;
-        }
+    gf1_cli_getwd_rsp rsp = {
+        0,
+    };
+    int ret = -1;
 
-        gf_log ("cli", GF_LOG_INFO, "Received resp to getwd");
+    GF_ASSERT(myframe);
 
-        cli_out ("%s", rsp.wd);
+    if (-1 == req->rpc_status) {
+        goto out;
+    }
 
-        ret = 0;
+    ret = xdr_to_generic(*iov, &rsp, (xdrproc_t)xdr_gf1_cli_getwd_rsp);
+    if (ret < 0) {
+        gf_log(((call_frame_t *)myframe)->this->name, GF_LOG_ERROR,
+               XDR_DECODE_FAIL);
+        goto out;
+    }
 
-out:
-        cli_cmd_broadcast_response (ret);
-        return ret;
-}
-
-int32_t
-gf_cli_getwd (call_frame_t *frame, xlator_t *this, void *data)
-{
-        int                      ret = -1;
-        gf1_cli_getwd_req        req = {0,};
+    if (rsp.op_ret == -1) {
+        cli_err("getwd failed");
+        ret = rsp.op_ret;
+        goto out;
+    }
 
-        GF_ASSERT (frame);
-        GF_ASSERT (this);
+    gf_log("cli", GF_LOG_INFO, "Received resp to getwd");
 
-        if (!frame || !this)
-                goto out;
+    cli_out("%s", rsp.wd);
 
-        ret = cli_cmd_submit (NULL, &req, frame, cli_rpc_prog,
-                              GLUSTER_CLI_GETWD, NULL,
-                              this, gf_cli_getwd_cbk,
-                              (xdrproc_t) xdr_gf1_cli_getwd_req);
+    ret = 0;
 
 out:
-        gf_log ("cli", GF_LOG_DEBUG, "Returning %d", ret);
+    cli_cmd_broadcast_response(ret);
+    if (rsp.wd) {
+        free(rsp.wd);
+    }
 
-        return ret;
+    return ret;
 }
 
-void
-cli_print_volume_status_mempool (dict_t *dict, char *prefix)
-{
-        int             ret = -1;
-        int32_t         mempool_count = 0;
-        char            *name = NULL;
-        int32_t         hotcount = 0;
-        int32_t         coldcount = 0;
-        uint64_t        paddedsizeof = 0;
-        uint64_t        alloccount = 0;
-        int32_t         maxalloc = 0;
-        uint64_t        pool_misses = 0;
-        int32_t         maxstdalloc = 0;
-        char            key[1024] = {0,};
-        int             i = 0;
-
-        GF_ASSERT (dict);
-        GF_ASSERT (prefix);
-
-        memset (key, 0, sizeof (key));
-        snprintf (key, sizeof (key), "%s.mempool-count",prefix);
-        ret = dict_get_int32 (dict, key, &mempool_count);
-        if (ret)
-                goto out;
-
-        cli_out ("Mempool Stats\n-------------");
-        cli_out ("%-30s %9s %9s %12s %10s %8s %8s %12s", "Name", "HotCount",
-                 "ColdCount", "PaddedSizeof", "AllocCount", "MaxAlloc",
-                 "Misses", "Max-StdAlloc");
-        cli_out ("%-30s %9s %9s %12s %10s %8s %8s %12s", "----", "--------",
-                 "---------", "------------", "----------",
-                 "--------", "--------", "------------");
-
-        for (i = 0; i < mempool_count; i++) {
-                memset (key, 0, sizeof (key));
-                snprintf (key, sizeof (key), "%s.pool%d.name", prefix, i);
-                ret = dict_get_str (dict, key, &name);
-                if (ret)
-                        goto out;
-
-                memset (key, 0, sizeof (key));
-                snprintf (key, sizeof (key), "%s.pool%d.hotcount", prefix, i);
-                ret = dict_get_int32 (dict, key, &hotcount);
-                if (ret)
-                        goto out;
-
-                memset (key, 0, sizeof (key));
-                snprintf (key, sizeof (key), "%s.pool%d.coldcount", prefix, i);
-                ret = dict_get_int32 (dict, key, &coldcount);
-                if (ret)
-                        goto out;
-
-                memset (key, 0, sizeof (key));
-                snprintf (key, sizeof (key), "%s.pool%d.paddedsizeof",
-                          prefix, i);
-                ret = dict_get_uint64 (dict, key, &paddedsizeof);
-                if (ret)
-                        goto out;
-
-                memset (key, 0, sizeof (key));
-                snprintf (key, sizeof (key), "%s.pool%d.alloccount", prefix, i);
-                ret = dict_get_uint64 (dict, key, &alloccount);
-                if (ret)
-                        goto out;
-
-                memset (key, 0, sizeof (key));
-                snprintf (key, sizeof (key), "%s.pool%d.max_alloc", prefix, i);
-                ret = dict_get_int32 (dict, key, &maxalloc);
-                if (ret)
-                        goto out;
+static int32_t
+gf_cli_getwd(call_frame_t *frame, xlator_t *this, void *data)
+{
+    int ret = -1;
+    gf1_cli_getwd_req req = {
+        0,
+    };
 
-                memset (key, 0, sizeof (key));
-                snprintf (key, sizeof (key), "%s.pool%d.max-stdalloc", prefix, i);
-                ret = dict_get_int32 (dict, key, &maxstdalloc);
-                if (ret)
-                        goto out;
+    GF_ASSERT(frame);
+    GF_ASSERT(this);
 
-                memset (key, 0, sizeof (key));
-                snprintf (key, sizeof (key), "%s.pool%d.pool-misses", prefix, i);
-                ret = dict_get_uint64 (dict, key, &pool_misses);
-                if (ret)
-                        goto out;
+    if (!frame || !this)
+        goto out;
 
-                cli_out ("%-30s %9d %9d %12"PRIu64" %10"PRIu64" %8d %8"PRIu64
-                         " %12d", name, hotcount, coldcount, paddedsizeof,
-                         alloccount, maxalloc, pool_misses, maxstdalloc);
-        }
+    ret = cli_cmd_submit(NULL, &req, frame, cli_rpc_prog, GLUSTER_CLI_GETWD,
+                         NULL, this, gf_cli_getwd_cbk,
+                         (xdrproc_t)xdr_gf1_cli_getwd_req);
 
 out:
-        return;
+    gf_log("cli", GF_LOG_DEBUG, RETURNING, ret);
 
+    return ret;
 }
 
-void
-cli_print_volume_status_mem (dict_t *dict, gf_boolean_t notbrick)
-{
-        int             ret = -1;
-        char            *volname = NULL;
-        char            *hostname = NULL;
-        char            *path = NULL;
-        int             online = -1;
-        char            key[1024] = {0,};
-        int             brick_index_max = -1;
-        int             other_count = 0;
-        int             index_max = 0;
-        int             val = 0;
-        int             i = 0;
-
-        GF_ASSERT (dict);
-
-        ret = dict_get_str (dict, "volname", &volname);
+static void
+cli_print_volume_status_mempool(dict_t *dict, char *prefix)
+{
+    int ret = -1;
+    int32_t mempool_count = 0;
+    char *name = NULL;
+    int32_t hotcount = 0;
+    int32_t coldcount = 0;
+    uint64_t paddedsizeof = 0;
+    uint64_t alloccount = 0;
+    int32_t maxalloc = 0;
+    uint64_t pool_misses = 0;
+    int32_t maxstdalloc = 0;
+    char key[128] = {
+        /* prefix is really small 'brick%d' really */
+        0,
+    };
+    int keylen;
+    int i = 0;
+
+    GF_ASSERT(dict);
+    GF_ASSERT(prefix);
+
+    keylen = snprintf(key, sizeof(key), "%s.mempool-count", prefix);
+    ret = dict_get_int32n(dict, key, keylen, &mempool_count);
+    if (ret)
+        goto out;
+
+    cli_out("Mempool Stats\n-------------");
+    cli_out("%-30s %9s %9s %12s %10s %8s %8s %12s", "Name", "HotCount",
+            "ColdCount", "PaddedSizeof", "AllocCount", "MaxAlloc", "Misses",
+            "Max-StdAlloc");
+    cli_out("%-30s %9s %9s %12s %10s %8s %8s %12s", "----", "--------",
+            "---------", "------------", "----------", "--------", "--------",
+            "------------");
+
+    for (i = 0; i < mempool_count; i++) {
+        keylen = snprintf(key, sizeof(key), "%s.pool%d.name", prefix, i);
+        ret = dict_get_strn(dict, key, keylen, &name);
         if (ret)
-                goto out;
-        cli_out ("Memory status for volume : %s", volname);
+            goto out;
 
-        ret = dict_get_int32 (dict, "brick-index-max", &brick_index_max);
+        keylen = snprintf(key, sizeof(key), "%s.pool%d.hotcount", prefix, i);
+        ret = dict_get_int32n(dict, key, keylen, &hotcount);
         if (ret)
-                goto out;
-        ret = dict_get_int32 (dict, "other-count", &other_count);
-        if (ret)
-                goto out;
-
-        index_max = brick_index_max + other_count;
+            goto out;
 
-        for (i = 0; i <= index_max; i++) {
-                cli_out ("----------------------------------------------");
-
-                memset (key, 0, sizeof (key));
-                snprintf (key, sizeof (key), "brick%d.hostname", i);
-                ret = dict_get_str (dict, key, &hostname);
-                if (ret)
-                        continue;
-                memset (key, 0, sizeof (key));
-                snprintf (key, sizeof (key), "brick%d.path", i);
-                ret = dict_get_str (dict, key, &path);
-                if (ret)
-                        continue;
-                if (notbrick)
-                        cli_out ("%s : %s", hostname, path);
-                else
-                        cli_out ("Brick : %s:%s", hostname, path);
-
-                memset (key, 0, sizeof (key));
-                snprintf (key, sizeof (key), "brick%d.status", i);
-                ret = dict_get_int32 (dict, key, &online);
-                if (ret)
-                        goto out;
-                if (!online) {
-                        if (notbrick)
-                                cli_out ("%s is offline", hostname);
-                        else
-                                cli_out ("Brick is offline");
-                        continue;
-                }
-
-                cli_out ("Mallinfo\n--------");
+        keylen = snprintf(key, sizeof(key), "%s.pool%d.coldcount", prefix, i);
+        ret = dict_get_int32n(dict, key, keylen, &coldcount);
+        if (ret)
+            goto out;
 
-                memset (key, 0, sizeof (key));
-                snprintf (key, sizeof (key), "brick%d.mallinfo.arena", i);
-                ret = dict_get_int32 (dict, key, &val);
-                if (ret)
-                        goto out;
-                cli_out ("%-8s : %d","Arena", val);
-
-                memset (key, 0, sizeof (key));
-                snprintf (key, sizeof (key), "brick%d.mallinfo.ordblks", i);
-                ret = dict_get_int32 (dict, key, &val);
-                if(ret)
-                        goto out;
-                cli_out ("%-8s : %d","Ordblks", val);
-
-                memset (key, 0, sizeof (key));
-                snprintf (key, sizeof (key), "brick%d.mallinfo.smblks", i);
-                ret = dict_get_int32 (dict, key, &val);
-                if(ret)
-                        goto out;
-                cli_out ("%-8s : %d","Smblks", val);
-
-                memset (key, 0, sizeof (key));
-                snprintf (key, sizeof (key), "brick%d.mallinfo.hblks", i);
-                ret = dict_get_int32 (dict, key, &val);
-                if(ret)
-                        goto out;
-                cli_out ("%-8s : %d", "Hblks", val);
-
-                memset (key, 0, sizeof (key));
-                snprintf (key, sizeof (key), "brick%d.mallinfo.hblkhd", i);
-                ret = dict_get_int32 (dict, key, &val);
-                if (ret)
-                        goto out;
-                cli_out ("%-8s : %d", "Hblkhd", val);
+        snprintf(key, sizeof(key), "%s.pool%d.paddedsizeof", prefix, i);
+        ret = dict_get_uint64(dict, key, &paddedsizeof);
+        if (ret)
+            goto out;
 
-                memset (key, 0, sizeof (key));
-                snprintf (key, sizeof (key), "brick%d.mallinfo.usmblks", i);
-                ret = dict_get_int32 (dict, key, &val);
-                if (ret)
-                        goto out;
-                cli_out ("%-8s : %d", "Usmblks", val);
+        snprintf(key, sizeof(key), "%s.pool%d.alloccount", prefix, i);
+        ret = dict_get_uint64(dict, key, &alloccount);
+        if (ret)
+            goto out;
 
-                memset (key, 0, sizeof (key));
-                snprintf (key, sizeof (key), "brick%d.mallinfo.fsmblks", i);
-                ret = dict_get_int32 (dict, key, &val);
-                if (ret)
-                        goto out;
-                cli_out ("%-8s : %d", "Fsmblks", val);
+        keylen = snprintf(key, sizeof(key), "%s.pool%d.max_alloc", prefix, i);
+        ret = dict_get_int32n(dict, key, keylen, &maxalloc);
+        if (ret)
+            goto out;
 
-                memset (key, 0, sizeof (key));
-                snprintf (key, sizeof (key), "brick%d.mallinfo.uordblks", i);
-                ret = dict_get_int32 (dict, key, &val);
-                if (ret)
-                        goto out;
-                cli_out ("%-8s : %d", "Uordblks", val);
+        keylen = snprintf(key, sizeof(key), "%s.pool%d.max-stdalloc", prefix,
+                          i);
+        ret = dict_get_int32n(dict, key, keylen, &maxstdalloc);
+        if (ret)
+            goto out;
 
-                memset (key, 0, sizeof (key));
-                snprintf (key, sizeof (key), "brick%d.mallinfo.fordblks", i);
-                ret = dict_get_int32 (dict, key, &val);
-                if (ret)
-                        goto out;
-                cli_out ("%-8s : %d", "Fordblks", val);
+        snprintf(key, sizeof(key), "%s.pool%d.pool-misses", prefix, i);
+        ret = dict_get_uint64(dict, key, &pool_misses);
+        if (ret)
+            goto out;
 
-                memset (key, 0, sizeof (key));
-                snprintf (key, sizeof (key), "brick%d.mallinfo.keepcost", i);
-                ret = dict_get_int32 (dict, key, &val);
-                if (ret)
-                        goto out;
-                cli_out ("%-8s : %d", "Keepcost", val);
+        cli_out("%-30s %9d %9d %12" PRIu64 " %10" PRIu64 " %8d %8" PRIu64
+                " %12d",
+                name, hotcount, coldcount, paddedsizeof, alloccount, maxalloc,
+                pool_misses, maxstdalloc);
+    }
 
-                cli_out (" ");
-                memset (key, 0, sizeof (key));
-                snprintf (key, sizeof (key), "brick%d", i);
-                cli_print_volume_status_mempool (dict, key);
-        }
 out:
-        cli_out ("----------------------------------------------\n");
-        return;
+    return;
 }
 
-void
-cli_print_volume_status_clients (dict_t *dict, gf_boolean_t notbrick)
-{
-        int             ret = -1;
-        char            *volname = NULL;
-        int             brick_index_max = -1;
-        int             other_count = 0;
-        int             index_max = 0;
-        char            *hostname = NULL;
-        char            *path = NULL;
-        int             online = -1;
-        int             client_count = 0;
-        char            *clientname = NULL;
-        uint64_t        bytesread = 0;
-        uint64_t        byteswrite = 0;
-        char            key[1024] = {0,};
-        int             i = 0;
-        int             j = 0;
-
-        GF_ASSERT (dict);
-
-        ret = dict_get_str (dict, "volname", &volname);
+static void
+cli_print_volume_status_mem(dict_t *dict, gf_boolean_t notbrick)
+{
+    int ret = -1;
+    char *volname = NULL;
+    char *hostname = NULL;
+    char *path = NULL;
+    int online = -1;
+    char key[64] = {
+        0,
+    };
+    int brick_index_max = -1;
+    int other_count = 0;
+    int index_max = 0;
+    int val = 0;
+    int i = 0;
+
+    GF_ASSERT(dict);
+
+    ret = dict_get_str_sizen(dict, "volname", &volname);
+    if (ret)
+        goto out;
+    cli_out("Memory status for volume : %s", volname);
+
+    ret = dict_get_int32_sizen(dict, "brick-index-max", &brick_index_max);
+    if (ret)
+        goto out;
+    ret = dict_get_int32_sizen(dict, "other-count", &other_count);
+    if (ret)
+        goto out;
+
+    index_max = brick_index_max + other_count;
+
+    for (i = 0; i <= index_max; i++) {
+        cli_out("----------------------------------------------");
+
+        snprintf(key, sizeof(key), "brick%d.hostname", i);
+        ret = dict_get_str(dict, key, &hostname);
         if (ret)
-                goto out;
-        cli_out ("Client connections for volume %s", volname);
-
-        ret = dict_get_int32 (dict, "brick-index-max", &brick_index_max);
+            continue;
+        snprintf(key, sizeof(key), "brick%d.path", i);
+        ret = dict_get_str(dict, key, &path);
         if (ret)
-                goto out;
-        ret = dict_get_int32 (dict, "other-count", &other_count);
-        if (ret)
-                goto out;
+            continue;
+        if (notbrick)
+            cli_out("%s : %s", hostname, path);
+        else
+            cli_out("Brick : %s:%s", hostname, path);
 
-        index_max = brick_index_max + other_count;
+        snprintf(key, sizeof(key), "brick%d.status", i);
+        ret = dict_get_int32(dict, key, &online);
+        if (ret)
+            goto out;
+        if (!online) {
+            if (notbrick)
+                cli_out("%s is offline", hostname);
+            else
+                cli_out("Brick is offline");
+            continue;
+        }
 
-        for (i = 0; i <= index_max; i++) {
-                cli_out ("----------------------------------------------");
+        cli_out("Mallinfo\n--------");
 
-                memset (key, 0, sizeof (key));
-                snprintf (key, sizeof (key), "brick%d.hostname", i);
-                ret = dict_get_str (dict, key, &hostname);
-                if (ret)
-                        goto out;
-                memset (key, 0, sizeof (key));
-                snprintf (key, sizeof (key), "brick%d.path", i);
-                ret = dict_get_str (dict, key, &path);
-                if (ret)
-                        goto out;
+        snprintf(key, sizeof(key), "brick%d.mallinfo.arena", i);
+        ret = dict_get_int32(dict, key, &val);
+        if (ret)
+            goto out;
+        cli_out("%-8s : %d", "Arena", val);
 
-                if (notbrick)
-                        cli_out ("%s : %s", hostname, path);
-                else
-                        cli_out ("Brick : %s:%s", hostname, path);
+        snprintf(key, sizeof(key), "brick%d.mallinfo.ordblks", i);
+        ret = dict_get_int32(dict, key, &val);
+        if (ret)
+            goto out;
+        cli_out("%-8s : %d", "Ordblks", val);
 
-                memset (key, 0, sizeof (key));
-                snprintf (key, sizeof (key), "brick%d.status", i);
-                ret = dict_get_int32 (dict, key, &online);
-                if (ret)
-                        goto out;
-                if (!online) {
-                        if (notbrick)
-                                cli_out ("%s is offline", hostname);
-                        else
-                                cli_out ("Brick is offline");
-                        continue;
-                }
+        snprintf(key, sizeof(key), "brick%d.mallinfo.smblks", i);
+        ret = dict_get_int32(dict, key, &val);
+        if (ret)
+            goto out;
+        cli_out("%-8s : %d", "Smblks", val);
 
-                memset (key, 0, sizeof (key));
-                snprintf (key, sizeof (key), "brick%d.clientcount", i);
-                ret = dict_get_int32 (dict, key, &client_count);
-                if (ret)
-                        goto out;
-
-                cli_out ("Clients connected : %d", client_count);
-                if (client_count == 0)
-                        continue;
-
-                cli_out ("%-48s %15s %15s", "Hostname", "BytesRead",
-                         "BytesWritten");
-                cli_out ("%-48s %15s %15s", "--------", "---------",
-                         "------------");
-                for (j =0; j < client_count; j++) {
-                        memset (key, 0, sizeof (key));
-                        snprintf (key, sizeof (key),
-                                  "brick%d.client%d.hostname", i, j);
-                        ret = dict_get_str (dict, key, &clientname);
-                        if (ret)
-                                goto out;
-
-                        memset (key, 0, sizeof (key));
-                        snprintf (key, sizeof (key),
-                                  "brick%d.client%d.bytesread", i, j);
-                        ret = dict_get_uint64 (dict, key, &bytesread);
-                        if (ret)
-                                goto out;
-
-                        memset (key, 0, sizeof (key));
-                        snprintf (key, sizeof (key),
-                                 "brick%d.client%d.byteswrite", i, j);
-                        ret = dict_get_uint64 (dict, key, &byteswrite);
-                        if (ret)
-                                goto out;
-
-                        cli_out ("%-48s %15"PRIu64" %15"PRIu64,
-                                 clientname, bytesread, byteswrite);
-                }
-        }
-out:
-        cli_out ("----------------------------------------------\n");
-        return;
-}
+        snprintf(key, sizeof(key), "brick%d.mallinfo.hblks", i);
+        ret = dict_get_int32(dict, key, &val);
+        if (ret)
+            goto out;
+        cli_out("%-8s : %d", "Hblks", val);
 
-void
-cli_print_volume_status_inode_entry (dict_t *dict, char *prefix)
-{
-        int             ret = -1;
-        char            key[1024] = {0,};
-        char            *gfid = NULL;
-        uint64_t        nlookup = 0;
-        uint32_t        ref = 0;
-        int             ia_type = 0;
-        char            inode_type;
+        snprintf(key, sizeof(key), "brick%d.mallinfo.hblkhd", i);
+        ret = dict_get_int32(dict, key, &val);
+        if (ret)
+            goto out;
+        cli_out("%-8s : %d", "Hblkhd", val);
 
-        GF_ASSERT (dict);
-        GF_ASSERT (prefix);
+        snprintf(key, sizeof(key), "brick%d.mallinfo.usmblks", i);
+        ret = dict_get_int32(dict, key, &val);
+        if (ret)
+            goto out;
+        cli_out("%-8s : %d", "Usmblks", val);
 
-        memset (key, 0, sizeof (key));
-        snprintf (key, sizeof (key), "%s.gfid", prefix);
-        ret = dict_get_str (dict, key, &gfid);
+        snprintf(key, sizeof(key), "brick%d.mallinfo.fsmblks", i);
+        ret = dict_get_int32(dict, key, &val);
         if (ret)
-                goto out;
+            goto out;
+        cli_out("%-8s : %d", "Fsmblks", val);
 
-        memset (key, 0, sizeof (key));
-        snprintf (key, sizeof (key), "%s.nlookup", prefix);
-        ret = dict_get_uint64 (dict, key, &nlookup);
+        snprintf(key, sizeof(key), "brick%d.mallinfo.uordblks", i);
+        ret = dict_get_int32(dict, key, &val);
         if (ret)
-                goto out;
+            goto out;
+        cli_out("%-8s : %d", "Uordblks", val);
 
-        memset (key, 0, sizeof (key));
-        snprintf (key, sizeof (key), "%s.ref", prefix);
-        ret = dict_get_uint32 (dict, key, &ref);
+        snprintf(key, sizeof(key), "brick%d.mallinfo.fordblks", i);
+        ret = dict_get_int32(dict, key, &val);
         if (ret)
-                goto out;
+            goto out;
+        cli_out("%-8s : %d", "Fordblks", val);
 
-        memset (key, 0, sizeof (key));
-        snprintf (key, sizeof (key), "%s.ia_type", prefix);
-        ret = dict_get_int32 (dict, key, &ia_type);
+        snprintf(key, sizeof(key), "brick%d.mallinfo.keepcost", i);
+        ret = dict_get_int32(dict, key, &val);
         if (ret)
-                goto out;
+            goto out;
+        cli_out("%-8s : %d", "Keepcost", val);
+
+        cli_out(" ");
+        snprintf(key, sizeof(key), "brick%d", i);
+        cli_print_volume_status_mempool(dict, key);
+    }
+out:
+    cli_out("----------------------------------------------\n");
+    return;
+}
+
+static void
+cli_print_volume_status_client_list(dict_t *dict, gf_boolean_t notbrick)
+{
+    int ret = -1;
+    char *volname = NULL;
+    int client_count = 0;
+    int current_count = 0;
+    char key[64] = {
+        0,
+    };
+    int i = 0;
+    int total = 0;
+    char *name = NULL;
+    gf_boolean_t is_fuse_done = _gf_false;
+    gf_boolean_t is_gfapi_done = _gf_false;
+    gf_boolean_t is_rebalance_done = _gf_false;
+    gf_boolean_t is_glustershd_done = _gf_false;
+    gf_boolean_t is_quotad_done = _gf_false;
+    gf_boolean_t is_snapd_done = _gf_false;
+
+    GF_ASSERT(dict);
+
+    ret = dict_get_str_sizen(dict, "volname", &volname);
+    if (ret)
+        goto out;
+    cli_out("Client connections for volume %s", volname);
+
+    ret = dict_get_int32_sizen(dict, "client-count", &client_count);
+    if (ret)
+        goto out;
+
+    cli_out("%-48s %15s", "Name", "count");
+    cli_out("%-48s %15s", "-----", "------");
+    for (i = 0; i < client_count; i++) {
+        name = NULL;
+        snprintf(key, sizeof(key), "client%d.name", i);
+        ret = dict_get_str(dict, key, &name);
+
+        if (!strncmp(name, "fuse", 4)) {
+            if (!is_fuse_done) {
+                is_fuse_done = _gf_true;
+                ret = dict_get_int32_sizen(dict, "fuse-count", &current_count);
+                if (ret)
+                    goto out;
+                total = total + current_count;
+                goto print;
+            }
+            continue;
+        } else if (!strncmp(name, "gfapi", 5)) {
+            if (!is_gfapi_done) {
+                is_gfapi_done = _gf_true;
+                ret = dict_get_int32_sizen(dict, "gfapi-count", &current_count);
+                if (ret)
+                    goto out;
+                total = total + current_count;
+                goto print;
+            }
+            continue;
+        } else if (!strcmp(name, "rebalance")) {
+            if (!is_rebalance_done) {
+                is_rebalance_done = _gf_true;
+                ret = dict_get_int32_sizen(dict, "rebalance-count",
+                                           &current_count);
+                if (ret)
+                    goto out;
+                total = total + current_count;
+                goto print;
+            }
+            continue;
+        } else if (!strcmp(name, "glustershd")) {
+            if (!is_glustershd_done) {
+                is_glustershd_done = _gf_true;
+                ret = dict_get_int32_sizen(dict, "glustershd-count",
+                                           &current_count);
+                if (ret)
+                    goto out;
+                total = total + current_count;
+                goto print;
+            }
+            continue;
+        } else if (!strcmp(name, "quotad")) {
+            if (!is_quotad_done) {
+                is_quotad_done = _gf_true;
+                ret = dict_get_int32_sizen(dict, "quotad-count",
+                                           &current_count);
+                if (ret)
+                    goto out;
+                total = total + current_count;
+                goto print;
+            }
+            continue;
+        } else if (!strcmp(name, "snapd")) {
+            if (!is_snapd_done) {
+                is_snapd_done = _gf_true;
+                ret = dict_get_int32_sizen(dict, "snapd-count", &current_count);
+                if (ret)
+                    goto out;
+                total = total + current_count;
+                goto print;
+            }
+            continue;
+        }
+
+    print:
+        cli_out("%-48s %15d", name, current_count);
+    }
+out:
+    cli_out("\ntotal clients for volume %s : %d ", volname, total);
+    cli_out(
+        "-----------------------------------------------------------------\n");
+    return;
+}
 
-        switch (ia_type) {
+static void
+cli_print_volume_status_clients(dict_t *dict, gf_boolean_t notbrick)
+{
+    int ret = -1;
+    char *volname = NULL;
+    int brick_index_max = -1;
+    int other_count = 0;
+    int index_max = 0;
+    char *hostname = NULL;
+    char *path = NULL;
+    int online = -1;
+    int client_count = 0;
+    char *clientname = NULL;
+    uint64_t bytesread = 0;
+    uint64_t byteswrite = 0;
+    uint32_t opversion = 0;
+    char key[128] = {
+        0,
+    };
+    int i = 0;
+    int j = 0;
+
+    GF_ASSERT(dict);
+
+    ret = dict_get_str_sizen(dict, "volname", &volname);
+    if (ret)
+        goto out;
+    cli_out("Client connections for volume %s", volname);
+
+    ret = dict_get_int32_sizen(dict, "brick-index-max", &brick_index_max);
+    if (ret)
+        goto out;
+    ret = dict_get_int32_sizen(dict, "other-count", &other_count);
+    if (ret)
+        goto out;
+
+    index_max = brick_index_max + other_count;
+
+    for (i = 0; i <= index_max; i++) {
+        hostname = NULL;
+        path = NULL;
+        online = -1;
+        client_count = 0;
+        clientname = NULL;
+        bytesread = 0;
+        byteswrite = 0;
+
+        cli_out("----------------------------------------------");
+
+        snprintf(key, sizeof(key), "brick%d.hostname", i);
+        ret = dict_get_str(dict, key, &hostname);
+
+        snprintf(key, sizeof(key), "brick%d.path", i);
+        ret = dict_get_str(dict, key, &path);
+
+        if (hostname && path) {
+            if (notbrick)
+                cli_out("%s : %s", hostname, path);
+            else
+                cli_out("Brick : %s:%s", hostname, path);
+        }
+        snprintf(key, sizeof(key), "brick%d.status", i);
+        ret = dict_get_int32(dict, key, &online);
+        if (!online) {
+            if (notbrick)
+                cli_out("%s is offline", hostname);
+            else
+                cli_out("Brick is offline");
+            continue;
+        }
+
+        snprintf(key, sizeof(key), "brick%d.clientcount", i);
+        ret = dict_get_int32(dict, key, &client_count);
+
+        if (hostname && path)
+            cli_out("Clients connected : %d", client_count);
+        if (client_count == 0)
+            continue;
+
+        cli_out("%-48s %15s %15s %15s", "Hostname", "BytesRead", "BytesWritten",
+                "OpVersion");
+        cli_out("%-48s %15s %15s %15s", "--------", "---------", "------------",
+                "---------");
+        for (j = 0; j < client_count; j++) {
+            snprintf(key, sizeof(key), "brick%d.client%d.hostname", i, j);
+            ret = dict_get_str(dict, key, &clientname);
+
+            snprintf(key, sizeof(key), "brick%d.client%d.bytesread", i, j);
+            ret = dict_get_uint64(dict, key, &bytesread);
+
+            snprintf(key, sizeof(key), "brick%d.client%d.byteswrite", i, j);
+            ret = dict_get_uint64(dict, key, &byteswrite);
+
+            snprintf(key, sizeof(key), "brick%d.client%d.opversion", i, j);
+            ret = dict_get_uint32(dict, key, &opversion);
+
+            cli_out("%-48s %15" PRIu64 " %15" PRIu64 " %15" PRIu32, clientname,
+                    bytesread, byteswrite, opversion);
+        }
+    }
+out:
+    cli_out("----------------------------------------------\n");
+    return;
+}
+
+#ifdef DEBUG /* this function is only used in debug */
+static void
+cli_print_volume_status_inode_entry(dict_t *dict, char *prefix)
+{
+    int ret = -1;
+    char key[1024] = {
+        0,
+    };
+    char *gfid = NULL;
+    uint64_t nlookup = 0;
+    uint32_t ref = 0;
+    int ia_type = 0;
+    char inode_type;
+
+    GF_ASSERT(dict);
+    GF_ASSERT(prefix);
+
+    snprintf(key, sizeof(key), "%s.gfid", prefix);
+    ret = dict_get_str(dict, key, &gfid);
+    if (ret)
+        goto out;
+
+    snprintf(key, sizeof(key), "%s.nlookup", prefix);
+    ret = dict_get_uint64(dict, key, &nlookup);
+    if (ret)
+        goto out;
+
+    snprintf(key, sizeof(key), "%s.ref", prefix);
+    ret = dict_get_uint32(dict, key, &ref);
+    if (ret)
+        goto out;
+
+    snprintf(key, sizeof(key), "%s.ia_type", prefix);
+    ret = dict_get_int32(dict, key, &ia_type);
+    if (ret)
+        goto out;
+
+    switch (ia_type) {
         case IA_IFREG:
-                inode_type = 'R';
-                break;
+            inode_type = 'R';
+            break;
         case IA_IFDIR:
-                inode_type = 'D';
-                break;
+            inode_type = 'D';
+            break;
         case IA_IFLNK:
-                inode_type = 'L';
-                break;
+            inode_type = 'L';
+            break;
         case IA_IFBLK:
-                inode_type = 'B';
-                break;
+            inode_type = 'B';
+            break;
         case IA_IFCHR:
-                inode_type = 'C';
-                break;
+            inode_type = 'C';
+            break;
         case IA_IFIFO:
-                inode_type = 'F';
-                break;
+            inode_type = 'F';
+            break;
         case IA_IFSOCK:
-                inode_type = 'S';
-                break;
+            inode_type = 'S';
+            break;
         default:
-                inode_type = 'I';
-                break;
-        }
+            inode_type = 'I';
+            break;
+    }
 
-        cli_out ("%-40s %14"PRIu64" %14"PRIu32" %9c",
-                 gfid, nlookup, ref, inode_type);
+    cli_out("%-40s %14" PRIu64 " %14" PRIu32 " %9c", gfid, nlookup, ref,
+            inode_type);
 
 out:
-        return;
-
+    return;
 }
+#endif
 
-void
-cli_print_volume_status_itables (dict_t *dict, char *prefix)
-{
-        int             ret = -1;
-        char            key[1024] = {0,};
-        uint32_t        active_size = 0;
-        uint32_t        lru_size = 0;
-        uint32_t        purge_size = 0;
-        int             i =0;
-
-        GF_ASSERT (dict);
-        GF_ASSERT (prefix);
+static void
+cli_print_volume_status_itables(dict_t *dict, char *prefix)
+{
+    int ret = -1;
+    char key[1024] = {
+        0,
+    };
+    uint32_t active_size = 0;
+    uint32_t lru_size = 0;
+    uint32_t purge_size = 0;
+    uint32_t lru_limit = 0;
+#ifdef DEBUG
+    int i = 0;
+#endif
+    GF_ASSERT(dict);
+    GF_ASSERT(prefix);
+
+    snprintf(key, sizeof(key), "%s.lru_limit", prefix);
+    ret = dict_get_uint32(dict, key, &lru_limit);
+    if (ret)
+        goto out;
+    cli_out("LRU limit     : %u", lru_limit);
+
+    snprintf(key, sizeof(key), "%s.active_size", prefix);
+    ret = dict_get_uint32(dict, key, &active_size);
+    if (ret)
+        goto out;
+
+#ifdef DEBUG
+    if (active_size != 0) {
+        cli_out("Active inodes:");
+        cli_out("%-40s %14s %14s %9s", "GFID", "Lookups", "Ref", "IA type");
+        cli_out("%-40s %14s %14s %9s", "----", "-------", "---", "-------");
+    }
+    for (i = 0; i < active_size; i++) {
+        snprintf(key, sizeof(key), "%s.active%d", prefix, i);
+        cli_print_volume_status_inode_entry(dict, key);
+    }
+    cli_out(" ");
+#else
+    cli_out("Active Inodes : %u", active_size);
 
-        memset (key, 0, sizeof (key));
-        snprintf (key, sizeof (key), "%s.active_size", prefix);
-        ret = dict_get_uint32 (dict, key, &active_size);
-        if (ret)
-                goto out;
-        if (active_size != 0) {
-                cli_out ("Active inodes:");
-                cli_out ("%-40s %14s %14s %9s", "GFID", "Lookups", "Ref",
-                         "IA type");
-                cli_out ("%-40s %14s %14s %9s", "----", "-------", "---",
-                         "-------");
-        }
-        for (i = 0; i < active_size; i++) {
-                memset (key, 0, sizeof (key));
-                snprintf (key, sizeof (key), "%s.active%d", prefix, i);
-                cli_print_volume_status_inode_entry (dict, key);
-        }
-        cli_out (" ");
+#endif
 
-        memset (key, 0, sizeof (key));
-        snprintf (key, sizeof (key), "%s.lru_size", prefix);
-        ret = dict_get_uint32 (dict, key, &lru_size);
-        if (ret)
-                goto out;
-        if (lru_size != 0) {
-                cli_out ("LRU inodes:");
-                cli_out ("%-40s %14s %14s %9s", "GFID", "Lookups", "Ref",
-                         "IA type");
-                cli_out ("%-40s %14s %14s %9s", "----", "-------", "---",
-                         "-------");
-        }
-        for (i = 0; i < lru_size; i++) {
-                memset (key, 0, sizeof (key));
-                snprintf (key, sizeof (key), "%s.lru%d", prefix, i);
-                cli_print_volume_status_inode_entry (dict, key);
-        }
-        cli_out (" ");
+    snprintf(key, sizeof(key), "%s.lru_size", prefix);
+    ret = dict_get_uint32(dict, key, &lru_size);
+    if (ret)
+        goto out;
+
+#ifdef DEBUG
+    if (lru_size != 0) {
+        cli_out("LRU inodes:");
+        cli_out("%-40s %14s %14s %9s", "GFID", "Lookups", "Ref", "IA type");
+        cli_out("%-40s %14s %14s %9s", "----", "-------", "---", "-------");
+    }
+    for (i = 0; i < lru_size; i++) {
+        snprintf(key, sizeof(key), "%s.lru%d", prefix, i);
+        cli_print_volume_status_inode_entry(dict, key);
+    }
+    cli_out(" ");
+#else
+    cli_out("LRU Inodes    : %u", lru_size);
+#endif
 
-        memset (key, 0, sizeof (key));
-        snprintf (key, sizeof (key), "%s.purge_size", prefix);
-        ret = dict_get_uint32 (dict, key, &purge_size);
-        if (ret)
-                goto out;
-        if (purge_size != 0) {
-                cli_out ("Purged inodes:");
-                cli_out ("%-40s %14s %14s %9s", "GFID", "Lookups", "Ref",
-                         "IA type");
-                cli_out ("%-40s %14s %14s %9s", "----", "-------", "---",
-                         "-------");
-        }
-        for (i = 0; i < purge_size; i++) {
-                memset (key, 0, sizeof (key));
-                snprintf (key, sizeof (key), "%s.purge%d", prefix, i);
-                cli_print_volume_status_inode_entry (dict, key);
-        }
+    snprintf(key, sizeof(key), "%s.purge_size", prefix);
+    ret = dict_get_uint32(dict, key, &purge_size);
+    if (ret)
+        goto out;
+#ifdef DEBUG
+    if (purge_size != 0) {
+        cli_out("Purged inodes:");
+        cli_out("%-40s %14s %14s %9s", "GFID", "Lookups", "Ref", "IA type");
+        cli_out("%-40s %14s %14s %9s", "----", "-------", "---", "-------");
+    }
+    for (i = 0; i < purge_size; i++) {
+        snprintf(key, sizeof(key), "%s.purge%d", prefix, i);
+        cli_print_volume_status_inode_entry(dict, key);
+    }
+#else
+    cli_out("Purge Inodes  : %u", purge_size);
+#endif
 
 out:
-        return;
+    return;
 }
 
-void
-cli_print_volume_status_inode (dict_t *dict, gf_boolean_t notbrick)
-{
-        int             ret = -1;
-        char            *volname = NULL;
-        int             brick_index_max = -1;
-        int             other_count = 0;
-        int             index_max = 0;
-        char            *hostname = NULL;
-        char            *path = NULL;
-        int             online = -1;
-        int             conn_count = 0;
-        char            key[1024] = {0,};
-        int             i = 0;
-        int             j = 0;
-
-        GF_ASSERT (dict);
-
-        ret = dict_get_str (dict, "volname", &volname);
-        if (ret)
-                goto out;
-        cli_out ("Inode tables for volume %s", volname);
-
-        ret = dict_get_int32 (dict, "brick-index-max", &brick_index_max);
+static void
+cli_print_volume_status_inode(dict_t *dict, gf_boolean_t notbrick)
+{
+    int ret = -1;
+    char *volname = NULL;
+    int brick_index_max = -1;
+    int other_count = 0;
+    int index_max = 0;
+    char *hostname = NULL;
+    char *path = NULL;
+    int online = -1;
+    int conn_count = 0;
+    char key[64] = {
+        0,
+    };
+    int i = 0;
+    int j = 0;
+
+    GF_ASSERT(dict);
+
+    ret = dict_get_str_sizen(dict, "volname", &volname);
+    if (ret)
+        goto out;
+    cli_out("Inode tables for volume %s", volname);
+
+    ret = dict_get_int32_sizen(dict, "brick-index-max", &brick_index_max);
+    if (ret)
+        goto out;
+    ret = dict_get_int32_sizen(dict, "other-count", &other_count);
+    if (ret)
+        goto out;
+
+    index_max = brick_index_max + other_count;
+
+    for (i = 0; i <= index_max; i++) {
+        cli_out("----------------------------------------------");
+
+        snprintf(key, sizeof(key), "brick%d.hostname", i);
+        ret = dict_get_str(dict, key, &hostname);
         if (ret)
-                goto out;
-        ret = dict_get_int32 (dict, "other-count", &other_count);
+            goto out;
+        snprintf(key, sizeof(key), "brick%d.path", i);
+        ret = dict_get_str(dict, key, &path);
         if (ret)
-                goto out;
-
-        index_max = brick_index_max + other_count;
-
-        for ( i = 0; i <= index_max; i++) {
-                cli_out ("----------------------------------------------");
-
-                memset (key, 0, sizeof (key));
-                snprintf (key, sizeof (key), "brick%d.hostname", i);
-                ret = dict_get_str (dict, key, &hostname);
-                if (ret)
-                        goto out;
-                memset (key, 0, sizeof (key));
-                snprintf (key, sizeof (key), "brick%d.path", i);
-                ret = dict_get_str (dict, key, &path);
-                if (ret)
-                        goto out;
-                if (notbrick)
-                        cli_out ("%s : %s", hostname, path);
-                else
-                        cli_out ("Brick : %s:%s", hostname, path);
+            goto out;
+        if (notbrick)
+            cli_out("%s : %s", hostname, path);
+        else
+            cli_out("Brick : %s:%s", hostname, path);
 
-                memset (key, 0, sizeof (key));
-                snprintf (key, sizeof (key), "brick%d.status", i);
-                ret = dict_get_int32 (dict, key, &online);
-                if (ret)
-                        goto out;
-                if (!online) {
-                        if (notbrick)
-                                cli_out ("%s is offline", hostname);
-                        else
-                                cli_out ("Brick is offline");
-                        continue;
-                }
+        snprintf(key, sizeof(key), "brick%d.status", i);
+        ret = dict_get_int32(dict, key, &online);
+        if (ret)
+            goto out;
+        if (!online) {
+            if (notbrick)
+                cli_out("%s is offline", hostname);
+            else
+                cli_out("Brick is offline");
+            continue;
+        }
 
-                memset (key, 0, sizeof (key));
-                snprintf (key, sizeof (key), "brick%d.conncount", i);
-                ret = dict_get_int32 (dict, key, &conn_count);
-                if (ret)
-                        goto out;
+        snprintf(key, sizeof(key), "brick%d.conncount", i);
+        ret = dict_get_int32(dict, key, &conn_count);
+        if (ret)
+            goto out;
 
-                for (j = 0; j < conn_count; j++) {
-                        if (conn_count > 1)
-                                cli_out ("Connection %d:", j+1);
+        for (j = 0; j < conn_count; j++) {
+            if (conn_count > 1)
+                cli_out("Connection %d:", j + 1);
 
-                        memset (key, 0, sizeof (key));
-                        snprintf (key, sizeof (key), "brick%d.conn%d.itable",
-                                  i, j);
-                        cli_print_volume_status_itables (dict, key);
-                        cli_out (" ");
-                }
+            snprintf(key, sizeof(key), "brick%d.conn%d.itable", i, j);
+            cli_print_volume_status_itables(dict, key);
+            cli_out(" ");
         }
+    }
 out:
-        cli_out ("----------------------------------------------");
-        return;
+    cli_out("----------------------------------------------");
+    return;
 }
 
 void
-cli_print_volume_status_fdtable (dict_t *dict, char *prefix)
-{
-        int             ret = -1;
-        char            key[1024] = {0,};
-        int             refcount = 0;
-        uint32_t        maxfds = 0;
-        int             firstfree = 0;
-        int             openfds = 0;
-        int             fd_pid = 0;
-        int             fd_refcount = 0;
-        int             fd_flags = 0;
-        int             i = 0;
-
-        GF_ASSERT (dict);
-        GF_ASSERT (prefix);
-
-        memset (key, 0, sizeof (key));
-        snprintf (key, sizeof (key), "%s.refcount", prefix);
-        ret = dict_get_int32 (dict, key, &refcount);
+cli_print_volume_status_fdtable(dict_t *dict, char *prefix)
+{
+    int ret = -1;
+    char key[256] = {
+        0,
+    };
+    int refcount = 0;
+    uint32_t maxfds = 0;
+    int firstfree = 0;
+    int openfds = 0;
+    int fd_pid = 0;
+    int fd_refcount = 0;
+    int fd_flags = 0;
+    int i = 0;
+
+    GF_ASSERT(dict);
+    GF_ASSERT(prefix);
+
+    snprintf(key, sizeof(key), "%s.refcount", prefix);
+    ret = dict_get_int32(dict, key, &refcount);
+    if (ret)
+        goto out;
+
+    snprintf(key, sizeof(key), "%s.maxfds", prefix);
+    ret = dict_get_uint32(dict, key, &maxfds);
+    if (ret)
+        goto out;
+
+    snprintf(key, sizeof(key), "%s.firstfree", prefix);
+    ret = dict_get_int32(dict, key, &firstfree);
+    if (ret)
+        goto out;
+
+    cli_out("RefCount = %d  MaxFDs = %d  FirstFree = %d", refcount, maxfds,
+            firstfree);
+
+    snprintf(key, sizeof(key), "%s.openfds", prefix);
+    ret = dict_get_int32(dict, key, &openfds);
+    if (ret)
+        goto out;
+    if (0 == openfds) {
+        cli_err("No open fds");
+        goto out;
+    }
+
+    cli_out("%-19s %-19s %-19s %-19s", "FD Entry", "PID", "RefCount", "Flags");
+    cli_out("%-19s %-19s %-19s %-19s", "--------", "---", "--------", "-----");
+
+    for (i = 0; i < maxfds; i++) {
+        snprintf(key, sizeof(key), "%s.fdentry%d.pid", prefix, i);
+        ret = dict_get_int32(dict, key, &fd_pid);
         if (ret)
-                goto out;
+            continue;
 
-        memset (key, 0, sizeof (key));
-        snprintf (key, sizeof (key), "%s.maxfds", prefix);
-        ret = dict_get_uint32 (dict, key, &maxfds);
+        snprintf(key, sizeof(key), "%s.fdentry%d.refcount", prefix, i);
+        ret = dict_get_int32(dict, key, &fd_refcount);
         if (ret)
-                goto out;
-
-        memset (key, 0 ,sizeof (key));
-        snprintf (key, sizeof (key), "%s.firstfree", prefix);
-        ret = dict_get_int32 (dict, key, &firstfree);
-        if (ret)
-                goto out;
-
-        cli_out ("RefCount = %d  MaxFDs = %d  FirstFree = %d",
-                 refcount, maxfds, firstfree);
+            continue;
 
-        memset (key, 0, sizeof (key));
-        snprintf (key, sizeof (key), "%s.openfds", prefix);
-        ret = dict_get_int32 (dict, key, &openfds);
+        snprintf(key, sizeof(key), "%s.fdentry%d.flags", prefix, i);
+        ret = dict_get_int32(dict, key, &fd_flags);
         if (ret)
-                goto out;
-        if (0 == openfds) {
-                cli_err ("No open fds");
-                goto out;
-        }
-
-        cli_out ("%-19s %-19s %-19s %-19s", "FD Entry", "PID",
-                 "RefCount", "Flags");
-        cli_out ("%-19s %-19s %-19s %-19s", "--------", "---",
-                 "--------", "-----");
-
-        for (i = 0; i < maxfds ; i++) {
-                memset (key, 0, sizeof (key));
-                snprintf (key, sizeof (key), "%s.fdentry%d.pid", prefix, i);
-                ret = dict_get_int32 (dict, key, &fd_pid);
-                if (ret)
-                        continue;
+            continue;
 
-                memset (key, 0, sizeof (key));
-                snprintf (key, sizeof (key), "%s.fdentry%d.refcount",
-                          prefix, i);
-                ret = dict_get_int32 (dict, key, &fd_refcount);
-                if (ret)
-                        continue;
-
-                memset (key, 0, sizeof (key));
-                snprintf (key, sizeof (key), "%s.fdentry%d.flags", prefix, i);
-                ret = dict_get_int32 (dict, key, &fd_flags);
-                if (ret)
-                        continue;
-
-                cli_out ("%-19d %-19d %-19d %-19d", i, fd_pid, fd_refcount,
-                         fd_flags);
-        }
+        cli_out("%-19d %-19d %-19d %-19d", i, fd_pid, fd_refcount, fd_flags);
+    }
 
 out:
-        return;
+    return;
 }
 
-void
-cli_print_volume_status_fd (dict_t *dict, gf_boolean_t notbrick)
-{
-        int             ret = -1;
-        char            *volname = NULL;
-        int             brick_index_max = -1;
-        int             other_count = 0;
-        int             index_max = 0;
-        char            *hostname = NULL;
-        char            *path = NULL;
-        int             online = -1;
-        int             conn_count = 0;
-        char            key[1024] = {0,};
-        int             i = 0;
-        int             j = 0;
-
-        GF_ASSERT (dict);
-
-        ret = dict_get_str (dict, "volname", &volname);
-        if (ret)
-                goto out;
-        cli_out ("FD tables for volume %s", volname);
-
-        ret = dict_get_int32 (dict, "brick-index-max", &brick_index_max);
+static void
+cli_print_volume_status_fd(dict_t *dict, gf_boolean_t notbrick)
+{
+    int ret = -1;
+    char *volname = NULL;
+    int brick_index_max = -1;
+    int other_count = 0;
+    int index_max = 0;
+    char *hostname = NULL;
+    char *path = NULL;
+    int online = -1;
+    int conn_count = 0;
+    char key[64] = {
+        0,
+    };
+    int i = 0;
+    int j = 0;
+
+    GF_ASSERT(dict);
+
+    ret = dict_get_str_sizen(dict, "volname", &volname);
+    if (ret)
+        goto out;
+    cli_out("FD tables for volume %s", volname);
+
+    ret = dict_get_int32_sizen(dict, "brick-index-max", &brick_index_max);
+    if (ret)
+        goto out;
+    ret = dict_get_int32_sizen(dict, "other-count", &other_count);
+    if (ret)
+        goto out;
+
+    index_max = brick_index_max + other_count;
+
+    for (i = 0; i <= index_max; i++) {
+        cli_out("----------------------------------------------");
+
+        snprintf(key, sizeof(key), "brick%d.hostname", i);
+        ret = dict_get_str(dict, key, &hostname);
         if (ret)
-                goto out;
-        ret = dict_get_int32 (dict, "other-count", &other_count);
+            goto out;
+        snprintf(key, sizeof(key), "brick%d.path", i);
+        ret = dict_get_str(dict, key, &path);
         if (ret)
-                goto out;
-
-        index_max = brick_index_max + other_count;
+            goto out;
 
-        for (i = 0; i <= index_max; i++) {
-                cli_out ("----------------------------------------------");
-
-                memset (key, 0, sizeof (key));
-                snprintf (key, sizeof (key), "brick%d.hostname", i);
-                ret = dict_get_str (dict, key, &hostname);
-                if (ret)
-                        goto out;
-                memset (key, 0, sizeof (key));
-                snprintf (key, sizeof (key), "brick%d.path", i);
-                ret = dict_get_str (dict, key, &path);
-                if (ret)
-                        goto out;
-
-                if (notbrick)
-                        cli_out ("%s : %s", hostname, path);
-                else
-                        cli_out ("Brick : %s:%s", hostname, path);
+        if (notbrick)
+            cli_out("%s : %s", hostname, path);
+        else
+            cli_out("Brick : %s:%s", hostname, path);
 
-                memset (key, 0, sizeof (key));
-                snprintf (key, sizeof (key), "brick%d.status", i);
-                ret = dict_get_int32 (dict, key, &online);
-                if (ret)
-                        goto out;
-                if (!online) {
-                        if (notbrick)
-                                cli_out ("%s is offline", hostname);
-                        else
-                                cli_out ("Brick is offline");
-                        continue;
-                }
+        snprintf(key, sizeof(key), "brick%d.status", i);
+        ret = dict_get_int32(dict, key, &online);
+        if (ret)
+            goto out;
+        if (!online) {
+            if (notbrick)
+                cli_out("%s is offline", hostname);
+            else
+                cli_out("Brick is offline");
+            continue;
+        }
 
-                memset (key, 0, sizeof (key));
-                snprintf (key, sizeof (key), "brick%d.conncount", i);
-                ret = dict_get_int32 (dict, key, &conn_count);
-                if (ret)
-                        goto out;
+        snprintf(key, sizeof(key), "brick%d.conncount", i);
+        ret = dict_get_int32(dict, key, &conn_count);
+        if (ret)
+            goto out;
 
-                for (j = 0; j < conn_count; j++) {
-                        cli_out ("Connection %d:", j+1);
+        for (j = 0; j < conn_count; j++) {
+            cli_out("Connection %d:", j + 1);
 
-                        memset (key, 0, sizeof (key));
-                        snprintf (key, sizeof (key), "brick%d.conn%d.fdtable",
-                                  i, j);
-                        cli_print_volume_status_fdtable (dict, key);
-                        cli_out (" ");
-                }
+            snprintf(key, sizeof(key), "brick%d.conn%d.fdtable", i, j);
+            cli_print_volume_status_fdtable(dict, key);
+            cli_out(" ");
         }
+    }
 out:
-        cli_out ("----------------------------------------------");
-        return;
+    cli_out("----------------------------------------------");
+    return;
 }
 
-void
-cli_print_volume_status_call_frame (dict_t *dict, char *prefix)
-{
-        int             ret = -1;
-        char            key[1024] = {0,};
-        int             ref_count = 0;
-        char            *translator = 0;
-        int             complete = 0;
-        char            *parent = NULL;
-        char            *wind_from = NULL;
-        char            *wind_to = NULL;
-        char            *unwind_from = NULL;
-        char            *unwind_to = NULL;
-
-        if (!dict || !prefix)
-                return;
+static void
+cli_print_volume_status_call_frame(dict_t *dict, char *prefix)
+{
+    int ret = -1;
+    char key[1024] = {
+        0,
+    };
+    int ref_count = 0;
+    char *translator = 0;
+    int complete = 0;
+    char *parent = NULL;
+    char *wind_from = NULL;
+    char *wind_to = NULL;
+    char *unwind_from = NULL;
+    char *unwind_to = NULL;
+
+    if (!dict || !prefix)
+        return;
 
-        memset (key, 0, sizeof (key));
-        snprintf (key, sizeof (key), "%s.refcount", prefix);
-        ret = dict_get_int32 (dict, key, &ref_count);
-        if (ret)
-                return;
+    snprintf(key, sizeof(key), "%s.refcount", prefix);
+    ret = dict_get_int32(dict, key, &ref_count);
+    if (ret)
+        return;
 
-        memset (key, 0, sizeof (key));
-        snprintf (key, sizeof (key), "%s.translator", prefix);
-        ret = dict_get_str (dict, key, &translator);
-        if (ret)
-                return;
+    snprintf(key, sizeof(key), "%s.translator", prefix);
+    ret = dict_get_str(dict, key, &translator);
+    if (ret)
+        return;
 
-        memset (key, 0, sizeof (key));
-        snprintf (key, sizeof (key), "%s.complete", prefix);
-        ret = dict_get_int32 (dict, key, &complete);
-        if (ret)
-                return;
+    snprintf(key, sizeof(key), "%s.complete", prefix);
+    ret = dict_get_int32(dict, key, &complete);
+    if (ret)
+        return;
 
-        cli_out ("  Ref Count   = %d", ref_count);
-        cli_out ("  Translator  = %s", translator);
-        cli_out ("  Completed   = %s", (complete ? "Yes" : "No"));
+    cli_out("  Ref Count   = %d", ref_count);
+    cli_out("  Translator  = %s", translator);
+    cli_out("  Completed   = %s", (complete ? "Yes" : "No"));
 
-        memset (key, 0, sizeof (key));
-        snprintf (key, sizeof (key), "%s.parent", prefix);
-        ret = dict_get_str (dict, key, &parent);
-        if (!ret)
-                cli_out ("  Parent      = %s", parent);
+    snprintf(key, sizeof(key), "%s.parent", prefix);
+    ret = dict_get_str(dict, key, &parent);
+    if (!ret)
+        cli_out("  Parent      = %s", parent);
 
-        memset (key, 0, sizeof (key));
-        snprintf (key, sizeof (key), "%s.windfrom", prefix);
-        ret = dict_get_str (dict, key, &wind_from);
-        if (!ret)
-                cli_out ("  Wind From   = %s", wind_from);
+    snprintf(key, sizeof(key), "%s.windfrom", prefix);
+    ret = dict_get_str(dict, key, &wind_from);
+    if (!ret)
+        cli_out("  Wind From   = %s", wind_from);
 
-        memset (key, 0, sizeof (key));
-        snprintf (key, sizeof (key), "%s.windto", prefix);
-        ret = dict_get_str (dict, key, &wind_to);
-        if (!ret)
-                cli_out ("  Wind To     = %s", wind_to);
+    snprintf(key, sizeof(key), "%s.windto", prefix);
+    ret = dict_get_str(dict, key, &wind_to);
+    if (!ret)
+        cli_out("  Wind To     = %s", wind_to);
 
-        memset (key, 0, sizeof (key));
-        snprintf (key, sizeof (key), "%s.unwindfrom", prefix);
-        ret = dict_get_str (dict, key, &unwind_from);
-        if (!ret)
-                cli_out ("  Unwind From = %s", unwind_from);
+    snprintf(key, sizeof(key), "%s.unwindfrom", prefix);
+    ret = dict_get_str(dict, key, &unwind_from);
+    if (!ret)
+        cli_out("  Unwind From = %s", unwind_from);
 
-        memset (key, 0, sizeof (key));
-        snprintf (key, sizeof (key), "%s.unwindto", prefix);
-        ret = dict_get_str (dict, key, &unwind_to);
-        if (!ret)
-                cli_out ("  Unwind To   = %s", unwind_to);
+    snprintf(key, sizeof(key), "%s.unwindto", prefix);
+    ret = dict_get_str(dict, key, &unwind_to);
+    if (!ret)
+        cli_out("  Unwind To   = %s", unwind_to);
 }
 
-void
-cli_print_volume_status_call_stack (dict_t *dict, char *prefix)
-{
-        int             ret = -1;
-        char            key[1024] = {0,};
-        int             uid = 0;
-        int             gid = 0;
-        int             pid = 0;
-        uint64_t        unique = 0;
-        //char            *op = NULL;
-        int             count = 0;
-        int             i = 0;
-
-        if (!dict || !prefix)
-                return;
-
-        memset (key, 0, sizeof (key));
-        snprintf (key, sizeof (key), "%s.uid", prefix);
-        ret = dict_get_int32 (dict, key, &uid);
-        if (ret)
-                return;
+static void
+cli_print_volume_status_call_stack(dict_t *dict, char *prefix)
+{
+    int ret = -1;
+    char key[256] = {
+        0,
+    };
+    int uid = 0;
+    int gid = 0;
+    int pid = 0;
+    uint64_t unique = 0;
+    // char            *op = NULL;
+    int count = 0;
+    int i = 0;
+
+    if (!prefix)
+        return;
 
-        memset (key, 0, sizeof (key));
-        snprintf (key, sizeof (key), "%s.gid", prefix);
-        ret = dict_get_int32 (dict, key, &gid);
-        if (ret)
-                return;
+    snprintf(key, sizeof(key), "%s.uid", prefix);
+    ret = dict_get_int32(dict, key, &uid);
+    if (ret)
+        return;
 
-        memset (key, 0, sizeof (key));
-        snprintf (key, sizeof (key), "%s.pid", prefix);
-        ret = dict_get_int32 (dict, key, &pid);
-        if (ret)
-                return;
+    snprintf(key, sizeof(key), "%s.gid", prefix);
+    ret = dict_get_int32(dict, key, &gid);
+    if (ret)
+        return;
 
-        memset (key, 0, sizeof (key));
-        snprintf (key, sizeof (key), "%s.unique", prefix);
-        ret = dict_get_uint64 (dict, key, &unique);
-        if (ret)
-                return;
+    snprintf(key, sizeof(key), "%s.pid", prefix);
+    ret = dict_get_int32(dict, key, &pid);
+    if (ret)
+        return;
 
-        /*
-        memset (key, 0, sizeof (key));
-        snprintf (key, sizeof (key), "%s.op", prefix);
-        ret = dict_get_str (dict, key, &op);
-        if (ret)
-                return;
-        */
+    snprintf(key, sizeof(key), "%s.unique", prefix);
+    ret = dict_get_uint64(dict, key, &unique);
+    if (ret)
+        return;
 
+    /*
+    snprintf (key, sizeof (key), "%s.op", prefix);
+    ret = dict_get_str (dict, key, &op);
+    if (ret)
+            return;
+    */
 
-        memset (key, 0, sizeof (key));
-        snprintf (key, sizeof (key), "%s.count", prefix);
-        ret = dict_get_int32 (dict, key, &count);
-        if (ret)
-                return;
+    snprintf(key, sizeof(key), "%s.count", prefix);
+    ret = dict_get_int32(dict, key, &count);
+    if (ret)
+        return;
 
-        cli_out (" UID    : %d", uid);
-        cli_out (" GID    : %d", gid);
-        cli_out (" PID    : %d", pid);
-        cli_out (" Unique : %"PRIu64, unique);
-        //cli_out ("\tOp     : %s", op);
-        cli_out (" Frames : %d", count);
+    cli_out(" UID    : %d", uid);
+    cli_out(" GID    : %d", gid);
+    cli_out(" PID    : %d", pid);
+    cli_out(" Unique : %" PRIu64, unique);
+    // cli_out ("\tOp     : %s", op);
+    cli_out(" Frames : %d", count);
 
-        for (i = 0; i < count; i++) {
-                cli_out (" Frame %d", i+1);
+    for (i = 0; i < count; i++) {
+        cli_out(" Frame %d", i + 1);
 
-                memset (key, 0, sizeof (key));
-                snprintf (key, sizeof (key), "%s.frame%d", prefix, i);
-                cli_print_volume_status_call_frame (dict, key);
-        }
+        snprintf(key, sizeof(key), "%s.frame%d", prefix, i);
+        cli_print_volume_status_call_frame(dict, key);
+    }
 
-        cli_out (" ");
+    cli_out(" ");
 }
 
-void
-cli_print_volume_status_callpool (dict_t *dict, gf_boolean_t notbrick)
-{
-        int             ret = -1;
-        char            *volname = NULL;
-        int             brick_index_max = -1;
-        int             other_count = 0;
-        int             index_max = 0;
-        char            *hostname = NULL;
-        char            *path = NULL;
-        int             online = -1;
-        int             call_count = 0;
-        char            key[1024] = {0,};
-        int             i = 0;
-        int             j = 0;
-
-        GF_ASSERT (dict);
-
-        ret = dict_get_str (dict, "volname", &volname);
-        if (ret)
-                goto out;
-        cli_out ("Pending calls for volume %s", volname);
-
-        ret = dict_get_int32 (dict, "brick-index-max", &brick_index_max);
+static void
+cli_print_volume_status_callpool(dict_t *dict, gf_boolean_t notbrick)
+{
+    int ret = -1;
+    char *volname = NULL;
+    int brick_index_max = -1;
+    int other_count = 0;
+    int index_max = 0;
+    char *hostname = NULL;
+    char *path = NULL;
+    int online = -1;
+    int call_count = 0;
+    char key[64] = {
+        0,
+    };
+    int i = 0;
+    int j = 0;
+
+    GF_ASSERT(dict);
+
+    ret = dict_get_str_sizen(dict, "volname", &volname);
+    if (ret)
+        goto out;
+    cli_out("Pending calls for volume %s", volname);
+
+    ret = dict_get_int32_sizen(dict, "brick-index-max", &brick_index_max);
+    if (ret)
+        goto out;
+    ret = dict_get_int32_sizen(dict, "other-count", &other_count);
+    if (ret)
+        goto out;
+
+    index_max = brick_index_max + other_count;
+
+    for (i = 0; i <= index_max; i++) {
+        cli_out("----------------------------------------------");
+
+        snprintf(key, sizeof(key), "brick%d.hostname", i);
+        ret = dict_get_str(dict, key, &hostname);
         if (ret)
-                goto out;
-        ret = dict_get_int32 (dict, "other-count", &other_count);
+            goto out;
+        snprintf(key, sizeof(key), "brick%d.path", i);
+        ret = dict_get_str(dict, key, &path);
         if (ret)
-                goto out;
-
-        index_max = brick_index_max + other_count;
-
-        for (i = 0; i <= index_max; i++) {
-                cli_out ("----------------------------------------------");
+            goto out;
 
-                memset (key, 0, sizeof (key));
-                snprintf (key, sizeof (key), "brick%d.hostname", i);
-                ret = dict_get_str (dict, key, &hostname);
-                if (ret)
-                        goto out;
-                memset (key, 0, sizeof (key));
-                snprintf (key, sizeof (key), "brick%d.path", i);
-                ret = dict_get_str (dict, key, &path);
-                if (ret)
-                        goto out;
-
-                if (notbrick)
-                        cli_out ("%s : %s", hostname, path);
-                else
-                        cli_out ("Brick : %s:%s", hostname, path);
+        if (notbrick)
+            cli_out("%s : %s", hostname, path);
+        else
+            cli_out("Brick : %s:%s", hostname, path);
 
-                memset (key, 0, sizeof (key));
-                snprintf (key, sizeof (key), "brick%d.status", i);
-                ret = dict_get_int32 (dict, key, &online);
-                if (ret)
-                        goto out;
-                if (!online) {
-                        if (notbrick)
-                                cli_out ("%s is offline", hostname);
-                        else
-                                cli_out ("Brick is offline");
-                        continue;
-                }
+        snprintf(key, sizeof(key), "brick%d.status", i);
+        ret = dict_get_int32(dict, key, &online);
+        if (ret)
+            goto out;
+        if (!online) {
+            if (notbrick)
+                cli_out("%s is offline", hostname);
+            else
+                cli_out("Brick is offline");
+            continue;
+        }
 
-                memset (key, 0, sizeof (key));
-                snprintf (key, sizeof (key), "brick%d.callpool.count", i);
-                ret = dict_get_int32 (dict, key, &call_count);
-                if (ret)
-                        goto out;
-                cli_out ("Pending calls: %d", call_count);
+        snprintf(key, sizeof(key), "brick%d.callpool.count", i);
+        ret = dict_get_int32(dict, key, &call_count);
+        if (ret)
+            goto out;
+        cli_out("Pending calls: %d", call_count);
 
-                if (0 == call_count)
-                        continue;
+        if (0 == call_count)
+            continue;
 
-                for (j = 0; j < call_count; j++) {
-                        cli_out ("Call Stack%d", j+1);
+        for (j = 0; j < call_count; j++) {
+            cli_out("Call Stack%d", j + 1);
 
-                        memset (key, 0, sizeof (key));
-                        snprintf (key, sizeof (key),
-                                  "brick%d.callpool.stack%d", i, j);
-                        cli_print_volume_status_call_stack (dict, key);
-                }
+            snprintf(key, sizeof(key), "brick%d.callpool.stack%d", i, j);
+            cli_print_volume_status_call_stack(dict, key);
         }
+    }
 
 out:
-        cli_out ("----------------------------------------------");
-        return;
+    cli_out("----------------------------------------------");
+    return;
 }
 
 static void
-cli_print_volume_status_tasks (dict_t *dict)
-{
-        int             ret         = -1;
-        int             i           = 0;
-        int             j           = 0;
-        int             count       = 0;
-        int             task_count  = 0;
-        int             status      = 0;
-        char           *op          = NULL;
-        char           *task_id_str = NULL;
-        char           *volname     = NULL;
-        char            key[1024]   = {0,};
-        char            task[1024]  = {0,};
-        char           *brick       = NULL;
-        char           *src_brick   = NULL;
-        char           *dest_brick  = NULL;
-
-        ret = dict_get_str (dict, "volname", &volname);
-        if (ret)
-                goto out;
+cli_print_volume_status_tasks(dict_t *dict)
+{
+    int ret = -1;
+    int i = 0;
+    int j = 0;
+    int count = 0;
+    int task_count = 0;
+    int status = 0;
+    char *op = NULL;
+    char *task_id_str = NULL;
+    char *volname = NULL;
+    char key[64] = {
+        0,
+    };
+    char task[32] = {
+        0,
+    };
+    char *brick = NULL;
+
+    ret = dict_get_int32_sizen(dict, "tasks", &task_count);
+    if (ret) {
+        gf_log("cli", GF_LOG_ERROR, "Failed to get tasks count");
+        return;
+    }
 
-        ret = dict_get_int32 (dict, "tasks", &task_count);
-        if (ret) {
-                gf_log ("cli", GF_LOG_ERROR, "Failed to get tasks count");
-                return;
-        }
+    ret = dict_get_str_sizen(dict, "volname", &volname);
+    if (ret)
+        goto out;
 
-        cli_out ("Task Status of Volume %s", volname);
-        cli_print_line (CLI_BRICK_STATUS_LINE_LEN);
+    cli_out("Task Status of Volume %s", volname);
+    cli_print_line(CLI_BRICK_STATUS_LINE_LEN);
 
-        if (task_count == 0) {
-                cli_out ("There are no active volume tasks");
-                cli_out (" ");
-                return;
-        }
+    if (task_count == 0) {
+        cli_out("There are no active volume tasks");
+        cli_out(" ");
+        return;
+    }
 
-        for (i = 0; i < task_count; i++) {
-                memset (key, 0, sizeof (key));
-                snprintf (key, sizeof (key), "task%d.type", i);
-                ret = dict_get_str(dict, key, &op);
-                if (ret)
-                        return;
-                cli_out ("%-20s : %-20s", "Task", op);
+    for (i = 0; i < task_count; i++) {
+        snprintf(key, sizeof(key), "task%d.type", i);
+        ret = dict_get_str(dict, key, &op);
+        if (ret)
+            return;
+        cli_out("%-20s : %-20s", "Task", op);
 
-                memset (key, 0, sizeof (key));
-                snprintf (key, sizeof (key), "task%d.id", i);
-                ret = dict_get_str (dict, key, &task_id_str);
-                if (ret)
-                        return;
-                cli_out ("%-20s : %-20s", "ID", task_id_str);
+        snprintf(key, sizeof(key), "task%d.id", i);
+        ret = dict_get_str(dict, key, &task_id_str);
+        if (ret)
+            return;
+        cli_out("%-20s : %-20s", "ID", task_id_str);
 
-                memset (key, 0, sizeof (key));
-                snprintf (key, sizeof (key), "task%d.status", i);
-                ret = dict_get_int32 (dict, key, &status);
-                if (ret)
-                        return;
+        snprintf(key, sizeof(key), "task%d.status", i);
+        ret = dict_get_int32(dict, key, &status);
+        if (ret)
+            return;
 
-                snprintf (task, sizeof (task), "task%d", i);
+        snprintf(task, sizeof(task), "task%d", i);
 
-                if (!strcmp (op, "Remove brick")) {
-                        memset (key, 0, sizeof (key));
-                        snprintf (key, sizeof (key), "%s.count", task);
-                        ret = dict_get_int32 (dict, key, &count);
-                        if (ret)
-                                goto out;
+        if (!strcmp(op, "Remove brick")) {
+            snprintf(key, sizeof(key), "%s.count", task);
+            ret = dict_get_int32(dict, key, &count);
+            if (ret)
+                goto out;
 
-                        cli_out ("%-20s", "Removed bricks:");
+            cli_out("%-20s", "Removed bricks:");
 
-                        for (j = 1; j <= count; j++) {
-                                memset (key, 0, sizeof (key));
-                                snprintf (key, sizeof (key),"%s.brick%d",
-                                          task, j);
-                                ret = dict_get_str (dict, key, &brick);
-                                if (ret)
-                                        goto out;
+            for (j = 1; j <= count; j++) {
+                snprintf(key, sizeof(key), "%s.brick%d", task, j);
+                ret = dict_get_str(dict, key, &brick);
+                if (ret)
+                    goto out;
 
-                                cli_out ("%-20s", brick);
-                        }
-                }
-                cli_out ("%-20s : %-20s", "Status",
-                         cli_vol_task_status_str[status]);
-                cli_out (" ");
+                cli_out("%-20s", brick);
+            }
         }
+        cli_out("%-20s : %-20s", "Status", cli_vol_task_status_str[status]);
+        cli_out(" ");
+    }
 
 out:
-        return;
+    return;
 }
 
 static int
-gf_cli_status_cbk (struct rpc_req *req, struct iovec *iov,
-                      int count, void *myframe)
-{
-        int                             ret             = -1;
-        int                             brick_index_max = -1;
-        int                             other_count     = 0;
-        int                             index_max       = 0;
-        int                             i               = 0;
-        int                             type            = -1;
-        int                             hot_brick_count = -1;
-        int                             pid             = -1;
-        uint32_t                        cmd             = 0;
-        gf_boolean_t                    notbrick        = _gf_false;
-        char                            key[1024]       = {0,};
-        char                           *hostname        = NULL;
-        char                           *path            = NULL;
-        char                           *volname         = NULL;
-        dict_t                         *dict            = NULL;
-        gf_cli_rsp                      rsp             = {0,};
-        cli_volume_status_t             status          = {0};
-        cli_local_t                    *local           = NULL;
-        gf_boolean_t                    wipe_local      = _gf_false;
-        char                            msg[1024]       = {0,};
-
-        if (req->rpc_status == -1)
-                goto out;
-
-        ret = xdr_to_generic (*iov, &rsp, (xdrproc_t)xdr_gf_cli_rsp);
-        if (ret < 0) {
-                gf_log (((call_frame_t *) myframe)->this->name, GF_LOG_ERROR,
-                        "Failed to decode xdr response");
-                goto out;
-        }
-
-        gf_log ("cli", GF_LOG_DEBUG, "Received response to status cmd");
-
-        local = ((call_frame_t *)myframe)->local;
+gf_cli_status_cbk(struct rpc_req *req, struct iovec *iov, int count,
+                  void *myframe)
+{
+    int ret = -1;
+    int brick_index_max = -1;
+    int other_count = 0;
+    int index_max = 0;
+    int i = 0;
+    int type = -1;
+    int hot_brick_count = -1;
+    int pid = -1;
+    uint32_t cmd = 0;
+    gf_boolean_t notbrick = _gf_false;
+    char key[64] = {
+        0,
+    };
+    char *hostname = NULL;
+    char *path = NULL;
+    char *volname = NULL;
+    dict_t *dict = NULL;
+    gf_cli_rsp rsp = {
+        0,
+    };
+    cli_volume_status_t status = {0};
+    cli_local_t *local = NULL;
+    gf_boolean_t wipe_local = _gf_false;
+    char msg[1024] = {
+        0,
+    };
+
+    GF_ASSERT(myframe);
+
+    if (req->rpc_status == -1)
+        goto out;
+
+    ret = xdr_to_generic(*iov, &rsp, (xdrproc_t)xdr_gf_cli_rsp);
+    if (ret < 0) {
+        gf_log(((call_frame_t *)myframe)->this->name, GF_LOG_ERROR,
+               XDR_DECODE_FAIL);
+        goto out;
+    }
+
+    gf_log("cli", GF_LOG_DEBUG, "Received response to status cmd");
+
+    local = ((call_frame_t *)myframe)->local;
+    if (!local) {
+        local = cli_local_get();
         if (!local) {
-                local = cli_local_get ();
-                if (!local) {
-                        ret = -1;
-                        gf_log ("cli", GF_LOG_ERROR, "Failed to get local");
-                        goto out;
-                }
-                wipe_local = _gf_true;
+            ret = -1;
+            gf_log("cli", GF_LOG_ERROR, "Failed to get local");
+            goto out;
         }
+        wipe_local = _gf_true;
+    }
 
-        if (rsp.op_ret) {
-                if (strcmp (rsp.op_errstr, ""))
-                        snprintf (msg, sizeof (msg), "%s", rsp.op_errstr);
-                else
-                        snprintf (msg, sizeof (msg), "Unable to obtain volume "
-                                  "status information.");
-
-                if (global_state->mode & GLUSTER_MODE_XML) {
-                        if (!local->all)
-                                cli_xml_output_str ("volStatus", msg,
-                                                    rsp.op_ret, rsp.op_errno,
-                                                    rsp.op_errstr);
-                        ret = 0;
-                        goto out;
-                }
-
-                cli_err ("%s", msg);
-                if (local && local->all) {
-                        ret = 0;
-                        cli_out (" ");
-                } else
-                        ret = -1;
+    if (rsp.op_ret) {
+        if (strcmp(rsp.op_errstr, ""))
+            snprintf(msg, sizeof(msg), "%s", rsp.op_errstr);
+        else
+            snprintf(msg, sizeof(msg),
+                     "Unable to obtain volume status information.");
 
-                goto out;
+        if (global_state->mode & GLUSTER_MODE_XML) {
+            if (!local->all)
+                cli_xml_output_str("volStatus", msg, rsp.op_ret, rsp.op_errno,
+                                   rsp.op_errstr);
+            ret = 0;
+            goto out;
         }
 
-        dict = dict_new ();
-        if (!dict)
-                goto out;
+        cli_err("%s", msg);
+        if (local && local->all) {
+            ret = 0;
+            cli_out(" ");
+        } else
+            ret = -1;
+
+        goto out;
+    }
+
+    dict = dict_new();
+    if (!dict) {
+        gf_log(THIS->name, GF_LOG_ERROR, "Failed to create the dict");
+        ret = -1;
+        goto out;
+    }
+
+    ret = dict_unserialize(rsp.dict.dict_val, rsp.dict.dict_len, &dict);
+    if (ret)
+        goto out;
+
+    ret = dict_get_uint32(dict, "cmd", &cmd);
+    if (ret)
+        goto out;
+
+    if ((cmd & GF_CLI_STATUS_ALL)) {
+        if (local && local->dict) {
+            dict_ref(dict);
+            ret = dict_set_static_ptr(local->dict, "rsp-dict", dict);
+            ret = 0;
+        } else {
+            gf_log("cli", GF_LOG_ERROR, "local not found");
+            ret = -1;
+        }
+        goto out;
+    }
+
+    if (global_state->mode & GLUSTER_MODE_XML) {
+        if (!local->all) {
+            ret = cli_xml_output_vol_status_begin(local, rsp.op_ret,
+                                                  rsp.op_errno, rsp.op_errstr);
+            if (ret) {
+                gf_log("cli", GF_LOG_ERROR, XML_ERROR);
+                goto xml_end;
+            }
+        }
+        if (cmd & GF_CLI_STATUS_TASKS) {
+            ret = cli_xml_output_vol_status_tasks_detail(local, dict);
+            if (ret) {
+                gf_log("cli", GF_LOG_ERROR, "Error outputting to xml");
+                goto xml_end;
+            }
+        } else {
+            ret = cli_xml_output_vol_status(local, dict);
+            if (ret) {
+                gf_log("cli", GF_LOG_ERROR, XML_ERROR);
+                goto xml_end;
+            }
+        }
+
+    xml_end:
+        if (!local->all) {
+            ret = cli_xml_output_vol_status_end(local);
+            if (ret) {
+                gf_log("cli", GF_LOG_ERROR, XML_ERROR);
+            }
+        }
+        goto out;
+    }
+
+    if ((cmd & GF_CLI_STATUS_NFS) || (cmd & GF_CLI_STATUS_SHD) ||
+        (cmd & GF_CLI_STATUS_QUOTAD) || (cmd & GF_CLI_STATUS_SNAPD) ||
+        (cmd & GF_CLI_STATUS_BITD) || (cmd & GF_CLI_STATUS_SCRUB))
+        notbrick = _gf_true;
+
+    switch (cmd & GF_CLI_STATUS_MASK) {
+        case GF_CLI_STATUS_MEM:
+            cli_print_volume_status_mem(dict, notbrick);
+            goto cont;
+            break;
+        case GF_CLI_STATUS_CLIENTS:
+            cli_print_volume_status_clients(dict, notbrick);
+            goto cont;
+            break;
+        case GF_CLI_STATUS_CLIENT_LIST:
+            cli_print_volume_status_client_list(dict, notbrick);
+            goto cont;
+            break;
+        case GF_CLI_STATUS_INODE:
+            cli_print_volume_status_inode(dict, notbrick);
+            goto cont;
+            break;
+        case GF_CLI_STATUS_FD:
+            cli_print_volume_status_fd(dict, notbrick);
+            goto cont;
+            break;
+        case GF_CLI_STATUS_CALLPOOL:
+            cli_print_volume_status_callpool(dict, notbrick);
+            goto cont;
+            break;
+        case GF_CLI_STATUS_TASKS:
+            cli_print_volume_status_tasks(dict);
+            goto cont;
+            break;
+        default:
+            break;
+    }
 
-        ret = dict_unserialize (rsp.dict.dict_val,
-                                rsp.dict.dict_len,
-                                &dict);
-        if (ret)
-                goto out;
+    ret = dict_get_str_sizen(dict, "volname", &volname);
+    if (ret)
+        goto out;
 
-        ret = dict_get_uint32 (dict, "cmd", &cmd);
-        if (ret)
-                goto out;
+    ret = dict_get_int32_sizen(dict, "brick-index-max", &brick_index_max);
+    if (ret)
+        goto out;
 
-        if ((cmd & GF_CLI_STATUS_ALL)) {
-                if (local && local->dict) {
-                        dict_ref (dict);
-                        ret = dict_set_static_ptr (local->dict, "rsp-dict", dict);
-                        ret = 0;
-                } else {
-                        gf_log ("cli", GF_LOG_ERROR, "local not found");
-                        ret = -1;
-                }
-                goto out;
-        }
+    ret = dict_get_int32_sizen(dict, "other-count", &other_count);
+    if (ret)
+        goto out;
 
-        if ((cmd & GF_CLI_STATUS_NFS) || (cmd & GF_CLI_STATUS_SHD) ||
-            (cmd & GF_CLI_STATUS_QUOTAD) || (cmd & GF_CLI_STATUS_SNAPD) ||
-            (cmd & GF_CLI_STATUS_BITD) || (cmd & GF_CLI_STATUS_SCRUB))
-                notbrick = _gf_true;
+    index_max = brick_index_max + other_count;
 
-        if (global_state->mode & GLUSTER_MODE_XML) {
-                if (!local->all) {
-                        ret = cli_xml_output_vol_status_begin (local,
-                                                               rsp.op_ret,
-                                                               rsp.op_errno,
-                                                               rsp.op_errstr);
-                        if (ret) {
-                                gf_log ("cli", GF_LOG_ERROR,
-                                        "Error outputting to xml");
-                                goto out;
-                        }
-                }
-                if (cmd & GF_CLI_STATUS_TASKS) {
-                        ret = cli_xml_output_vol_status_tasks_detail (local,
-                                                                      dict);
-                        if (ret) {
-                                gf_log ("cli", GF_LOG_ERROR,"Error outputting "
-                                        "to xml");
-                                goto out;
-                        }
-                } else {
-                        ret = cli_xml_output_vol_status (local, dict);
-                        if (ret) {
-                                gf_log ("cli", GF_LOG_ERROR,
-                                        "Error outputting to xml");
-                                goto out;
-                        }
-                }
+    ret = dict_get_int32_sizen(dict, "type", &type);
+    if (ret)
+        goto out;
 
-                if (!local->all) {
-                        ret = cli_xml_output_vol_status_end (local);
-                        if (ret) {
-                                gf_log ("cli", GF_LOG_ERROR,
-                                        "Error outputting to xml");
-                        }
-                }
-                goto out;
-        }
+    ret = dict_get_int32_sizen(dict, "hot_brick_count", &hot_brick_count);
+    if (ret)
+        goto out;
 
-        status.brick = GF_CALLOC (1, PATH_MAX + 256, gf_common_mt_strdup);
-        if (!status.brick) {
-                errno   = ENOMEM;
-                ret     = -1;
-                goto out;
-        }
-        switch (cmd & GF_CLI_STATUS_MASK) {
-                case GF_CLI_STATUS_MEM:
-                        cli_print_volume_status_mem (dict, notbrick);
-                        goto cont;
-                        break;
-                case GF_CLI_STATUS_CLIENTS:
-                        cli_print_volume_status_clients (dict, notbrick);
-                        goto cont;
-                        break;
-                case GF_CLI_STATUS_INODE:
-                        cli_print_volume_status_inode (dict, notbrick);
-                        goto cont;
-                        break;
-                case GF_CLI_STATUS_FD:
-                        cli_print_volume_status_fd (dict, notbrick);
-                        goto cont;
-                        break;
-                case GF_CLI_STATUS_CALLPOOL:
-                        cli_print_volume_status_callpool (dict, notbrick);
-                        goto cont;
-                        break;
-                case GF_CLI_STATUS_TASKS:
-                        cli_print_volume_status_tasks (dict);
-                        goto cont;
-                        break;
-                default:
-                        break;
-        }
-
-        ret = dict_get_str (dict, "volname", &volname);
-        if (ret)
-                goto out;
+    cli_out("Status of volume: %s", volname);
 
-        ret = dict_get_int32 (dict, "brick-index-max", &brick_index_max);
-        if (ret)
-                goto out;
+    if ((cmd & GF_CLI_STATUS_DETAIL) == 0) {
+        cli_out("%-*s %s  %s  %s  %s", CLI_VOL_STATUS_BRICK_LEN,
+                "Gluster process", "TCP Port", "RDMA Port", "Online", "Pid");
+        cli_print_line(CLI_BRICK_STATUS_LINE_LEN);
+    }
 
-        ret = dict_get_int32 (dict, "other-count", &other_count);
-        if (ret)
-                goto out;
+    status.brick = GF_MALLOC(PATH_MAX + 256, gf_common_mt_strdup);
+    if (!status.brick) {
+        errno = ENOMEM;
+        ret = -1;
+        goto out;
+    }
 
-        index_max = brick_index_max + other_count;
+    for (i = 0; i <= index_max; i++) {
+        status.rdma_port = 0;
 
-        ret = dict_get_int32 (dict, "type", &type);
+        snprintf(key, sizeof(key), "brick%d.hostname", i);
+        ret = dict_get_str(dict, key, &hostname);
         if (ret)
-                goto out;
+            continue;
 
-        ret = dict_get_int32 (dict, "hot_brick_count", &hot_brick_count);
+        snprintf(key, sizeof(key), "brick%d.path", i);
+        ret = dict_get_str(dict, key, &path);
         if (ret)
-                goto out;
-
-        cli_out ("Status of volume: %s", volname);
+            continue;
 
-        if ((cmd & GF_CLI_STATUS_DETAIL) == 0) {
-                cli_out ("%-*s %s  %s  %s  %s", CLI_VOL_STATUS_BRICK_LEN,
-                         "Gluster process", "TCP Port", "RDMA Port",
-                         "Online", "Pid");
-                cli_print_line (CLI_BRICK_STATUS_LINE_LEN);
-        }
-        if (type == GF_CLUSTER_TYPE_TIER) {
-                cli_out ("Hot Bricks:");
+        /* Brick/not-brick is handled separately here as all
+         * types of nodes are contained in the default output
+         */
+        status.brick[0] = '\0';
+        if (!strcmp(hostname, "NFS Server") ||
+            !strcmp(hostname, "Self-heal Daemon") ||
+            !strcmp(hostname, "Quota Daemon") ||
+            !strcmp(hostname, "Snapshot Daemon") ||
+            !strcmp(hostname, "Scrubber Daemon") ||
+            !strcmp(hostname, "Bitrot Daemon"))
+            snprintf(status.brick, PATH_MAX + 255, "%s on %s", hostname, path);
+        else {
+            snprintf(key, sizeof(key), "brick%d.rdma_port", i);
+            ret = dict_get_int32(dict, key, &(status.rdma_port));
+            if (ret)
+                continue;
+            snprintf(status.brick, PATH_MAX + 255, "Brick %s:%s", hostname,
+                     path);
         }
-        for (i = 0; i <= index_max; i++) {
 
-                if (type == GF_CLUSTER_TYPE_TIER && i == hot_brick_count) {
-                        cli_out ("Cold Bricks:");
-                }
-                status.rdma_port = 0;
-
-                memset (key, 0, sizeof (key));
-                snprintf (key, sizeof (key), "brick%d.hostname", i);
-                ret = dict_get_str (dict, key, &hostname);
-                if (ret)
-                        continue;
-
-                memset (key, 0, sizeof (key));
-                snprintf (key, sizeof (key), "brick%d.path", i);
-                ret = dict_get_str (dict, key, &path);
-                if (ret)
-                        continue;
-
-                /* Brick/not-brick is handled separately here as all
-                 * types of nodes are contained in the default output
-                 */
-                memset (status.brick, 0, PATH_MAX + 255);
-                if (!strcmp (hostname, "NFS Server") ||
-                    !strcmp (hostname, "Self-heal Daemon") ||
-                    !strcmp (hostname, "Quota Daemon") ||
-                    !strcmp (hostname, "Snapshot Daemon") ||
-                    !strcmp (hostname, "Scrubber Daemon") ||
-                    !strcmp (hostname, "Bitrot Daemon"))
-                        snprintf (status.brick, PATH_MAX + 255, "%s on %s",
-                                  hostname, path);
-                else {
-                        memset (key, 0, sizeof (key));
-                        snprintf (key, sizeof (key), "brick%d.rdma_port", i);
-                        ret = dict_get_int32 (dict, key, &(status.rdma_port));
-                        if (ret)
-                                continue;
-                        snprintf (status.brick, PATH_MAX + 255, "Brick %s:%s",
-                                  hostname, path);
-                }
+        snprintf(key, sizeof(key), "brick%d.port", i);
+        ret = dict_get_int32(dict, key, &(status.port));
+        if (ret)
+            continue;
 
-                memset (key, 0, sizeof (key));
-                snprintf (key, sizeof (key), "brick%d.port", i);
-                ret = dict_get_int32 (dict, key, &(status.port));
-                if (ret)
-                        continue;
+        snprintf(key, sizeof(key), "brick%d.status", i);
+        ret = dict_get_int32(dict, key, &(status.online));
+        if (ret)
+            continue;
 
-                memset (key, 0, sizeof (key));
-                snprintf (key, sizeof (key), "brick%d.status", i);
-                ret = dict_get_int32 (dict, key, &(status.online));
-                if (ret)
-                        continue;
+        snprintf(key, sizeof(key), "brick%d.pid", i);
+        ret = dict_get_int32(dict, key, &pid);
+        if (ret)
+            continue;
+        if (pid == -1)
+            ret = gf_asprintf(&(status.pid_str), "%s", "N/A");
+        else
+            ret = gf_asprintf(&(status.pid_str), "%d", pid);
 
-                memset (key, 0, sizeof (key));
-                snprintf (key, sizeof (key), "brick%d.pid", i);
-                ret = dict_get_int32 (dict, key, &pid);
-                if (ret)
-                        continue;
-                if (pid == -1)
-                        ret = gf_asprintf (&(status.pid_str), "%s", "N/A");
-                else
-                        ret = gf_asprintf (&(status.pid_str), "%d", pid);
-
-                if (ret == -1)
-                        goto out;
-
-                if ((cmd & GF_CLI_STATUS_DETAIL)) {
-                        ret = cli_get_detail_status (dict, i, &status);
-                        if (ret)
-                                goto out;
-                        cli_print_line (CLI_BRICK_STATUS_LINE_LEN);
-                        cli_print_detailed_status (&status);
-                } else {
-                        cli_print_brick_status (&status);
-                }
+        if (ret == -1)
+            goto out;
 
+        if ((cmd & GF_CLI_STATUS_DETAIL)) {
+            ret = cli_get_detail_status(dict, i, &status);
+            if (ret)
+                goto out;
+            cli_print_line(CLI_BRICK_STATUS_LINE_LEN);
+            cli_print_detailed_status(&status);
+        } else {
+            cli_print_brick_status(&status);
         }
-        cli_out (" ");
 
-        if ((cmd & GF_CLI_STATUS_MASK) == GF_CLI_STATUS_NONE)
-                cli_print_volume_status_tasks (dict);
+        /* Allocatated memory using gf_asprintf*/
+        GF_FREE(status.pid_str);
+    }
+    cli_out(" ");
+
+    if ((cmd & GF_CLI_STATUS_MASK) == GF_CLI_STATUS_NONE)
+        cli_print_volume_status_tasks(dict);
 cont:
-        ret = rsp.op_ret;
+    ret = rsp.op_ret;
 
 out:
-        if (dict)
-                dict_unref (dict);
-        GF_FREE (status.brick);
-        if (local && wipe_local) {
-                cli_local_wipe (local);
-        }
+    if (dict)
+        dict_unref(dict);
+    GF_FREE(status.brick);
+    if (local && wipe_local) {
+        cli_local_wipe(local);
+    }
 
-        cli_cmd_broadcast_response (ret);
-        return ret;
+    cli_cmd_broadcast_response(ret);
+    gf_free_xdr_cli_rsp(rsp);
+    return ret;
 }
 
-int32_t
-gf_cli_status_volume (call_frame_t *frame, xlator_t *this,
-                         void *data)
+static int32_t
+gf_cli_status_volume(call_frame_t *frame, xlator_t *this, void *data)
 {
-        gf_cli_req                      req  = {{0,}};
-        int                             ret  = -1;
-        dict_t                         *dict = NULL;
-
-        if (!frame || !this || !data)
-                goto out;
+    gf_cli_req req = {{
+        0,
+    }};
+    int ret = -1;
+    dict_t *dict = NULL;
 
-        dict = data;
+    dict = data;
 
-        ret = cli_to_glusterd (&req, frame, gf_cli_status_cbk,
-                               (xdrproc_t) xdr_gf_cli_req, dict,
-                               GLUSTER_CLI_STATUS_VOLUME, this, cli_rpc_prog,
-                               NULL);
- out:
-        gf_log ("cli", GF_LOG_DEBUG, "Returning: %d", ret);
-        return ret;
+    ret = cli_to_glusterd(&req, frame, gf_cli_status_cbk,
+                          (xdrproc_t)xdr_gf_cli_req, dict,
+                          GLUSTER_CLI_STATUS_VOLUME, this, cli_rpc_prog, NULL);
+    gf_log("cli", GF_LOG_DEBUG, RETURNING, ret);
+    GF_FREE(req.dict.dict_val);
+    return ret;
 }
 
-int
-gf_cli_status_volume_all (call_frame_t *frame, xlator_t *this, void *data)
-{
-        int              i            = 0;
-        int              ret          = -1;
-        int              vol_count    = -1;
-        uint32_t         cmd          = 0;
-        char             key[1024]    = {0};
-        char            *volname      = NULL;
-        void            *vol_dict     = NULL;
-        dict_t          *dict         = NULL;
-        cli_local_t     *local        = NULL;
-
-        if (frame->local) {
-                local = frame->local;
-                local->all = _gf_true;
-        } else
-                goto out;
+static int
+gf_cli_status_volume_all(call_frame_t *frame, xlator_t *this, void *data)
+{
+    int i = 0;
+    int ret = -1;
+    int vol_count = -1;
+    uint32_t cmd = 0;
+    char key[1024] = {0};
+    char *volname = NULL;
+    void *vol_dict = NULL;
+    dict_t *dict = NULL;
+    cli_local_t *local = NULL;
 
-        ret = dict_get_uint32 (local->dict, "cmd", &cmd);
-        if (ret)
-                goto out;
+    if (!frame)
+        goto out;
 
+    if (!frame->local)
+        goto out;
 
-        ret = gf_cli_status_volume (frame, this, data);
-        if (ret)
-                goto out;
+    local = frame->local;
 
-        ret = dict_get_ptr (local->dict, "rsp-dict", &vol_dict);
-        if (ret)
-                goto out;
+    ret = dict_get_uint32(local->dict, "cmd", &cmd);
+    if (ret)
+        goto out;
 
-        ret = dict_get_int32 ((dict_t *)vol_dict, "vol_count", &vol_count);
-        if (ret) {
-                cli_err ("Failed to get names of volumes");
-                goto out;
-        }
+    local->all = _gf_true;
 
-        /* remove the "all" flag in cmd */
-        cmd &= ~GF_CLI_STATUS_ALL;
-        cmd |= GF_CLI_STATUS_VOL;
+    ret = gf_cli_status_volume(frame, this, data);
 
-        if (global_state->mode & GLUSTER_MODE_XML) {
-                //TODO: Pass proper op_* values
-                ret = cli_xml_output_vol_status_begin (local, 0,0, NULL);
-                if (ret) {
-                        gf_log ("cli", GF_LOG_ERROR,
-                                "Error outputting to xml");
-                        goto out;
-                }
-        }
+    if (ret)
+        goto out;
 
-        if (vol_count == 0 && !(global_state->mode & GLUSTER_MODE_XML)) {
-                cli_err ("No volumes present");
-                ret = 0;
-                goto out;
-        }
+    ret = dict_get_ptr(local->dict, "rsp-dict", &vol_dict);
+    if (ret)
+        goto out;
 
-        for (i = 0; i < vol_count; i++) {
+    ret = dict_get_int32_sizen((dict_t *)vol_dict, "vol_count", &vol_count);
+    if (ret) {
+        cli_err("Failed to get names of volumes");
+        goto out;
+    }
 
-                dict = dict_new ();
-                if (!dict)
-                        goto out;
+    /* remove the "all" flag in cmd */
+    cmd &= ~GF_CLI_STATUS_ALL;
+    cmd |= GF_CLI_STATUS_VOL;
 
-                memset (key, 0, sizeof (key));
-                snprintf (key, sizeof (key), "vol%d", i);
-                ret = dict_get_str (vol_dict, key, &volname);
-                if (ret)
-                        goto out;
+    if (global_state->mode & GLUSTER_MODE_XML) {
+        // TODO: Pass proper op_* values
+        ret = cli_xml_output_vol_status_begin(local, 0, 0, NULL);
+        if (ret) {
+            gf_log("cli", GF_LOG_ERROR, XML_ERROR);
+            goto xml_end;
+        }
+    }
 
-                ret = dict_set_str (dict, "volname", volname);
-                if (ret)
-                        goto out;
+    if (vol_count == 0 && !(global_state->mode & GLUSTER_MODE_XML)) {
+        cli_err("No volumes present");
+        ret = 0;
+        goto out;
+    }
 
-                ret = dict_set_uint32 (dict, "cmd", cmd);
-                if (ret)
-                        goto out;
+    for (i = 0; i < vol_count; i++) {
+        dict = dict_new();
+        if (!dict)
+            goto out;
 
-                ret = gf_cli_status_volume (frame, this, dict);
-                if (ret)
-                        goto out;
+        ret = snprintf(key, sizeof(key), "vol%d", i);
+        ret = dict_get_strn(vol_dict, key, ret, &volname);
+        if (ret)
+            goto out;
 
-                dict_unref (dict);
-        }
+        ret = dict_set_str_sizen(dict, "volname", volname);
+        if (ret)
+            goto out;
 
-        if (global_state->mode & GLUSTER_MODE_XML) {
-                ret = cli_xml_output_vol_status_end (local);
-        }
+        ret = dict_set_uint32(dict, "cmd", cmd);
+        if (ret)
+            goto out;
 
- out:
+        ret = gf_cli_status_volume(frame, this, dict);
         if (ret)
-                gf_log ("cli", GF_LOG_ERROR, "status all failed");
+            goto out;
+
+        dict_unref(dict);
+    }
+
+xml_end:
+    if (global_state->mode & GLUSTER_MODE_XML) {
+        ret = cli_xml_output_vol_status_end(local);
+    }
 
-        if (vol_dict)
-                dict_unref (vol_dict);
+out:
+    if (ret)
+        gf_log("cli", GF_LOG_ERROR, "status all failed");
+
+    if (vol_dict)
+        dict_unref(vol_dict);
 
-        if (ret && dict)
-                dict_unref (dict);
+    if (ret && dict)
+        dict_unref(dict);
 
-        if (local)
-                cli_local_wipe (local);
+    if (local)
+        cli_local_wipe(local);
 
-        if (frame)
-                frame->local = NULL;
+    if (frame)
+        frame->local = NULL;
 
-        return ret;
+    return ret;
 }
 
 static int
-gf_cli_mount_cbk (struct rpc_req *req, struct iovec *iov,
-                  int count, void *myframe)
+gf_cli_mount_cbk(struct rpc_req *req, struct iovec *iov, int count,
+                 void *myframe)
 {
-        gf1_cli_mount_rsp rsp   = {0,};
-        int               ret   = -1;
+    gf1_cli_mount_rsp rsp = {
+        0,
+    };
+    int ret = -1;
 
-        if (-1 == req->rpc_status) {
-                goto out;
-        }
+    GF_ASSERT(myframe);
 
-        ret = xdr_to_generic (*iov, &rsp, (xdrproc_t)xdr_gf1_cli_mount_rsp);
-        if (ret < 0) {
-                gf_log (((call_frame_t *) myframe)->this->name, GF_LOG_ERROR,
-                        "Failed to decode xdr response");
-                goto out;
-        }
+    if (-1 == req->rpc_status) {
+        goto out;
+    }
 
-        gf_log ("cli", GF_LOG_INFO, "Received resp to mount");
+    ret = xdr_to_generic(*iov, &rsp, (xdrproc_t)xdr_gf1_cli_mount_rsp);
+    if (ret < 0) {
+        gf_log(((call_frame_t *)myframe)->this->name, GF_LOG_ERROR,
+               XDR_DECODE_FAIL);
+        goto out;
+    }
 
-        if (rsp.op_ret == 0) {
-                ret = 0;
-                cli_out ("%s", rsp.path);
-        } else {
-                /* weird sounding but easy to parse... */
-                cli_err ("%d : failed with this errno (%s)",
-                         rsp.op_errno, strerror (rsp.op_errno));
-                ret = -1;
-        }
+    gf_log("cli", GF_LOG_INFO, "Received resp to mount");
+
+    if (rsp.op_ret == 0) {
+        ret = 0;
+        cli_out("%s", rsp.path);
+    } else {
+        /* weird sounding but easy to parse... */
+        cli_err("%d : failed with this errno (%s)", rsp.op_errno,
+                strerror(rsp.op_errno));
+        ret = -1;
+    }
 
 out:
-        cli_cmd_broadcast_response (ret);
-        return ret;
+    cli_cmd_broadcast_response(ret);
+    if (rsp.path) {
+        free(rsp.path);
+    }
+    return ret;
 }
 
-int32_t
-gf_cli_mount (call_frame_t *frame, xlator_t *this, void *data)
+static int32_t
+gf_cli_mount(call_frame_t *frame, xlator_t *this, void *data)
 {
-        gf1_cli_mount_req  req  = {0,};
-        int                ret  = -1;
-        void            **dataa = data;
-        char             *label = NULL;
-        dict_t            *dict = NULL;
+    gf1_cli_mount_req req = {
+        0,
+    };
+    int ret = -1;
+    void **dataa = data;
+    char *label = NULL;
+    dict_t *dict = NULL;
 
-        if (!frame || !this || !data)
-                goto out;
+    if (!frame || !this || !data)
+        goto out;
 
-        label = dataa[0];
-        dict  = dataa[1];
+    label = dataa[0];
+    dict = dataa[1];
 
-        req.label = label;
-        ret = dict_allocate_and_serialize (dict, &req.dict.dict_val,
-                                           &req.dict.dict_len);
-        if (ret) {
-                ret = -1;
-                goto out;
-        }
+    req.label = label;
+    ret = dict_allocate_and_serialize(dict, &req.dict.dict_val,
+                                      &req.dict.dict_len);
+    if (ret) {
+        ret = -1;
+        goto out;
+    }
 
-        ret = cli_cmd_submit (NULL, &req, frame, cli_rpc_prog,
-                              GLUSTER_CLI_MOUNT, NULL,
-                              this, gf_cli_mount_cbk,
-                              (xdrproc_t)xdr_gf1_cli_mount_req);
+    ret = cli_cmd_submit(NULL, &req, frame, cli_rpc_prog, GLUSTER_CLI_MOUNT,
+                         NULL, this, gf_cli_mount_cbk,
+                         (xdrproc_t)xdr_gf1_cli_mount_req);
 
 out:
-        gf_log ("cli", GF_LOG_DEBUG, "Returning %d", ret);
-        return ret;
+    GF_FREE(req.dict.dict_val);
+    gf_log("cli", GF_LOG_DEBUG, RETURNING, ret);
+    return ret;
 }
 
 static int
-gf_cli_umount_cbk (struct rpc_req *req, struct iovec *iov,
-                   int count, void *myframe)
+gf_cli_umount_cbk(struct rpc_req *req, struct iovec *iov, int count,
+                  void *myframe)
 {
-        gf1_cli_umount_rsp rsp   = {0,};
-        int               ret   = -1;
+    gf1_cli_umount_rsp rsp = {
+        0,
+    };
+    int ret = -1;
 
-        if (-1 == req->rpc_status) {
-                goto out;
-        }
+    GF_ASSERT(myframe);
 
-        ret = xdr_to_generic (*iov, &rsp, (xdrproc_t)xdr_gf1_cli_umount_rsp);
-        if (ret < 0) {
-                gf_log (((call_frame_t *) myframe)->this->name, GF_LOG_ERROR,
-                        "Failed to decode xdr response");
-                goto out;
-        }
+    if (-1 == req->rpc_status) {
+        goto out;
+    }
 
-        gf_log ("cli", GF_LOG_INFO, "Received resp to mount");
+    ret = xdr_to_generic(*iov, &rsp, (xdrproc_t)xdr_gf1_cli_umount_rsp);
+    if (ret < 0) {
+        gf_log(((call_frame_t *)myframe)->this->name, GF_LOG_ERROR,
+               XDR_DECODE_FAIL);
+        goto out;
+    }
 
-        if (rsp.op_ret == 0)
-                ret = 0;
-        else {
-                cli_err ("umount failed");
-                ret = -1;
-        }
+    gf_log("cli", GF_LOG_INFO, "Received resp to mount");
+
+    if (rsp.op_ret == 0)
+        ret = 0;
+    else {
+        cli_err("umount failed");
+        ret = -1;
+    }
 
 out:
-        cli_cmd_broadcast_response (ret);
-        return ret;
+    cli_cmd_broadcast_response(ret);
+    return ret;
 }
 
-int32_t
-gf_cli_umount (call_frame_t *frame, xlator_t *this, void *data)
+static int32_t
+gf_cli_umount(call_frame_t *frame, xlator_t *this, void *data)
 {
-        gf1_cli_umount_req  req  = {0,};
-        int                ret  = -1;
-        dict_t            *dict = NULL;
+    gf1_cli_umount_req req = {
+        0,
+    };
+    int ret = -1;
+    dict_t *dict = NULL;
 
-        if (!frame || !this || !data)
-                goto out;
+    if (!frame || !this || !data)
+        goto out;
 
-        dict = data;
+    dict = data;
 
-        ret = dict_get_str (dict, "path", &req.path);
-        if (ret == 0)
-                ret = dict_get_int32 (dict, "lazy", &req.lazy);
+    ret = dict_get_str_sizen(dict, "path", &req.path);
+    if (ret == 0)
+        ret = dict_get_int32_sizen(dict, "lazy", &req.lazy);
 
-        if (ret) {
-                ret = -1;
-                goto out;
-        }
+    if (ret) {
+        ret = -1;
+        goto out;
+    }
 
-        ret = cli_cmd_submit (NULL, &req, frame, cli_rpc_prog,
-                              GLUSTER_CLI_UMOUNT, NULL,
-                              this, gf_cli_umount_cbk,
-                              (xdrproc_t)xdr_gf1_cli_umount_req);
+    ret = cli_cmd_submit(NULL, &req, frame, cli_rpc_prog, GLUSTER_CLI_UMOUNT,
+                         NULL, this, gf_cli_umount_cbk,
+                         (xdrproc_t)xdr_gf1_cli_umount_req);
 
- out:
-        gf_log ("cli", GF_LOG_DEBUG, "Returning %d", ret);
-        return ret;
+out:
+    gf_log("cli", GF_LOG_DEBUG, RETURNING, ret);
+    return ret;
 }
 
-void
-cmd_heal_volume_statistics_out (dict_t *dict, int  brick)
-{
-
-        uint64_t        num_entries = 0;
-        int             ret = 0;
-        char            key[256] = {0};
-        char            *hostname = NULL;
-        uint64_t        i = 0;
-        uint64_t        healed_count = 0;
-        uint64_t        split_brain_count = 0;
-        uint64_t        heal_failed_count = 0;
-        char            *start_time_str = NULL;
-        char            *end_time_str = NULL;
-        char            *crawl_type = NULL;
-        int             progress = -1;
-
-        snprintf (key, sizeof key, "%d-hostname", brick);
-        ret = dict_get_str (dict, key, &hostname);
+static void
+cmd_heal_volume_statistics_out(dict_t *dict, int brick)
+{
+    uint64_t num_entries = 0;
+    int ret = 0;
+    char key[256] = {0};
+    char *hostname = NULL;
+    uint64_t i = 0;
+    uint64_t healed_count = 0;
+    uint64_t split_brain_count = 0;
+    uint64_t heal_failed_count = 0;
+    char *start_time_str = NULL;
+    char *end_time_str = NULL;
+    char *crawl_type = NULL;
+    int progress = -1;
+
+    snprintf(key, sizeof key, "%d-hostname", brick);
+    ret = dict_get_str(dict, key, &hostname);
+    if (ret)
+        goto out;
+    cli_out("------------------------------------------------");
+    cli_out("\nCrawl statistics for brick no %d", brick);
+    cli_out("Hostname of brick %s", hostname);
+
+    snprintf(key, sizeof key, "statistics-%d-count", brick);
+    ret = dict_get_uint64(dict, key, &num_entries);
+    if (ret)
+        goto out;
+
+    for (i = 0; i < num_entries; i++) {
+        snprintf(key, sizeof key, "statistics_crawl_type-%d-%" PRIu64, brick,
+                 i);
+        ret = dict_get_str(dict, key, &crawl_type);
         if (ret)
-                goto out;
-        cli_out ("------------------------------------------------");
-        cli_out ("\nCrawl statistics for brick no %d", brick);
-        cli_out ("Hostname of brick %s", hostname);
+            goto out;
 
-        snprintf (key, sizeof key, "statistics-%d-count", brick);
-        ret = dict_get_uint64 (dict, key, &num_entries);
+        snprintf(key, sizeof key, "statistics_healed_cnt-%d-%" PRIu64, brick,
+                 i);
+        ret = dict_get_uint64(dict, key, &healed_count);
         if (ret)
-                goto out;
-
-        for (i = 0; i < num_entries; i++)
-        {
-                snprintf (key, sizeof key, "statistics_crawl_type-%d-%"PRIu64,
-                          brick, i);
-                ret = dict_get_str (dict, key, &crawl_type);
-                if (ret)
-                        goto out;
+            goto out;
 
-                snprintf (key, sizeof key, "statistics_healed_cnt-%d-%"PRIu64,
-                          brick,i);
-                ret = dict_get_uint64 (dict, key, &healed_count);
-                if (ret)
-                        goto out;
-
-                snprintf (key, sizeof key, "statistics_sb_cnt-%d-%"PRIu64,
-                          brick, i);
-                ret = dict_get_uint64 (dict, key, &split_brain_count);
-                if (ret)
-                        goto out;
-                snprintf (key, sizeof key, "statistics_heal_failed_cnt-%d-%"PRIu64,
-                          brick, i);
-                ret = dict_get_uint64 (dict, key, &heal_failed_count);
-                if (ret)
-                        goto out;
-                snprintf (key, sizeof key, "statistics_strt_time-%d-%"PRIu64,
-                          brick, i);
-                ret = dict_get_str (dict, key,  &start_time_str);
-                if (ret)
-                        goto out;
-                snprintf (key, sizeof key, "statistics_end_time-%d-%"PRIu64,
-                          brick, i);
-                ret = dict_get_str (dict, key, &end_time_str);
-                if (ret)
-                        goto out;
-                snprintf (key, sizeof key, "statistics_inprogress-%d-%"PRIu64,
-                          brick, i);
-                ret = dict_get_int32 (dict, key, &progress);
-                if (ret)
-                        goto out;
-
-                cli_out ("\nStarting time of crawl: %s", start_time_str);
-                if (progress == 1)
-                        cli_out ("Crawl is in progress");
-                else
-                        cli_out ("Ending time of crawl: %s", end_time_str);
-
-                cli_out ("Type of crawl: %s", crawl_type);
-                cli_out ("No. of entries healed: %"PRIu64,
-                         healed_count);
-                cli_out ("No. of entries in split-brain: %"PRIu64,
-                        split_brain_count);
-                cli_out ("No. of heal failed entries: %"PRIu64,
-                         heal_failed_count);
-
-        }
-
-
-out:
-        return;
-}
-
-void
-cmd_heal_volume_brick_out (dict_t *dict, int brick)
-{
-        uint64_t        num_entries = 0;
-        int             ret = 0;
-        char            key[256] = {0};
-        char           *hostname = NULL;
-        char           *path = NULL;
-        char           *status = NULL;
-        uint64_t        i = 0;
-        uint32_t        time = 0;
-        char            timestr[32] = {0};
-        char            *shd_status = NULL;
-
-        snprintf (key, sizeof key, "%d-hostname", brick);
-        ret = dict_get_str (dict, key, &hostname);
+        snprintf(key, sizeof key, "statistics_sb_cnt-%d-%" PRIu64, brick, i);
+        ret = dict_get_uint64(dict, key, &split_brain_count);
         if (ret)
-                goto out;
-        snprintf (key, sizeof key, "%d-path", brick);
-        ret = dict_get_str (dict, key, &path);
+            goto out;
+        snprintf(key, sizeof key, "statistics_heal_failed_cnt-%d-%" PRIu64,
+                 brick, i);
+        ret = dict_get_uint64(dict, key, &heal_failed_count);
         if (ret)
-                goto out;
-        cli_out ("\nBrick %s:%s", hostname, path);
-
-        snprintf (key, sizeof key, "%d-status", brick);
-        ret = dict_get_str (dict, key, &status);
-        if (status && strlen (status))
-                cli_out ("Status: %s", status);
+            goto out;
+        snprintf(key, sizeof key, "statistics_strt_time-%d-%" PRIu64, brick, i);
+        ret = dict_get_str(dict, key, &start_time_str);
+        if (ret)
+            goto out;
+        snprintf(key, sizeof key, "statistics_end_time-%d-%" PRIu64, brick, i);
+        ret = dict_get_str(dict, key, &end_time_str);
+        if (ret)
+            goto out;
+        snprintf(key, sizeof key, "statistics_inprogress-%d-%" PRIu64, brick,
+                 i);
+        ret = dict_get_int32(dict, key, &progress);
+        if (ret)
+            goto out;
 
-        snprintf (key, sizeof key, "%d-shd-status",brick);
-        ret = dict_get_str (dict, key, &shd_status);
+        cli_out("\nStarting time of crawl: %s", start_time_str);
+        if (progress == 1)
+            cli_out("Crawl is in progress");
+        else
+            cli_out("Ending time of crawl: %s", end_time_str);
 
-        if(!shd_status)
-        {
-                snprintf (key, sizeof key, "%d-count", brick);
-                ret = dict_get_uint64 (dict, key, &num_entries);
-                cli_out ("Number of entries: %"PRIu64, num_entries);
-
-
-                for (i = 0; i < num_entries; i++) {
-                        snprintf (key, sizeof key, "%d-%"PRIu64, brick, i);
-                        ret = dict_get_str (dict, key, &path);
-                        if (ret)
-                                continue;
-                        time = 0;
-                        snprintf (key, sizeof key, "%d-%"PRIu64"-time",
-                                  brick, i);
-                        ret = dict_get_uint32 (dict, key, &time);
-                        if (!time) {
-                                cli_out ("%s", path);
-                        } else {
-                                gf_time_fmt (timestr, sizeof timestr,
-                                             time, gf_timefmt_FT);
-                                if (i == 0) {
-                                cli_out ("at                    path on brick");
-                                cli_out ("-----------------------------------");
-                                }
-                                cli_out ("%s %s", timestr, path);
-                        }
-                }
-        }
+        cli_out("Type of crawl: %s", crawl_type);
+        cli_out("No. of entries healed: %" PRIu64, healed_count);
+        cli_out("No. of entries in split-brain: %" PRIu64, split_brain_count);
+        cli_out("No. of heal failed entries: %" PRIu64, heal_failed_count);
+    }
 
 out:
-        return;
+    return;
 }
 
+static void
+cmd_heal_volume_brick_out(dict_t *dict, int brick)
+{
+    uint64_t num_entries = 0;
+    int ret = 0;
+    char key[64] = {0};
+    char *hostname = NULL;
+    char *path = NULL;
+    char *status = NULL;
+    uint64_t i = 0;
+    uint32_t time = 0;
+    char timestr[GF_TIMESTR_SIZE] = {0};
+    char *shd_status = NULL;
+
+    snprintf(key, sizeof key, "%d-hostname", brick);
+    ret = dict_get_str(dict, key, &hostname);
+    if (ret)
+        goto out;
+    snprintf(key, sizeof key, "%d-path", brick);
+    ret = dict_get_str(dict, key, &path);
+    if (ret)
+        goto out;
+    cli_out("\nBrick %s:%s", hostname, path);
+
+    snprintf(key, sizeof key, "%d-status", brick);
+    ret = dict_get_str(dict, key, &status);
+    if (status && status[0] != '\0')
+        cli_out("Status: %s", status);
+
+    snprintf(key, sizeof key, "%d-shd-status", brick);
+    ret = dict_get_str(dict, key, &shd_status);
+
+    if (!shd_status) {
+        snprintf(key, sizeof key, "%d-count", brick);
+        ret = dict_get_uint64(dict, key, &num_entries);
+        cli_out("Number of entries: %" PRIu64, num_entries);
+
+        for (i = 0; i < num_entries; i++) {
+            snprintf(key, sizeof key, "%d-%" PRIu64, brick, i);
+            ret = dict_get_str(dict, key, &path);
+            if (ret)
+                continue;
+            time = 0;
+            snprintf(key, sizeof key, "%d-%" PRIu64 "-time", brick, i);
+            ret = dict_get_uint32(dict, key, &time);
+            if (ret || !time) {
+                cli_out("%s", path);
+            } else {
+                gf_time_fmt(timestr, sizeof timestr, time, gf_timefmt_FT);
+                if (i == 0) {
+                    cli_out("at                    path on brick");
+                    cli_out("-----------------------------------");
+                }
+                cli_out("%s %s", timestr, path);
+            }
+        }
+    }
+
+out:
+    return;
+}
 
-void
-cmd_heal_volume_statistics_heal_count_out (dict_t *dict, int brick)
-{
-        uint64_t        num_entries = 0;
-        int             ret = 0;
-        char            key[256] = {0};
-        char           *hostname = NULL;
-        char           *path = NULL;
-        char           *status = NULL;
-        char            *shd_status = NULL;
-
-        snprintf (key, sizeof key, "%d-hostname", brick);
-        ret = dict_get_str (dict, key, &hostname);
-        if (ret)
-                goto out;
-        snprintf (key, sizeof key, "%d-path", brick);
-        ret = dict_get_str (dict, key, &path);
+static void
+cmd_heal_volume_statistics_heal_count_out(dict_t *dict, int brick)
+{
+    uint64_t num_entries = 0;
+    int ret = 0;
+    char key[64] = {0};
+    char *hostname = NULL;
+    char *path = NULL;
+    char *status = NULL;
+    char *shd_status = NULL;
+
+    snprintf(key, sizeof key, "%d-hostname", brick);
+    ret = dict_get_str(dict, key, &hostname);
+    if (ret)
+        goto out;
+    snprintf(key, sizeof key, "%d-path", brick);
+    ret = dict_get_str(dict, key, &path);
+    if (ret)
+        goto out;
+    cli_out("\nBrick %s:%s", hostname, path);
+
+    snprintf(key, sizeof key, "%d-status", brick);
+    ret = dict_get_str(dict, key, &status);
+    if (status && strlen(status))
+        cli_out("Status: %s", status);
+
+    snprintf(key, sizeof key, "%d-shd-status", brick);
+    ret = dict_get_str(dict, key, &shd_status);
+
+    if (!shd_status) {
+        snprintf(key, sizeof key, "%d-hardlinks", brick);
+        ret = dict_get_uint64(dict, key, &num_entries);
         if (ret)
-                goto out;
-        cli_out ("\nBrick %s:%s", hostname, path);
-
-        snprintf (key, sizeof key, "%d-status", brick);
-        ret = dict_get_str (dict, key, &status);
-        if (status && strlen (status))
-                cli_out ("Status: %s", status);
-
-        snprintf (key, sizeof key, "%d-shd-status",brick);
-        ret = dict_get_str (dict, key, &shd_status);
-
-        if(!shd_status)
-        {
-                snprintf (key, sizeof key, "%d-hardlinks", brick);
-                ret = dict_get_uint64 (dict, key, &num_entries);
-                if (ret)
-                        cli_out ("No gathered input for this brick");
-                else
-                        cli_out ("Number of entries: %"PRIu64, num_entries);
-
-
-        }
+            cli_out("No gathered input for this brick");
+        else
+            cli_out("Number of entries: %" PRIu64, num_entries);
+    }
 
 out:
-        return;
+    return;
 }
 
-int
-gf_is_cli_heal_get_command (gf_xl_afr_op_t heal_op)
-{
-        /* If the command is get command value is 1 otherwise 0, for
-           invalid commands -1 */
-        int    get_cmds[GF_SHD_OP_HEAL_DISABLE + 1] = {
-                       [GF_SHD_OP_INVALID] = -1,
-                       [GF_SHD_OP_HEAL_INDEX] = 0,
-                       [GF_SHD_OP_HEAL_FULL] = 0,
-                       [GF_SHD_OP_INDEX_SUMMARY] = 1,
-                       [GF_SHD_OP_HEALED_FILES] = 1,
-                       [GF_SHD_OP_HEAL_FAILED_FILES] = 1,
-                       [GF_SHD_OP_SPLIT_BRAIN_FILES] = 1,
-                       [GF_SHD_OP_STATISTICS] = 1,
-                       [GF_SHD_OP_STATISTICS_HEAL_COUNT] = 1,
-                       [GF_SHD_OP_STATISTICS_HEAL_COUNT_PER_REPLICA] = 1,
-                       [GF_SHD_OP_HEAL_ENABLE] = 0,
-                       [GF_SHD_OP_HEAL_DISABLE] = 0,
-                };
-
-        if (heal_op > GF_SHD_OP_INVALID && heal_op <= GF_SHD_OP_HEAL_DISABLE)
-                return get_cmds[heal_op] == 1;
-        return _gf_false;
+static int
+gf_is_cli_heal_get_command(gf_xl_afr_op_t heal_op)
+{
+    /* If the command is get command value is 1 otherwise 0, for
+       invalid commands -1 */
+    static int get_cmds[GF_SHD_OP_HEAL_DISABLE + 1] = {
+        [GF_SHD_OP_INVALID] = -1,
+        [GF_SHD_OP_HEAL_INDEX] = 0,
+        [GF_SHD_OP_HEAL_FULL] = 0,
+        [GF_SHD_OP_INDEX_SUMMARY] = 1,
+        [GF_SHD_OP_SPLIT_BRAIN_FILES] = 1,
+        [GF_SHD_OP_STATISTICS] = 1,
+        [GF_SHD_OP_STATISTICS_HEAL_COUNT] = 1,
+        [GF_SHD_OP_STATISTICS_HEAL_COUNT_PER_REPLICA] = 1,
+        [GF_SHD_OP_HEAL_ENABLE] = 0,
+        [GF_SHD_OP_HEAL_DISABLE] = 0,
+    };
+
+    if (heal_op > GF_SHD_OP_INVALID && heal_op <= GF_SHD_OP_HEAL_DISABLE)
+        return get_cmds[heal_op] == 1;
+    return _gf_false;
 }
 
 int
-gf_cli_heal_volume_cbk (struct rpc_req *req, struct iovec *iov,
-                             int count, void *myframe)
-{
-        gf_cli_rsp              rsp   = {0,};
-        int                     ret   = -1;
-        cli_local_t             *local = NULL;
-        char                    *volname = NULL;
-        call_frame_t            *frame = NULL;
-        dict_t                  *input_dict = NULL;
-        dict_t                  *dict = NULL;
-        int                     brick_count = 0;
-        int                     i = 0;
-        gf_xl_afr_op_t          heal_op = GF_SHD_OP_INVALID;
-        char                    *operation = NULL;
-        char                    *substr = NULL;
-        char                    *heal_op_str = NULL;
-
-        if (-1 == req->rpc_status) {
-                goto out;
-        }
-
-        frame = myframe;
-
-        ret = xdr_to_generic (*iov, &rsp, (xdrproc_t)xdr_gf_cli_rsp);
-        if (ret < 0) {
-                gf_log (frame->this->name, GF_LOG_ERROR,
-                        "Failed to decode xdr response");
-                goto out;
-        }
-
-        if (frame)
-                local = frame->local;
-
-        if (local) {
-                input_dict = local->dict;
-                ret = dict_get_int32 (input_dict, "heal-op",
-                                      (int32_t*)&heal_op);
-        }
-//TODO: Proper XML output
-//#if (HAVE_LIB_XML)
-//        if (global_state->mode & GLUSTER_MODE_XML) {
-//                ret = cli_xml_output_dict ("volHeal", dict, rsp.op_ret,
-//                                           rsp.op_errno, rsp.op_errstr);
-//                if (ret)
-//                        gf_log ("cli", GF_LOG_ERROR,
-//                                "Error outputting to xml");
-//                goto out;
-//        }
-//#endif
-
-        ret = dict_get_str (input_dict, "volname", &volname);
-        if (ret) {
-                gf_log (frame->this->name, GF_LOG_ERROR, "failed to get volname");
-                goto out;
-        }
-
-        gf_log ("cli", GF_LOG_INFO, "Received resp to heal volume");
-
-        operation = "Gathering ";
-        substr = "";
-        switch (heal_op) {
-        case    GF_SHD_OP_HEAL_INDEX:
-                operation   = "Launching heal operation ";
-                heal_op_str = "to perform index self heal";
-                substr      = "\nUse heal info commands to check"
-                              " status";
-                break;
-        case    GF_SHD_OP_HEAL_FULL:
-                operation   = "Launching heal operation ";
-                heal_op_str = "to perform full self heal";
-                substr      = "\nUse heal info commands to check"
-                              " status";
-                break;
-        case    GF_SHD_OP_INDEX_SUMMARY:
-                heal_op_str = "list of entries to be healed";
-                break;
-        case    GF_SHD_OP_HEALED_FILES:
-                heal_op_str = "list of healed entries";
-                break;
-        case    GF_SHD_OP_HEAL_FAILED_FILES:
-                heal_op_str = "list of heal failed entries";
-                break;
-        case    GF_SHD_OP_SPLIT_BRAIN_FILES:
-                heal_op_str = "list of split brain entries";
-                break;
-        case    GF_SHD_OP_STATISTICS:
-                heal_op_str =  "crawl statistics";
-                break;
-        case    GF_SHD_OP_STATISTICS_HEAL_COUNT:
-                heal_op_str = "count of entries to be healed";
-                break;
-        case    GF_SHD_OP_STATISTICS_HEAL_COUNT_PER_REPLICA:
-                heal_op_str = "count of entries to be healed per replica";
-                break;
-        /* The below 2 cases are never hit; they're coded only to make
-         * compiler warnings go away.*/
-        case    GF_SHD_OP_SBRAIN_HEAL_FROM_BIGGER_FILE:
-        case    GF_SHD_OP_SBRAIN_HEAL_FROM_BRICK:
-                break;
-
-        case    GF_SHD_OP_INVALID:
-                heal_op_str = "invalid heal op";
-                break;
-        case    GF_SHD_OP_HEAL_ENABLE:
-                operation   = "";
-                heal_op_str = "Enable heal";
-                break;
-        case    GF_SHD_OP_HEAL_DISABLE:
-                operation   = "";
-                heal_op_str = "Disable heal";
-                break;
-        }
-
-        if (rsp.op_ret) {
-                if (strcmp (rsp.op_errstr, "")) {
-                        cli_err ("%s", rsp.op_errstr);
-                } else {
-                        cli_err ("%s%s on volume %s has been unsuccessful",
-                                 operation, heal_op_str, volname);
-                }
-
-                ret = rsp.op_ret;
-                goto out;
-        } else {
-                cli_out ("%s%s on volume %s has been successful %s", operation,
-                         heal_op_str, volname, substr);
+gf_cli_heal_volume_cbk(struct rpc_req *req, struct iovec *iov, int count,
+                       void *myframe)
+{
+    gf_cli_rsp rsp = {
+        0,
+    };
+    int ret = -1;
+    cli_local_t *local = NULL;
+    char *volname = NULL;
+    call_frame_t *frame = NULL;
+    dict_t *dict = NULL;
+    int brick_count = 0;
+    int i = 0;
+    gf_xl_afr_op_t heal_op = GF_SHD_OP_INVALID;
+    const char *operation = NULL;
+    const char *substr = NULL;
+    const char *heal_op_str = NULL;
+
+    GF_ASSERT(myframe);
+
+    if (-1 == req->rpc_status) {
+        goto out;
+    }
+
+    frame = myframe;
+
+    GF_ASSERT(frame->local);
+
+    local = frame->local;
+
+    ret = xdr_to_generic(*iov, &rsp, (xdrproc_t)xdr_gf_cli_rsp);
+    if (ret < 0) {
+        gf_log(frame->this->name, GF_LOG_ERROR, XDR_DECODE_FAIL);
+        goto out;
+    }
+
+    ret = dict_get_int32_sizen(local->dict, "heal-op", (int32_t *)&heal_op);
+    // TODO: Proper XML output
+    //#if (HAVE_LIB_XML)
+    //        if (global_state->mode & GLUSTER_MODE_XML) {
+    //                ret = cli_xml_output_dict ("volHeal", dict, rsp.op_ret,
+    //                                           rsp.op_errno, rsp.op_errstr);
+    //                if (ret)
+    //                        gf_log ("cli", GF_LOG_ERROR, XML_ERROR);
+    //                goto out;
+    //        }
+    //#endif
+
+    ret = dict_get_str_sizen(local->dict, "volname", &volname);
+    if (ret) {
+        gf_log(frame->this->name, GF_LOG_ERROR, "failed to get volname");
+        goto out;
+    }
+
+    gf_log("cli", GF_LOG_INFO, "Received resp to heal volume");
+
+    operation = "Gathering ";
+    substr = "";
+    switch (heal_op) {
+        case GF_SHD_OP_HEAL_INDEX:
+            operation = "Launching heal operation ";
+            heal_op_str = "to perform index self heal";
+            substr = "\nUse heal info commands to check status.";
+            break;
+        case GF_SHD_OP_HEAL_FULL:
+            operation = "Launching heal operation ";
+            heal_op_str = "to perform full self heal";
+            substr = "\nUse heal info commands to check status.";
+            break;
+        case GF_SHD_OP_INDEX_SUMMARY:
+            heal_op_str = "list of entries to be healed";
+            break;
+        case GF_SHD_OP_SPLIT_BRAIN_FILES:
+            heal_op_str = "list of split brain entries";
+            break;
+        case GF_SHD_OP_STATISTICS:
+            heal_op_str = "crawl statistics";
+            break;
+        case GF_SHD_OP_STATISTICS_HEAL_COUNT:
+            heal_op_str = "count of entries to be healed";
+            break;
+        case GF_SHD_OP_STATISTICS_HEAL_COUNT_PER_REPLICA:
+            heal_op_str = "count of entries to be healed per replica";
+            break;
+        case GF_SHD_OP_SBRAIN_HEAL_FROM_BIGGER_FILE:
+        case GF_SHD_OP_SBRAIN_HEAL_FROM_LATEST_MTIME:
+        case GF_SHD_OP_SBRAIN_HEAL_FROM_BRICK:
+        case GF_SHD_OP_HEAL_SUMMARY:
+        case GF_SHD_OP_HEALED_FILES:
+        case GF_SHD_OP_HEAL_FAILED_FILES:
+            /* These cases are never hit; they're coded just to silence the
+             * compiler warnings.*/
+            break;
+
+        case GF_SHD_OP_INVALID:
+            heal_op_str = "invalid heal op";
+            break;
+        case GF_SHD_OP_HEAL_ENABLE:
+            operation = "";
+            heal_op_str = "Enable heal";
+            break;
+        case GF_SHD_OP_HEAL_DISABLE:
+            operation = "";
+            heal_op_str = "Disable heal";
+            break;
+        case GF_SHD_OP_GRANULAR_ENTRY_HEAL_ENABLE:
+            operation = "";
+            heal_op_str = "Enable granular entry heal";
+            break;
+        case GF_SHD_OP_GRANULAR_ENTRY_HEAL_DISABLE:
+            operation = "";
+            heal_op_str = "Disable granular entry heal";
+            break;
+    }
+
+    if (rsp.op_ret) {
+        if (strcmp(rsp.op_errstr, "")) {
+            cli_err("%s%s on volume %s has been unsuccessful:", operation,
+                    heal_op_str, volname);
+            cli_err("%s", rsp.op_errstr);
         }
-
         ret = rsp.op_ret;
-        if (!gf_is_cli_heal_get_command (heal_op))
-                goto out;
-
-        dict = dict_new ();
-        if (!dict) {
-                ret = -1;
-                goto out;
-        }
-
-        ret = dict_unserialize (rsp.dict.dict_val,
-                                rsp.dict.dict_len,
-                                &dict);
-
-        if (ret) {
-                gf_log ("", GF_LOG_ERROR,
-                                "Unable to allocate memory");
-                goto out;
-        } else {
-                dict->extra_stdfree = rsp.dict.dict_val;
-        }
-        ret = dict_get_int32 (dict, "count", &brick_count);
-        if (ret)
-                goto out;
-
-        if (!brick_count) {
-                cli_err ("All bricks of volume %s are down.", volname);
-                goto out;
-        }
-
-        switch (heal_op) {
+        goto out;
+    } else {
+        cli_out("%s%s on volume %s has been successful %s", operation,
+                heal_op_str, volname, substr);
+    }
+
+    ret = rsp.op_ret;
+    if (!gf_is_cli_heal_get_command(heal_op))
+        goto out;
+
+    dict = dict_new();
+    if (!dict) {
+        ret = -1;
+        goto out;
+    }
+
+    ret = dict_unserialize(rsp.dict.dict_val, rsp.dict.dict_len, &dict);
+
+    if (ret) {
+        gf_log("", GF_LOG_ERROR, DICT_UNSERIALIZE_FAIL);
+        goto out;
+    }
+
+    ret = dict_get_int32_sizen(dict, "count", &brick_count);
+    if (ret)
+        goto out;
+
+    if (!brick_count) {
+        cli_err("All bricks of volume %s are down.", volname);
+        ret = -1;
+        goto out;
+    }
+
+    switch (heal_op) {
         case GF_SHD_OP_STATISTICS:
-                for (i = 0; i < brick_count; i++)
-                        cmd_heal_volume_statistics_out (dict, i);
-                break;
+            for (i = 0; i < brick_count; i++)
+                cmd_heal_volume_statistics_out(dict, i);
+            break;
         case GF_SHD_OP_STATISTICS_HEAL_COUNT:
         case GF_SHD_OP_STATISTICS_HEAL_COUNT_PER_REPLICA:
-                for (i = 0; i < brick_count; i++)
-                        cmd_heal_volume_statistics_heal_count_out (dict,
-                                                                   i);
-                break;
+            for (i = 0; i < brick_count; i++)
+                cmd_heal_volume_statistics_heal_count_out(dict, i);
+            break;
         case GF_SHD_OP_INDEX_SUMMARY:
-        case GF_SHD_OP_HEALED_FILES:
-        case GF_SHD_OP_HEAL_FAILED_FILES:
         case GF_SHD_OP_SPLIT_BRAIN_FILES:
-                for (i = 0; i < brick_count; i++)
-                        cmd_heal_volume_brick_out (dict, i);
-                break;
+            for (i = 0; i < brick_count; i++)
+                cmd_heal_volume_brick_out(dict, i);
+            break;
         default:
-                break;
-        }
+            break;
+    }
 
-        ret = rsp.op_ret;
+    ret = rsp.op_ret;
 
 out:
-        cli_cmd_broadcast_response (ret);
-        free (rsp.op_errstr);
-        if (dict)
-                dict_unref (dict);
-        return ret;
+    cli_cmd_broadcast_response(ret);
+    gf_free_xdr_cli_rsp(rsp);
+    if (dict)
+        dict_unref(dict);
+    return ret;
 }
 
-int32_t
-gf_cli_heal_volume (call_frame_t *frame, xlator_t *this,
-                         void *data)
+static int32_t
+gf_cli_heal_volume(call_frame_t *frame, xlator_t *this, void *data)
 {
-        gf_cli_req              req = {{0,}};
-        int                     ret = 0;
-        dict_t                  *dict = NULL;
-
-        if (!frame || !this ||  !data) {
-                ret = -1;
-                goto out;
-        }
+    gf_cli_req req = {{
+        0,
+    }};
+    int ret = 0;
+    dict_t *dict = NULL;
 
-        dict = data;
+    dict = data;
 
-        ret = cli_to_glusterd (&req, frame, gf_cli_heal_volume_cbk,
-                               (xdrproc_t) xdr_gf_cli_req, dict,
-                               GLUSTER_CLI_HEAL_VOLUME, this, cli_rpc_prog,
-                               NULL);
+    ret = cli_to_glusterd(&req, frame, gf_cli_heal_volume_cbk,
+                          (xdrproc_t)xdr_gf_cli_req, dict,
+                          GLUSTER_CLI_HEAL_VOLUME, this, cli_rpc_prog, NULL);
 
-out:
-        gf_log ("cli", GF_LOG_DEBUG, "Returning %d", ret);
+    gf_log("cli", GF_LOG_DEBUG, RETURNING, ret);
 
-        GF_FREE (req.dict.dict_val);
+    GF_FREE(req.dict.dict_val);
 
-        return ret;
+    return ret;
 }
 
-int32_t
-gf_cli_statedump_volume_cbk (struct rpc_req *req, struct iovec *iov,
-                                int count, void *myframe)
-{
-        gf_cli_rsp                      rsp = {0,};
-        int                             ret = -1;
-        char                            msg[1024] = {0,};
-
-        if (-1 == req->rpc_status)
-                goto out;
-        ret = xdr_to_generic (*iov, &rsp,
-                              (xdrproc_t)xdr_gf_cli_rsp);
-        if (ret < 0) {
-                gf_log (((call_frame_t *) myframe)->this->name, GF_LOG_ERROR,
-                        "Failed to decode xdr response");
-                goto out;
-        }
-        gf_log ("cli", GF_LOG_DEBUG, "Received response to statedump");
-        if (rsp.op_ret)
-                snprintf (msg, sizeof(msg), "%s", rsp.op_errstr);
-        else
-                snprintf (msg, sizeof (msg), "Volume statedump successful");
-
-        if (global_state->mode & GLUSTER_MODE_XML) {
-                ret = cli_xml_output_str ("volStatedump", msg, rsp.op_ret,
-                                          rsp.op_errno, rsp.op_errstr);
-                if (ret)
-                        gf_log ("cli", GF_LOG_ERROR,
-                                "Error outputting to xml");
-                goto out;
-        }
+static int32_t
+gf_cli_statedump_volume_cbk(struct rpc_req *req, struct iovec *iov, int count,
+                            void *myframe)
+{
+    gf_cli_rsp rsp = {
+        0,
+    };
+    int ret = -1;
+    char msg[1024] = "Volume statedump successful";
+
+    GF_ASSERT(myframe);
+
+    if (-1 == req->rpc_status)
+        goto out;
+
+    ret = xdr_to_generic(*iov, &rsp, (xdrproc_t)xdr_gf_cli_rsp);
+    if (ret < 0) {
+        gf_log(((call_frame_t *)myframe)->this->name, GF_LOG_ERROR,
+               XDR_DECODE_FAIL);
+        goto out;
+    }
+    gf_log("cli", GF_LOG_DEBUG, "Received response to statedump");
+    if (rsp.op_ret)
+        snprintf(msg, sizeof(msg), "%s", rsp.op_errstr);
+
+    if (global_state->mode & GLUSTER_MODE_XML) {
+        ret = cli_xml_output_str("volStatedump", msg, rsp.op_ret, rsp.op_errno,
+                                 rsp.op_errstr);
+        if (ret)
+            gf_log("cli", GF_LOG_ERROR, XML_ERROR);
+        goto out;
+    }
 
-        if (rsp.op_ret)
-                cli_err ("volume statedump: failed: %s", msg);
-        else
-                cli_out ("volume statedump: success");
-        ret = rsp.op_ret;
+    if (rsp.op_ret)
+        cli_err("volume statedump: failed: %s", msg);
+    else
+        cli_out("volume statedump: success");
+    ret = rsp.op_ret;
 
 out:
-        cli_cmd_broadcast_response (ret);
-        return ret;
+    cli_cmd_broadcast_response(ret);
+    gf_free_xdr_cli_rsp(rsp);
+    return ret;
 }
 
-int32_t
-gf_cli_statedump_volume (call_frame_t *frame, xlator_t *this,
-                            void *data)
+static int32_t
+gf_cli_statedump_volume(call_frame_t *frame, xlator_t *this, void *data)
 {
-        gf_cli_req                      req = {{0,}};
-        dict_t                          *options = NULL;
-        int                             ret = -1;
-
-        if (!frame || !this || !data)
-                goto out;
-
-        options = data;
+    gf_cli_req req = {{
+        0,
+    }};
+    dict_t *options = NULL;
+    int ret = -1;
 
-        ret = cli_to_glusterd (&req, frame, gf_cli_statedump_volume_cbk,
-                               (xdrproc_t) xdr_gf_cli_req, options,
-                               GLUSTER_CLI_STATEDUMP_VOLUME, this, cli_rpc_prog,
-                               NULL);
+    options = data;
 
-out:
-        gf_log ("cli", GF_LOG_DEBUG, "Returning %d", ret);
+    ret = cli_to_glusterd(
+        &req, frame, gf_cli_statedump_volume_cbk, (xdrproc_t)xdr_gf_cli_req,
+        options, GLUSTER_CLI_STATEDUMP_VOLUME, this, cli_rpc_prog, NULL);
+    gf_log("cli", GF_LOG_DEBUG, RETURNING, ret);
 
-        GF_FREE (req.dict.dict_val);
-        return ret;
+    GF_FREE(req.dict.dict_val);
+    return ret;
 }
 
-int32_t
-gf_cli_list_volume_cbk (struct rpc_req *req, struct iovec *iov,
-                                int count, void *myframe)
-{
-        int             ret = -1;
-        gf_cli_rsp      rsp = {0,};
-        dict_t          *dict = NULL;
-        int             vol_count = 0;;
-        char            *volname = NULL;
-        char            key[1024] = {0,};
-        int             i = 0;
-
-        if (-1 == req->rpc_status)
-                goto out;
-        ret = xdr_to_generic (*iov, &rsp,
-                              (xdrproc_t)xdr_gf_cli_rsp);
-        if (ret < 0) {
-                gf_log (((call_frame_t *) myframe)->this->name, GF_LOG_ERROR,
-                        "Failed to decode xdr response");
-                goto out;
-        }
-
-        dict = dict_new ();
-        if (!dict) {
-                ret = -1;
-                goto out;
-        }
+static int32_t
+gf_cli_list_volume_cbk(struct rpc_req *req, struct iovec *iov, int count,
+                       void *myframe)
+{
+    int ret = -1;
+    gf_cli_rsp rsp = {
+        0,
+    };
+    dict_t *dict = NULL;
+    int vol_count = 0;
+    ;
+    char *volname = NULL;
+    char key[1024] = {
+        0,
+    };
+    int i = 0;
+
+    GF_ASSERT(myframe);
+
+    if (-1 == req->rpc_status)
+        goto out;
+
+    ret = xdr_to_generic(*iov, &rsp, (xdrproc_t)xdr_gf_cli_rsp);
+    if (ret < 0) {
+        gf_log(((call_frame_t *)myframe)->this->name, GF_LOG_ERROR,
+               XDR_DECODE_FAIL);
+        goto out;
+    }
+
+    dict = dict_new();
+    if (!dict) {
+        ret = -1;
+        goto out;
+    }
+
+    ret = dict_unserialize(rsp.dict.dict_val, rsp.dict.dict_len, &dict);
+    if (ret) {
+        gf_log("cli", GF_LOG_ERROR, DICT_UNSERIALIZE_FAIL);
+        goto out;
+    }
+
+    if (global_state->mode & GLUSTER_MODE_XML) {
+        ret = cli_xml_output_vol_list(dict, rsp.op_ret, rsp.op_errno,
+                                      rsp.op_errstr);
+        if (ret)
+            gf_log("cli", GF_LOG_ERROR, XML_ERROR);
+        goto out;
+    }
+
+    if (rsp.op_ret)
+        cli_err("%s", rsp.op_errstr);
+    else {
+        ret = dict_get_int32_sizen(dict, "count", &vol_count);
+        if (ret)
+            goto out;
 
-        ret = dict_unserialize (rsp.dict.dict_val, rsp.dict.dict_len, &dict);
-        if (ret) {
-                gf_log ("cli", GF_LOG_ERROR, "Unable to allocate memory");
-                goto out;
+        if (vol_count == 0) {
+            cli_err("No volumes present in cluster");
+            goto out;
         }
-
-        if (global_state->mode & GLUSTER_MODE_XML) {
-                ret = cli_xml_output_vol_list (dict, rsp.op_ret, rsp.op_errno,
-                                               rsp.op_errstr);
-                if (ret)
-                        gf_log ("cli", GF_LOG_ERROR,
-                                "Error outputting to xml");
+        for (i = 0; i < vol_count; i++) {
+            ret = snprintf(key, sizeof(key), "volume%d", i);
+            ret = dict_get_strn(dict, key, ret, &volname);
+            if (ret)
                 goto out;
+            cli_out("%s", volname);
         }
+    }
 
-        if (rsp.op_ret)
-                cli_err ("%s", rsp.op_errstr);
-        else {
-                ret = dict_get_int32 (dict, "count", &vol_count);
-                if (ret)
-                        goto out;
-
-                if (vol_count == 0) {
-                        cli_err ("No volumes present in cluster");
-                        goto out;
-                }
-                for (i = 0; i < vol_count; i++) {
-                        memset (key, 0, sizeof (key));
-                        snprintf (key, sizeof (key), "volume%d", i);
-                        ret = dict_get_str (dict, key, &volname);
-                        if (ret)
-                                goto out;
-                        cli_out ("%s", volname);
-                }
-        }
-
-        ret = rsp.op_ret;
+    ret = rsp.op_ret;
 
 out:
-        cli_cmd_broadcast_response (ret);
-        return ret;
+    cli_cmd_broadcast_response(ret);
+    gf_free_xdr_cli_rsp(rsp);
+
+    if (dict)
+        dict_unref(dict);
+    return ret;
 }
 
-int32_t
-gf_cli_list_volume (call_frame_t *frame, xlator_t *this, void *data)
+static int32_t
+gf_cli_list_volume(call_frame_t *frame, xlator_t *this, void *data)
 {
-        int             ret = -1;
-        gf_cli_req      req = {{0,}};
-
-        if (!frame || !this)
-                goto out;
+    int ret = -1;
+    gf_cli_req req = {{
+        0,
+    }};
 
-        ret = cli_cmd_submit (NULL, &req, frame, cli_rpc_prog,
-                              GLUSTER_CLI_LIST_VOLUME, NULL,
-                              this, gf_cli_list_volume_cbk,
-                              (xdrproc_t)xdr_gf_cli_req);
-
-out:
-        gf_log ("cli", GF_LOG_DEBUG, "Returning %d", ret);
-        return ret;
+    ret = cli_cmd_submit(NULL, &req, frame, cli_rpc_prog,
+                         GLUSTER_CLI_LIST_VOLUME, NULL, this,
+                         gf_cli_list_volume_cbk, (xdrproc_t)xdr_gf_cli_req);
+    gf_log("cli", GF_LOG_DEBUG, RETURNING, ret);
+    return ret;
 }
 
-int32_t
-gf_cli_clearlocks_volume_cbk (struct rpc_req *req, struct iovec *iov,
-                                  int count, void *myframe)
-{
-        gf_cli_rsp                      rsp = {0,};
-        int                             ret = -1;
-        char                            *lk_summary = NULL;
-        char                            *volname = NULL;
-        dict_t                          *dict = NULL;
-
-        if (-1 == req->rpc_status)
-                goto out;
-        ret = xdr_to_generic (*iov, &rsp,
-                              (xdrproc_t)xdr_gf_cli_rsp);
-        if (ret < 0) {
-                gf_log (((call_frame_t *) myframe)->this->name, GF_LOG_ERROR,
-                        "Failed to decode xdr response");
-                goto out;
+static int32_t
+gf_cli_clearlocks_volume_cbk(struct rpc_req *req, struct iovec *iov, int count,
+                             void *myframe)
+{
+    gf_cli_rsp rsp = {
+        0,
+    };
+    int ret = -1;
+    char *lk_summary = NULL;
+    dict_t *dict = NULL;
+
+    GF_ASSERT(myframe);
+
+    if (-1 == req->rpc_status)
+        goto out;
+
+    ret = xdr_to_generic(*iov, &rsp, (xdrproc_t)xdr_gf_cli_rsp);
+    if (ret < 0) {
+        gf_log(((call_frame_t *)myframe)->this->name, GF_LOG_ERROR,
+               XDR_DECODE_FAIL);
+        goto out;
+    }
+    gf_log("cli", GF_LOG_DEBUG, "Received response to clear-locks");
+
+    if (rsp.op_ret) {
+        cli_err("Volume clear-locks unsuccessful");
+        cli_err("%s", rsp.op_errstr);
+
+    } else {
+        if (!rsp.dict.dict_len) {
+            cli_err("Possibly no locks cleared");
+            ret = 0;
+            goto out;
         }
-        gf_log ("cli", GF_LOG_DEBUG, "Received response to clear-locks");
-
-        if (rsp.op_ret) {
-                cli_err ("Volume clear-locks unsuccessful");
-                cli_err ("%s", rsp.op_errstr);
-
-        } else {
-                if (!rsp.dict.dict_len) {
-                        cli_err ("Possibly no locks cleared");
-                        ret = 0;
-                        goto out;
-                }
 
-                dict = dict_new ();
-
-                if (!dict) {
-                        ret = -1;
-                        goto out;
-                }
+        dict = dict_new();
 
-                ret = dict_unserialize (rsp.dict.dict_val,
-                                        rsp.dict.dict_len,
-                                        &dict);
-
-                if (ret) {
-                        gf_log ("cli", GF_LOG_ERROR,
-                                "Unable to serialize response dictionary");
-                        goto out;
-                }
+        if (!dict) {
+            ret = -1;
+            goto out;
+        }
 
-                ret = dict_get_str (dict, "volname", &volname);
-                if (ret) {
-                        gf_log ("cli", GF_LOG_ERROR, "Unable to get volname "
-                                "from dictionary");
-                        goto out;
-                }
+        ret = dict_unserialize(rsp.dict.dict_val, rsp.dict.dict_len, &dict);
 
-                ret = dict_get_str (dict, "lk-summary", &lk_summary);
-                if (ret) {
-                        gf_log ("cli", GF_LOG_ERROR, "Unable to get lock "
-                                "summary from dictionary");
-                        goto out;
-                }
-                cli_out ("Volume clear-locks successful");
-                cli_out ("%s", lk_summary);
+        if (ret) {
+            gf_log("cli", GF_LOG_ERROR, DICT_UNSERIALIZE_FAIL);
+            goto out;
+        }
 
+        ret = dict_get_str(dict, "lk-summary", &lk_summary);
+        if (ret) {
+            gf_log("cli", GF_LOG_ERROR,
+                   "Unable to get lock summary from dictionary");
+            goto out;
         }
+        cli_out("Volume clear-locks successful");
+        cli_out("%s", lk_summary);
+    }
 
-        ret = rsp.op_ret;
+    ret = rsp.op_ret;
 
 out:
-        if (dict)
-                dict_unref (dict);
-        cli_cmd_broadcast_response (ret);
-        return ret;
+    if (dict)
+        dict_unref(dict);
+    cli_cmd_broadcast_response(ret);
+    gf_free_xdr_cli_rsp(rsp);
+    return ret;
 }
 
-int32_t
-gf_cli_clearlocks_volume (call_frame_t *frame, xlator_t *this,
-                             void *data)
+static int32_t
+gf_cli_clearlocks_volume(call_frame_t *frame, xlator_t *this, void *data)
 {
-        gf_cli_req                      req = {{0,}};
-        dict_t                          *options = NULL;
-        int                             ret = -1;
+    gf_cli_req req = {{
+        0,
+    }};
+    dict_t *options = NULL;
+    int ret = -1;
 
-        if (!frame || !this || !data)
-                goto out;
+    options = data;
 
-        options = data;
+    ret = cli_to_glusterd(
+        &req, frame, gf_cli_clearlocks_volume_cbk, (xdrproc_t)xdr_gf_cli_req,
+        options, GLUSTER_CLI_CLRLOCKS_VOLUME, this, cli_rpc_prog, NULL);
+    gf_log("cli", GF_LOG_DEBUG, RETURNING, ret);
 
-        ret = cli_to_glusterd (&req, frame, gf_cli_clearlocks_volume_cbk,
-                               (xdrproc_t) xdr_gf_cli_req, options,
-                               GLUSTER_CLI_CLRLOCKS_VOLUME, this, cli_rpc_prog,
-                               NULL);
-out:
-        gf_log ("cli", GF_LOG_DEBUG, "Returning %d", ret);
-
-        GF_FREE (req.dict.dict_val);
-        return ret;
+    GF_FREE(req.dict.dict_val);
+    return ret;
 }
 
-int32_t
-cli_snapshot_remove_reply (gf_cli_rsp *rsp, dict_t *dict, call_frame_t *frame)
+static int32_t
+cli_snapshot_remove_reply(gf_cli_rsp *rsp, dict_t *dict, call_frame_t *frame)
 {
-        int32_t         ret             = -1;
-        char            *snap_name      = NULL;
-        int32_t         delete_cmd      = -1;
-        cli_local_t     *local          = NULL;
+    int32_t ret = -1;
+    char *snap_name = NULL;
+    int32_t delete_cmd = -1;
+    cli_local_t *local = NULL;
 
-        GF_ASSERT (frame);
-        GF_ASSERT (rsp);
-        GF_ASSERT (dict);
+    GF_ASSERT(frame);
+    GF_ASSERT(rsp);
+    GF_ASSERT(dict);
 
-        local = frame->local;
+    local = frame->local;
 
-        ret = dict_get_int32 (dict, "sub-cmd", &delete_cmd);
-        if (ret) {
-                gf_log ("cli", GF_LOG_ERROR, "Could not get sub-cmd");
-                goto end;
-        }
+    ret = dict_get_int32_sizen(dict, "sub-cmd", &delete_cmd);
+    if (ret) {
+        gf_log("cli", GF_LOG_ERROR, "Could not get sub-cmd");
+        goto end;
+    }
 
-        if ((global_state->mode & GLUSTER_MODE_XML) &&
-            (delete_cmd == GF_SNAP_DELETE_TYPE_SNAP)) {
-                ret = cli_xml_output_snap_delete_begin (local, rsp->op_ret,
-                                                        rsp->op_errno,
-                                                        rsp->op_errstr);
-                if (ret) {
-                        gf_log ("cli", GF_LOG_ERROR, "Failed to create "
-                                "xml output for delete");
-                        goto end;
-                }
+    if ((global_state->mode & GLUSTER_MODE_XML) &&
+        (delete_cmd == GF_SNAP_DELETE_TYPE_SNAP)) {
+        ret = cli_xml_output_snap_delete_begin(local, rsp->op_ret,
+                                               rsp->op_errno, rsp->op_errstr);
+        if (ret) {
+            gf_log("cli", GF_LOG_ERROR,
+                   "Failed to create xml output for delete");
+            goto end;
+        }
+    }
+
+    if (rsp->op_ret && !(global_state->mode & GLUSTER_MODE_XML)) {
+        cli_err("snapshot delete: failed: %s",
+                rsp->op_errstr ? rsp->op_errstr
+                               : "Please check log file for details");
+        ret = rsp->op_ret;
+        goto out;
+    }
+
+    if (delete_cmd == GF_SNAP_DELETE_TYPE_ALL ||
+        delete_cmd == GF_SNAP_DELETE_TYPE_VOL) {
+        local = ((call_frame_t *)frame)->local;
+        if (!local) {
+            ret = -1;
+            gf_log("cli", GF_LOG_ERROR, "frame->local is NULL");
+            goto out;
         }
 
+        /* During first call back of snapshot delete of type
+         * ALL and VOL, We will get the snapcount and snapnames.
+         * Hence to make the subsequent rpc calls for individual
+         * snapshot delete, We need to save it in local dictionary.
+         */
+        dict_copy(dict, local->dict);
+        ret = 0;
+        goto out;
+    }
 
-        if (rsp->op_ret && !(global_state->mode & GLUSTER_MODE_XML)) {
-                        cli_err ("snapshot delete: failed: %s",
-                                  rsp->op_errstr ? rsp->op_errstr :
-                                 "Please check log file for details");
-                ret = rsp->op_ret;
-                goto out;
+    if (global_state->mode & GLUSTER_MODE_XML) {
+        ret = cli_xml_snapshot_delete(local, dict, rsp);
+        if (ret) {
+            gf_log("cli", GF_LOG_ERROR,
+                   "Failed to create xml output for snapshot delete command");
+            goto out;
         }
-
-        if (delete_cmd == GF_SNAP_DELETE_TYPE_ALL ||
-            delete_cmd == GF_SNAP_DELETE_TYPE_VOL) {
-                local = ((call_frame_t *) frame) -> local;
-                if (!local) {
-                        ret = -1;
-                        gf_log ("cli", GF_LOG_ERROR, "frame->local is NULL");
-                        goto out;
-                }
-
-                /* During first call back of snapshot delete of type
-                 * ALL and VOL, We will get the snapcount and snapnames.
-                 * Hence to make the subsequent rpc calls for individual
-                 * snapshot delete, We need to save it in local dictionary.
-                 */
-                dict_copy (dict, local->dict);
-                ret = 0;
-                goto out;
+        /* Error out in case of the op already failed */
+        if (rsp->op_ret) {
+            ret = rsp->op_ret;
+            goto out;
         }
-
-        if (global_state->mode & GLUSTER_MODE_XML) {
-                ret = cli_xml_snapshot_delete (local->writer, local->doc,
-                                               dict, rsp);
-                if (ret) {
-                        gf_log ("cli", GF_LOG_ERROR, "Failed to create "
-                                "xml output for snapshot delete command");
-                        goto out;
-                }
-                /* Error out in case of the op already failed */
-                if (rsp->op_ret) {
-                        ret = rsp->op_ret;
-                        goto out;
-                }
-        } else {
-                ret = dict_get_str (dict, "snapname", &snap_name);
-                if (ret) {
-                        gf_log ("cli", GF_LOG_ERROR, "Failed to get snapname");
-                        goto out;
-                }
-
-                cli_out ("snapshot delete: %s: snap removed successfully",
-                         snap_name);
+    } else {
+        ret = dict_get_str_sizen(dict, "snapname", &snap_name);
+        if (ret) {
+            gf_log("cli", GF_LOG_ERROR, "Failed to get snapname");
+            goto out;
         }
-        ret = 0;
+
+        cli_out("snapshot delete: %s: snap removed successfully", snap_name);
+    }
+    ret = 0;
 
 out:
-        if ((global_state->mode & GLUSTER_MODE_XML) &&
-            (delete_cmd == GF_SNAP_DELETE_TYPE_SNAP)) {
-                ret = cli_xml_output_snap_delete_end (local);
-        }
+    if ((global_state->mode & GLUSTER_MODE_XML) &&
+        (delete_cmd == GF_SNAP_DELETE_TYPE_SNAP)) {
+        ret = cli_xml_output_snap_delete_end(local);
+    }
 end:
-        return ret;
+    return ret;
 }
 
-int
-cli_snapshot_config_display (dict_t *dict, gf_cli_rsp *rsp)
-{
-        char                buf[PATH_MAX]        = "";
-        char               *volname              = NULL;
-        int                 ret                  = -1;
-        int                 config_command       = 0;
-        uint64_t            value                = 0;
-        uint64_t            hard_limit           = 0;
-        uint64_t            soft_limit           = 0;
-        uint64_t            i                    = 0;
-        uint64_t            voldisplaycount      = 0;
-        char               *auto_delete          = NULL;
-        char               *snap_activate        = NULL;
-
-        GF_ASSERT (dict);
-        GF_ASSERT (rsp);
+static int
+cli_snapshot_config_display(dict_t *dict, gf_cli_rsp *rsp)
+{
+    char buf[PATH_MAX] = "";
+    char *volname = NULL;
+    int ret = -1;
+    int config_command = 0;
+    uint64_t value = 0;
+    uint64_t hard_limit = 0;
+    uint64_t soft_limit = 0;
+    uint64_t i = 0;
+    uint64_t voldisplaycount = 0;
+    char *auto_delete = NULL;
+    char *snap_activate = NULL;
+
+    GF_ASSERT(dict);
+    GF_ASSERT(rsp);
+
+    if (rsp->op_ret) {
+        cli_err("Snapshot Config : failed: %s",
+                rsp->op_errstr ? rsp->op_errstr
+                               : "Please check log file for details");
+        ret = rsp->op_ret;
+        goto out;
+    }
+
+    ret = dict_get_int32_sizen(dict, "config-command", &config_command);
+    if (ret) {
+        gf_log("cli", GF_LOG_ERROR, "Could not fetch config type");
+        goto out;
+    }
+
+    ret = dict_get_uint64(dict, "snap-max-hard-limit", &hard_limit);
+    /* Ignore the error, as the key specified is optional */
+    ret = dict_get_uint64(dict, "snap-max-soft-limit", &soft_limit);
+
+    ret = dict_get_str_sizen(dict, "auto-delete", &auto_delete);
+
+    ret = dict_get_str_sizen(dict, "snap-activate-on-create", &snap_activate);
+
+    if (!hard_limit && !soft_limit &&
+        config_command != GF_SNAP_CONFIG_DISPLAY && !auto_delete &&
+        !snap_activate) {
+        ret = -1;
+        gf_log(THIS->name, GF_LOG_ERROR, "Could not fetch config-key");
+        goto out;
+    }
+
+    ret = dict_get_str_sizen(dict, "volname", &volname);
+    /* Ignore the error, as volname is optional */
+
+    if (!volname) {
+        volname = "System";
+    }
+
+    switch (config_command) {
+        case GF_SNAP_CONFIG_TYPE_SET:
+            if (hard_limit && soft_limit) {
+                cli_out(
+                    "snapshot config: snap-max-hard-limit "
+                    "& snap-max-soft-limit for system set successfully");
+            } else if (hard_limit) {
+                cli_out(
+                    "snapshot config: snap-max-hard-limit "
+                    "for %s set successfully",
+                    volname);
+            } else if (soft_limit) {
+                cli_out(
+                    "snapshot config: snap-max-soft-limit "
+                    "for %s set successfully",
+                    volname);
+            } else if (auto_delete) {
+                cli_out("snapshot config: auto-delete successfully set");
+            } else if (snap_activate) {
+                cli_out("snapshot config: activate-on-create successfully set");
+            }
+            break;
 
-        if (rsp->op_ret) {
-                cli_err ("Snapshot Config : failed: %s",
-                         rsp->op_errstr ? rsp->op_errstr :
-                         "Please check log file for details");
-                ret = rsp->op_ret;
+        case GF_SNAP_CONFIG_DISPLAY:
+            cli_out("\nSnapshot System Configuration:");
+            ret = dict_get_uint64(dict, "snap-max-hard-limit", &value);
+            if (ret) {
+                gf_log("cli", GF_LOG_ERROR,
+                       "Could not fetch snap_max_hard_limit for %s", volname);
+                ret = -1;
                 goto out;
-        }
+            }
+            cli_out("snap-max-hard-limit : %" PRIu64, value);
 
-        ret = dict_get_int32 (dict, "config-command", &config_command);
-        if (ret) {
-                gf_log ("cli", GF_LOG_ERROR, "Could not fetch config type");
+            ret = dict_get_uint64(dict, "snap-max-soft-limit", &soft_limit);
+            if (ret) {
+                gf_log("cli", GF_LOG_ERROR,
+                       "Could not fetch snap-max-soft-limit for %s", volname);
+                ret = -1;
                 goto out;
-        }
-
-        ret = dict_get_str (dict, "volname", &volname);
-        /* Ignore the error, as volname is optional */
+            }
+            cli_out("snap-max-soft-limit : %" PRIu64 "%%", soft_limit);
 
-        if (!volname) {
-                volname = "System";
-        }
-
-        ret = dict_get_uint64 (dict, "snap-max-hard-limit", &hard_limit);
-        /* Ignore the error, as the key specified is optional */
-        ret = dict_get_uint64 (dict, "snap-max-soft-limit", &soft_limit);
+            cli_out("auto-delete : %s", auto_delete);
 
-        ret = dict_get_str (dict, "auto-delete", &auto_delete);
+            cli_out("activate-on-create : %s\n", snap_activate);
 
-        ret = dict_get_str (dict, "snap-activate-on-create", &snap_activate);
+            cli_out("Snapshot Volume Configuration:");
 
-        if (!hard_limit && !soft_limit
-                        && config_command != GF_SNAP_CONFIG_DISPLAY
-                        && !auto_delete && !snap_activate) {
+            ret = dict_get_uint64(dict, "voldisplaycount", &voldisplaycount);
+            if (ret) {
+                gf_log("cli", GF_LOG_ERROR, "Could not fetch voldisplaycount");
                 ret = -1;
-                gf_log(THIS->name, GF_LOG_ERROR,
-                       "Could not fetch config-key");
                 goto out;
-        }
-
-        switch (config_command) {
-        case GF_SNAP_CONFIG_TYPE_SET:
-                if (hard_limit && soft_limit) {
-                        cli_out ("snapshot config: snap-max-hard-limit "
-                                "& snap-max-soft-limit for system set "
-                                 "successfully");
-                } else if (hard_limit) {
-                        cli_out ("snapshot config: snap-max-hard-limit "
-                                 "for %s set successfully",
-                                 volname);
-                } else if (soft_limit) {
-                        cli_out ("snapshot config: snap-max-soft-limit "
-                                 "for %s set successfully",
-                                 volname);
-                } else if (auto_delete) {
-                        cli_out ("snapshot config: auto-delete "
-                                 "successfully set");
-                } else if (snap_activate) {
-                        cli_out ("snapshot config: activate-on-create "
-                                 "successfully set");
-                }
-                break;
+            }
 
-        case GF_SNAP_CONFIG_DISPLAY:
-                cli_out ("\nSnapshot System Configuration:");
-                ret = dict_get_uint64 (dict, "snap-max-hard-limit",
-                                       &value);
+            for (i = 0; i < voldisplaycount; i++) {
+                ret = snprintf(buf, sizeof(buf), "volume%" PRIu64 "-volname",
+                               i);
+                ret = dict_get_strn(dict, buf, ret, &volname);
                 if (ret) {
-                        gf_log ("cli", GF_LOG_ERROR, "Could not fetch "
-                                "snap_max_hard_limit for %s", volname);
-                        ret = -1;
-                        goto out;
+                    gf_log("cli", GF_LOG_ERROR, "Could not fetch %s", buf);
+                    ret = -1;
+                    goto out;
                 }
-                cli_out ("snap-max-hard-limit : %"PRIu64, value);
+                cli_out("\nVolume : %s", volname);
 
-                ret = dict_get_uint64 (dict, "snap-max-soft-limit",
-                                       &soft_limit);
+                snprintf(buf, sizeof(buf),
+                         "volume%" PRIu64 "-snap-max-hard-limit", i);
+                ret = dict_get_uint64(dict, buf, &value);
                 if (ret) {
-                        gf_log ("cli", GF_LOG_ERROR, "Could not fetch "
-                               "snap-max-soft-limit for %s", volname);
-                        ret = -1;
-                        goto out;
+                    gf_log("cli", GF_LOG_ERROR, "Could not fetch %s", buf);
+                    ret = -1;
+                    goto out;
                 }
-                cli_out ("snap-max-soft-limit : %"PRIu64"%%",
-                         soft_limit);
-
-                cli_out ("auto-delete : %s", auto_delete);
-
-                cli_out ("activate-on-create : %s\n", snap_activate);
+                cli_out("snap-max-hard-limit : %" PRIu64, value);
 
-                cli_out ("Snapshot Volume Configuration:");
-
-                ret = dict_get_uint64 (dict, "voldisplaycount",
-                                       &voldisplaycount);
+                snprintf(buf, sizeof(buf),
+                         "volume%" PRIu64 "-active-hard-limit", i);
+                ret = dict_get_uint64(dict, buf, &value);
                 if (ret) {
-                        gf_log ("cli", GF_LOG_ERROR,
-                               "Could not fetch voldisplaycount");
-                        ret = -1;
-                        goto out;
-                }
-
-                for (i = 0; i < voldisplaycount; i++) {
-                        snprintf (buf, sizeof(buf), "volume%"PRIu64"-volname", i);
-                        ret = dict_get_str (dict, buf, &volname);
-                        if (ret) {
-                                gf_log ("cli", GF_LOG_ERROR, "Could not fetch "
-                                       " %s", buf);
-                                ret = -1;
-                                goto out;
-                        }
-                        cli_out ("\nVolume : %s", volname);
-
-                        snprintf (buf, sizeof(buf),
-                                  "volume%"PRIu64"-snap-max-hard-limit", i);
-                        ret = dict_get_uint64 (dict, buf, &value);
-                        if (ret) {
-                                gf_log ("cli", GF_LOG_ERROR, "Could not fetch "
-                                       " %s", buf);
-                                ret = -1;
-                                goto out;
-                        }
-                        cli_out ("snap-max-hard-limit : %"PRIu64, value);
-
-                        snprintf (buf, sizeof(buf),
-                                  "volume%"PRIu64"-active-hard-limit", i);
-                        ret = dict_get_uint64 (dict, buf, &value);
-                        if (ret) {
-                                 gf_log ("cli", GF_LOG_ERROR, "Could not fetch"
-                                        " effective snap_max_hard_limit for "
-                                        "%s", volname);
-                                ret = -1;
-                                goto out;
-                        }
-                        cli_out ("Effective snap-max-hard-limit : %"PRIu64,
-                                 value);
-
-                        snprintf (buf, sizeof(buf),
-                                      "volume%"PRIu64"-snap-max-soft-limit", i);
-                        ret = dict_get_uint64 (dict, buf, &value);
-                        if (ret) {
-                                gf_log ("cli", GF_LOG_ERROR, "Could not fetch "
-                                       " %s", buf);
-                                ret = -1;
-                                goto out;
-                        }
-                        cli_out ("Effective snap-max-soft-limit : %"PRIu64" "
-                                 "(%"PRIu64"%%)", value, soft_limit);
-                }
-                break;
+                    gf_log("cli", GF_LOG_ERROR,
+                           "Could not fetch"
+                           " effective snap_max_hard_limit for %s",
+                           volname);
+                    ret = -1;
+                    goto out;
+                }
+                cli_out("Effective snap-max-hard-limit : %" PRIu64, value);
+
+                snprintf(buf, sizeof(buf),
+                         "volume%" PRIu64 "-snap-max-soft-limit", i);
+                ret = dict_get_uint64(dict, buf, &value);
+                if (ret) {
+                    gf_log("cli", GF_LOG_ERROR, "Could not fetch %s", buf);
+                    ret = -1;
+                    goto out;
+                }
+                cli_out("Effective snap-max-soft-limit : %" PRIu64
+                        " "
+                        "(%" PRIu64 "%%)",
+                        value, soft_limit);
+            }
+            break;
         default:
-                break;
-        }
+            break;
+    }
 
-        ret = 0;
+    ret = 0;
 out:
-        return ret;
+    return ret;
 }
 
 /* This function is used to print the volume related information
@@ -9103,1989 +8651,2299 @@ out:
  * arg - 0, dict       : Response Dictionary.
  * arg - 1, prefix str : snaplist.snap{0..}.vol{0..}.*
  */
-int
-cli_get_each_volinfo_in_snap (dict_t *dict, char *keyprefix,
-                              gf_boolean_t snap_driven) {
-        char             key[PATH_MAX]  =  "";
-        char            *get_buffer     =  NULL;
-        int              value          =  0;
-        int              ret            = -1;
-        char             indent[5]      = "\t";
-        char            *volname        =  NULL;
-
-        GF_ASSERT (dict);
-        GF_ASSERT (keyprefix);
-
-        if (snap_driven) {
-                ret = snprintf (key, sizeof (key), "%s.volname", keyprefix);
-                if (ret < 0) {
-                        goto out;
-                }
-
-                ret = dict_get_str (dict, key, &get_buffer);
-                if (ret) {
-                        gf_log ("cli", GF_LOG_ERROR, "Failed to get %s", key);
-                        goto out;
-                }
-                cli_out ("%s" INDENT_MAIN_HEAD "%s", indent,
-                        "Snap Volume Name", ":", get_buffer);
-
-                ret = snprintf (key, sizeof (key),
-                                "%s.origin-volname", keyprefix);
-                if (ret < 0) {
-                        goto out;
-                }
+static int
+cli_get_each_volinfo_in_snap(dict_t *dict, char *keyprefix,
+                             gf_boolean_t snap_driven)
+{
+    char key[PATH_MAX] = "";
+    char *get_buffer = NULL;
+    int value = 0;
+    int ret = -1;
+    char indent[5] = "\t";
+    char *volname = NULL;
 
-                ret = dict_get_str (dict, key, &volname);
-                if (ret) {
-                        gf_log ("cli", GF_LOG_WARNING, "Failed to get %s", key);
-                        cli_out ("%-12s", "Origin:");
-                }
-                cli_out ("%s" INDENT_MAIN_HEAD "%s", indent,
-                        "Origin Volume name", ":", volname);
+    GF_ASSERT(dict);
+    GF_ASSERT(keyprefix);
 
+    if (snap_driven) {
+        ret = snprintf(key, sizeof(key), "%s.volname", keyprefix);
+        if (ret < 0) {
+            goto out;
+        }
 
-                ret = snprintf (key, sizeof (key), "%s.snapcount",
-                                keyprefix);
-                if (ret < 0) {
-                        goto out;
-                }
+        ret = dict_get_str(dict, key, &get_buffer);
+        if (ret) {
+            gf_log("cli", GF_LOG_ERROR, "Failed to get %s", key);
+            goto out;
+        }
+        cli_out("%s" INDENT_MAIN_HEAD "%s", indent, "Snap Volume Name", ":",
+                get_buffer);
 
-                ret = dict_get_int32 (dict, key, &value);
-                if (ret) {
-                        gf_log ("cli", GF_LOG_ERROR, "Failed to get %s", key);
-                        goto out;
-                }
-                cli_out ("%s%s %s      %s %d", indent, "Snaps taken for",
-                        volname, ":", value);
+        ret = snprintf(key, sizeof(key), "%s.origin-volname", keyprefix);
+        if (ret < 0) {
+            goto out;
+        }
 
-                ret = snprintf (key, sizeof (key), "%s.snaps-available",
-                                keyprefix);
-                if (ret < 0) {
-                        goto out;
-                }
+        ret = dict_get_str(dict, key, &volname);
+        if (ret) {
+            gf_log("cli", GF_LOG_WARNING, "Failed to get %s", key);
+            cli_out("%-12s", "Origin:");
+        }
+        cli_out("%s" INDENT_MAIN_HEAD "%s", indent, "Origin Volume name", ":",
+                volname);
 
-                ret = dict_get_int32 (dict, key, &value);
-                if (ret) {
-                        gf_log ("cli", GF_LOG_ERROR, "Failed to get %s", key);
-                        goto out;
-                }
-                cli_out ("%s%s %s  %s %d", indent, "Snaps available for",
-                         volname, ":", value);
+        ret = snprintf(key, sizeof(key), "%s.snapcount", keyprefix);
+        if (ret < 0) {
+            goto out;
         }
 
+        ret = dict_get_int32(dict, key, &value);
+        if (ret) {
+            gf_log("cli", GF_LOG_ERROR, "Failed to get %s", key);
+            goto out;
+        }
+        cli_out("%s%s %s      %s %d", indent, "Snaps taken for", volname, ":",
+                value);
 
-        ret = snprintf (key, sizeof (key), "%s.vol-status", keyprefix);
+        ret = snprintf(key, sizeof(key), "%s.snaps-available", keyprefix);
         if (ret < 0) {
-                goto out;
+            goto out;
         }
 
-        ret = dict_get_str (dict, key, &get_buffer);
+        ret = dict_get_int32(dict, key, &value);
         if (ret) {
-                gf_log ("cli", GF_LOG_ERROR, "Failed to get %s", key);
-                goto out;
+            gf_log("cli", GF_LOG_ERROR, "Failed to get %s", key);
+            goto out;
         }
-        cli_out ("%s" INDENT_MAIN_HEAD "%s", indent, "Status",
-                 ":", get_buffer);
+        cli_out("%s%s %s  %s %d", indent, "Snaps available for", volname, ":",
+                value);
+    }
+
+    ret = snprintf(key, sizeof(key), "%s.vol-status", keyprefix);
+    if (ret < 0) {
+        goto out;
+    }
+
+    ret = dict_get_str(dict, key, &get_buffer);
+    if (ret) {
+        gf_log("cli", GF_LOG_ERROR, "Failed to get %s", key);
+        goto out;
+    }
+    cli_out("%s" INDENT_MAIN_HEAD "%s", indent, "Status", ":", get_buffer);
 out:
-        return ret;
+    return ret;
 }
 
 /* This function is used to print snap related information
  * arg - 0, dict       : Response dictionary.
  * arg - 1, prefix_str : snaplist.snap{0..}.*
  */
-int
-cli_get_volinfo_in_snap (dict_t *dict, char *keyprefix) {
+static int
+cli_get_volinfo_in_snap(dict_t *dict, char *keyprefix)
+{
+    char key[PATH_MAX] = "";
+    int i = 0;
+    int volcount = 0;
+    int ret = -1;
 
-        char            key[PATH_MAX]   = "";
-        int             i               = 0;
-        int             volcount        = 0;
-        int             ret             = -1;
+    GF_ASSERT(dict);
+    GF_ASSERT(keyprefix);
 
-        GF_ASSERT (dict);
-        GF_ASSERT (keyprefix);
+    ret = snprintf(key, sizeof(key), "%s.vol-count", keyprefix);
+    if (ret < 0) {
+        goto out;
+    }
 
-        ret = snprintf (key, sizeof (key), "%s.vol-count", keyprefix);
+    ret = dict_get_int32(dict, key, &volcount);
+    for (i = 1; i <= volcount; i++) {
+        ret = snprintf(key, sizeof(key), "%s.vol%d", keyprefix, i);
         if (ret < 0) {
-                goto out;
+            goto out;
         }
-
-        ret = dict_get_int32 (dict, key, &volcount);
-        for (i = 1 ; i <= volcount ; i++) {
-                ret = snprintf (key, sizeof (key),
-                                "%s.vol%d", keyprefix, i);
-                if (ret < 0) {
-                        goto out;
-                }
-                ret = cli_get_each_volinfo_in_snap (dict, key, _gf_true);
-                if (ret) {
-                        gf_log ("cli", GF_LOG_ERROR, "Could not list "
-                                "details of volume in a snap");
-                        goto out;
-                }
-                cli_out (" ");
+        ret = cli_get_each_volinfo_in_snap(dict, key, _gf_true);
+        if (ret) {
+            gf_log("cli", GF_LOG_ERROR,
+                   "Could not list details of volume in a snap");
+            goto out;
         }
+        cli_out(" ");
+    }
 
 out:
-        return ret;
+    return ret;
 }
 
-int
-cli_get_each_snap_info (dict_t *dict, char *prefix_str,
-                        gf_boolean_t snap_driven) {
-        char            key_buffer[PATH_MAX] = "";
-        char           *get_buffer           = NULL;
-        int             ret                  = -1;
-        char            indent[5]            = "";
+static int
+cli_get_each_snap_info(dict_t *dict, char *prefix_str, gf_boolean_t snap_driven)
+{
+    char key_buffer[PATH_MAX] = "";
+    char *get_buffer = NULL;
+    int ret = -1;
+    char indent[5] = "";
+
+    GF_ASSERT(dict);
+    GF_ASSERT(prefix_str);
+
+    if (!snap_driven)
+        strcat(indent, "\t");
+
+    ret = snprintf(key_buffer, sizeof(key_buffer), "%s.snapname", prefix_str);
+    if (ret < 0) {
+        goto out;
+    }
+
+    ret = dict_get_str(dict, key_buffer, &get_buffer);
+    if (ret) {
+        gf_log("cli", GF_LOG_ERROR, "Unable to fetch snapname %s ", key_buffer);
+        goto out;
+    }
+    cli_out("%s" INDENT_MAIN_HEAD "%s", indent, "Snapshot", ":", get_buffer);
+
+    ret = snprintf(key_buffer, sizeof(key_buffer), "%s.snap-id", prefix_str);
+    if (ret < 0) {
+        goto out;
+    }
+
+    ret = dict_get_str(dict, key_buffer, &get_buffer);
+    if (ret) {
+        gf_log("cli", GF_LOG_ERROR, "Unable to fetch snap-id %s ", key_buffer);
+        goto out;
+    }
+    cli_out("%s" INDENT_MAIN_HEAD "%s", indent, "Snap UUID", ":", get_buffer);
+
+    ret = snprintf(key_buffer, sizeof(key_buffer), "%s.snap-desc", prefix_str);
+    if (ret < 0) {
+        goto out;
+    }
+
+    ret = dict_get_str(dict, key_buffer, &get_buffer);
+    if (!ret) {
+        /* Ignore error for description */
+        cli_out("%s" INDENT_MAIN_HEAD "%s", indent, "Description", ":",
+                get_buffer);
+    }
+
+    ret = snprintf(key_buffer, sizeof(key_buffer), "%s.snap-time", prefix_str);
+    if (ret < 0) {
+        goto out;
+    }
+
+    ret = dict_get_str(dict, key_buffer, &get_buffer);
+    if (ret) {
+        gf_log("cli", GF_LOG_ERROR, "Unable to fetch snap-time %s ",
+               prefix_str);
+        goto out;
+    }
+    cli_out("%s" INDENT_MAIN_HEAD "%s", indent, "Created", ":", get_buffer);
+
+    if (snap_driven) {
+        cli_out("%-12s", "Snap Volumes:\n");
+        ret = cli_get_volinfo_in_snap(dict, prefix_str);
+        if (ret) {
+            gf_log("cli", GF_LOG_ERROR, "Unable to list details of the snaps");
+            goto out;
+        }
+    }
+out:
+    return ret;
+}
 
-        GF_ASSERT (dict);
-        GF_ASSERT (prefix_str);
+/* This is a generic function to print snap related information.
+ * arg - 0, dict : Response Dictionary
+ */
+static int
+cli_call_snapshot_info(dict_t *dict, gf_boolean_t bool_snap_driven)
+{
+    int snap_count = 0;
+    char key[32] = "";
+    int ret = -1;
+    int i = 0;
 
-        if (!snap_driven)
-                strcat (indent, "\t");
+    GF_ASSERT(dict);
 
-        ret = snprintf (key_buffer, sizeof (key_buffer), "%s.snapname",
-                        prefix_str);
-        if (ret < 0) {
-                goto out;
-        }
+    ret = dict_get_int32_sizen(dict, "snapcount", &snap_count);
+    if (ret) {
+        gf_log("cli", GF_LOG_ERROR, "Unable to get snapcount");
+        goto out;
+    }
 
-        ret = dict_get_str (dict, key_buffer, &get_buffer);
-        if (ret) {
-                gf_log ("cli", GF_LOG_ERROR, "Unable to fetch snapname %s ",
-                        key_buffer);
-                goto out;
-        }
-        cli_out ("%s" INDENT_MAIN_HEAD "%s", indent, "Snapshot",
-                ":", get_buffer);
+    if (snap_count == 0) {
+        cli_out("No snapshots present");
+    }
 
-        ret = snprintf (key_buffer, sizeof (key_buffer), "%s.snap-id",
-                        prefix_str);
+    for (i = 1; i <= snap_count; i++) {
+        ret = snprintf(key, sizeof(key), "snap%d", i);
         if (ret < 0) {
-                goto out;
+            goto out;
         }
-
-        ret = dict_get_str (dict, key_buffer, &get_buffer);
+        ret = cli_get_each_snap_info(dict, key, bool_snap_driven);
         if (ret) {
-                gf_log ("cli", GF_LOG_ERROR, "Unable to fetch snap-id %s ",
-                        key_buffer);
-                goto out;
-        }
-        cli_out ("%s" INDENT_MAIN_HEAD "%s", indent, "Snap UUID",
-                ":", get_buffer);
-
-        ret = snprintf (key_buffer, sizeof (key_buffer), "%s.snap-desc",
-                        prefix_str);
-        if (ret < 0) {
-                goto out;
+            gf_log("cli", GF_LOG_ERROR, "Unable to print snap details");
+            goto out;
         }
+    }
+out:
+    return ret;
+}
 
-        ret = dict_get_str (dict, key_buffer, &get_buffer);
-        if (!ret) {
-                /* Ignore error for description */
-                cli_out ("%s" INDENT_MAIN_HEAD "%s", indent,
-                         "Description", ":", get_buffer);
+static int
+cli_get_snaps_in_volume(dict_t *dict)
+{
+    int ret = -1;
+    int i = 0;
+    int count = 0;
+    int avail = 0;
+    char key[32] = "";
+    char *get_buffer = NULL;
+
+    GF_ASSERT(dict);
+
+    ret = dict_get_str_sizen(dict, "origin-volname", &get_buffer);
+    if (ret) {
+        gf_log("cli", GF_LOG_ERROR, "Could not fetch origin-volname");
+        goto out;
+    }
+    cli_out(INDENT_MAIN_HEAD "%s", "Volume Name", ":", get_buffer);
+
+    ret = dict_get_int32_sizen(dict, "snapcount", &avail);
+    if (ret) {
+        gf_log("cli", GF_LOG_ERROR, "Could not fetch snapcount");
+        goto out;
+    }
+    cli_out(INDENT_MAIN_HEAD "%d", "Snaps Taken", ":", avail);
+
+    ret = dict_get_int32_sizen(dict, "snaps-available", &count);
+    if (ret) {
+        gf_log("cli", GF_LOG_ERROR, "Could not fetch snaps-available");
+        goto out;
+    }
+    cli_out(INDENT_MAIN_HEAD "%d", "Snaps Available", ":", count);
+
+    for (i = 1; i <= avail; i++) {
+        snprintf(key, sizeof(key), "snap%d", i);
+        ret = cli_get_each_snap_info(dict, key, _gf_false);
+        if (ret) {
+            gf_log("cli", GF_LOG_ERROR, "Unable to print snap details");
+            goto out;
         }
 
-        ret = snprintf (key_buffer, sizeof (key_buffer), "%s.snap-time",
-                        prefix_str);
+        ret = snprintf(key, sizeof(key), "snap%d.vol1", i);
         if (ret < 0) {
-                goto out;
+            goto out;
         }
-
-        ret = dict_get_str (dict, key_buffer, &get_buffer);
+        ret = cli_get_each_volinfo_in_snap(dict, key, _gf_false);
         if (ret) {
-                gf_log ("cli", GF_LOG_ERROR, "Unable to fetch snap-time %s ",
-                        prefix_str);
-                goto out;
+            gf_log("cli", GF_LOG_ERROR,
+                   "Could not get volume related information");
+            goto out;
         }
-        cli_out ("%s" INDENT_MAIN_HEAD "%s", indent, "Created",
-                ":", get_buffer);
 
-        if (snap_driven) {
-                cli_out ("%-12s", "Snap Volumes:\n");
-                ret = cli_get_volinfo_in_snap (dict, prefix_str);
-                if (ret) {
-                        gf_log ("cli", GF_LOG_ERROR, "Unable to list details "
-                                "of the snaps");
-                        goto out;
-                }
-        }
+        cli_out(" ");
+    }
 out:
-        return ret;
+    return ret;
 }
 
-/* This is a generic function to print snap related information.
- * arg - 0, dict : Response Dictionary
- */
-int
-cli_call_snapshot_info (dict_t *dict, gf_boolean_t bool_snap_driven) {
-        int             snap_count       =  0;
-        char            key[PATH_MAX]    =  "";
-        int             ret              =  -1;
-        int             i                =   0;
+static int
+cli_snapshot_list(dict_t *dict)
+{
+    int snapcount = 0;
+    char key[32] = "";
+    int ret = -1;
+    int i = 0;
+    char *get_buffer = NULL;
 
-        GF_ASSERT (dict);
+    GF_ASSERT(dict);
 
-        ret = dict_get_int32 (dict, "snapcount", &snap_count);
-        if (ret) {
-                gf_log ("cli", GF_LOG_ERROR, "Unable to get snapcount");
-                goto out;
-        }
+    ret = dict_get_int32_sizen(dict, "snapcount", &snapcount);
+    if (ret) {
+        gf_log("cli", GF_LOG_ERROR, "Could not fetch snap count");
+        goto out;
+    }
 
-        if (snap_count == 0) {
-                cli_out ("No snapshots present");
+    if (snapcount == 0) {
+        cli_out("No snapshots present");
+    }
+
+    for (i = 1; i <= snapcount; i++) {
+        ret = snprintf(key, sizeof(key), "snapname%d", i);
+        if (ret < 0) {
+            goto out;
         }
 
-        for (i = 1 ; i <= snap_count ; i++) {
-                ret = snprintf (key, sizeof (key), "snap%d", i);
-                if (ret < 0) {
-                        goto out;
-                }
-                ret = cli_get_each_snap_info (dict, key, bool_snap_driven);
-                if (ret) {
-                        gf_log ("cli", GF_LOG_ERROR,
-                                "Unable to print snap details");
-                        goto out;
-                }
+        ret = dict_get_strn(dict, key, ret, &get_buffer);
+        if (ret) {
+            gf_log("cli", GF_LOG_ERROR, "Could not get %s ", key);
+            goto out;
+        } else {
+            cli_out("%s", get_buffer);
         }
+    }
 out:
-        return ret;
+    return ret;
 }
 
-int
-cli_get_snaps_in_volume (dict_t *dict) {
-        int      ret             =  -1;
-        int      i               =  0;
-        int      count           =  0;
-        int      avail           =  0;
-        char     key[PATH_MAX]   =  "";
-        char    *get_buffer      =  NULL;
-
-        GF_ASSERT (dict);
+static int
+cli_get_snap_volume_status(dict_t *dict, char *key_prefix)
+{
+    int ret = -1;
+    char key[PATH_MAX] = "";
+    char *buffer = NULL;
+    int brickcount = 0;
+    int i = 0;
+    int pid = 0;
+
+    GF_ASSERT(dict);
+    GF_ASSERT(key_prefix);
+
+    ret = snprintf(key, sizeof(key), "%s.brickcount", key_prefix);
+    if (ret < 0) {
+        goto out;
+    }
+    ret = dict_get_int32(dict, key, &brickcount);
+    if (ret) {
+        gf_log("cli", GF_LOG_ERROR, "Failed to fetch brickcount");
+        goto out;
+    }
+
+    for (i = 0; i < brickcount; i++) {
+        ret = snprintf(key, sizeof(key), "%s.brick%d.path", key_prefix, i);
+        if (ret < 0) {
+            goto out;
+        }
 
-        ret = dict_get_str (dict, "origin-volname", &get_buffer);
+        ret = dict_get_str(dict, key, &buffer);
         if (ret) {
-                gf_log ("cli", GF_LOG_ERROR, "Could not fetch origin-volname");
-                goto out;
+            gf_log("cli", GF_LOG_INFO, "Unable to get Brick Path");
+            continue;
         }
-        cli_out (INDENT_MAIN_HEAD "%s", "Volume Name", ":", get_buffer);
+        cli_out("\n\t%-17s %s   %s", "Brick Path", ":", buffer);
 
-        ret = dict_get_int32 (dict, "snapcount", &avail);
-        if (ret) {
-                gf_log ("cli", GF_LOG_ERROR, "Could not fetch snapcount");
-                goto out;
+        ret = snprintf(key, sizeof(key), "%s.brick%d.vgname", key_prefix, i);
+        if (ret < 0) {
+            goto out;
         }
-        cli_out (INDENT_MAIN_HEAD "%d", "Snaps Taken", ":", avail);
 
-        ret = dict_get_int32 (dict, "snaps-available", &count);
+        ret = dict_get_str(dict, key, &buffer);
         if (ret) {
-                gf_log ("cli", GF_LOG_ERROR, "Could not fetch snaps-available");
-                goto out;
-        }
-        cli_out (INDENT_MAIN_HEAD "%d", "Snaps Available", ":", count);
+            gf_log("cli", GF_LOG_INFO, "Unable to get Volume Group");
+            cli_out("\t%-17s %s   %s", "Volume Group", ":", "N/A");
+        } else
+            cli_out("\t%-17s %s   %s", "Volume Group", ":", buffer);
 
-        for (i = 1 ; i <= avail ; i++) {
-                snprintf (key, sizeof (key), "snap%d", i);
-                ret = cli_get_each_snap_info (dict, key, _gf_false);
-                if (ret) {
-                        gf_log ("cli", GF_LOG_ERROR,
-                                "Unable to print snap details");
-                        goto out;
-                }
+        ret = snprintf(key, sizeof(key), "%s.brick%d.status", key_prefix, i);
+        if (ret < 0) {
+            goto out;
+        }
 
-                ret = snprintf (key, sizeof (key), "snap%d.vol1", i);
-                if (ret < 0) {
-                        goto out;
-                }
-                ret = cli_get_each_volinfo_in_snap (dict, key, _gf_false);
-                if (ret) {
-                        gf_log ("cli", GF_LOG_ERROR, "Could not get volume "
-                                "related information");
-                        goto out;
-                }
+        ret = dict_get_str(dict, key, &buffer);
+        if (ret) {
+            gf_log("cli", GF_LOG_INFO, "Unable to get Brick Running");
+            cli_out("\t%-17s %s   %s", "Brick Running", ":", "N/A");
+        } else
+            cli_out("\t%-17s %s   %s", "Brick Running", ":", buffer);
 
-                cli_out (" ");
+        ret = snprintf(key, sizeof(key), "%s.brick%d.pid", key_prefix, i);
+        if (ret < 0) {
+            goto out;
         }
-out:
-        return ret;
-}
 
-int
-cli_snapshot_list (dict_t *dict) {
-        int     snapcount       =       0;
-        char    key[PATH_MAX]   =       "";
-        int     ret             =       -1;
-        int     i               =       0;
-        char    *get_buffer     =       NULL;
+        ret = dict_get_int32(dict, key, &pid);
+        if (ret) {
+            gf_log("cli", GF_LOG_INFO, "Unable to get pid");
+            cli_out("\t%-17s %s   %s", "Brick PID", ":", "N/A");
+        } else
+            cli_out("\t%-17s %s   %d", "Brick PID", ":", pid);
 
-        GF_ASSERT (dict);
+        ret = snprintf(key, sizeof(key), "%s.brick%d.data", key_prefix, i);
+        if (ret < 0) {
+            goto out;
+        }
 
-        ret = dict_get_int32 (dict, "snapcount", &snapcount);
+        ret = dict_get_str(dict, key, &buffer);
         if (ret) {
-                gf_log ("cli", GF_LOG_ERROR, "Could not fetch snap count");
-                goto out;
-        }
+            gf_log("cli", GF_LOG_INFO, "Unable to get Data Percent");
+            cli_out("\t%-17s %s   %s", "Data Percentage", ":", "N/A");
+        } else
+            cli_out("\t%-17s %s   %s", "Data Percentage", ":", buffer);
 
-        if (snapcount == 0) {
-                cli_out ("No snapshots present");
+        ret = snprintf(key, sizeof(key), "%s.brick%d.lvsize", key_prefix, i);
+        if (ret < 0) {
+            goto out;
         }
+        ret = dict_get_str(dict, key, &buffer);
+        if (ret) {
+            gf_log("cli", GF_LOG_INFO, "Unable to get LV Size");
+            cli_out("\t%-17s %s   %s", "LV Size", ":", "N/A");
+        } else
+            cli_out("\t%-17s %s   %s", "LV Size", ":", buffer);
+    }
 
-        for (i = 1 ; i <= snapcount ; i++) {
-                ret = snprintf (key, sizeof (key), "snapname%d", i);
-                if (ret < 0) {
-                        goto out;
-                }
+    ret = 0;
+out:
+    return ret;
+}
 
-                ret = dict_get_str (dict, key, &get_buffer);
-                if (ret) {
-                        gf_log ("cli", GF_LOG_ERROR, "Could not get %s ", key);
-                        goto out;
-                } else {
-                        cli_out ("%s", get_buffer);
-                }
+static int
+cli_get_single_snap_status(dict_t *dict, char *keyprefix)
+{
+    int ret = -1;
+    char key[64] = ""; /* keyprefix is ""status.snap0" */
+    int i = 0;
+    int volcount = 0;
+    char *get_buffer = NULL;
+
+    GF_ASSERT(dict);
+    GF_ASSERT(keyprefix);
+
+    ret = snprintf(key, sizeof(key), "%s.snapname", keyprefix);
+    if (ret < 0) {
+        goto out;
+    }
+
+    ret = dict_get_str(dict, key, &get_buffer);
+    if (ret) {
+        gf_log("cli", GF_LOG_ERROR, "Unable to get snapname");
+        goto out;
+    }
+    cli_out("\nSnap Name : %s", get_buffer);
+
+    ret = snprintf(key, sizeof(key), "%s.uuid", keyprefix);
+    if (ret < 0) {
+        goto out;
+    }
+
+    ret = dict_get_str(dict, key, &get_buffer);
+    if (ret) {
+        gf_log("cli", GF_LOG_ERROR, "Unable to get snap UUID");
+        goto out;
+    }
+    cli_out("Snap UUID : %s", get_buffer);
+
+    ret = snprintf(key, sizeof(key), "%s.volcount", keyprefix);
+    if (ret < 0) {
+        goto out;
+    }
+
+    ret = dict_get_int32(dict, key, &volcount);
+    if (ret) {
+        gf_log("cli", GF_LOG_ERROR, "Unable to get volume count");
+        goto out;
+    }
+
+    for (i = 0; i < volcount; i++) {
+        ret = snprintf(key, sizeof(key), "%s.vol%d", keyprefix, i);
+        if (ret < 0) {
+            goto out;
+        }
+
+        ret = cli_get_snap_volume_status(dict, key);
+        if (ret) {
+            gf_log("cli", GF_LOG_ERROR, "Could not get snap volume status");
+            goto out;
         }
+    }
 out:
-        return ret;
+    return ret;
 }
 
-int
-cli_get_snap_volume_status (dict_t *dict, char *key_prefix)
+static int32_t
+cli_populate_req_dict_for_delete(dict_t *snap_dict, dict_t *dict, size_t index)
+{
+    int32_t ret = -1;
+    char key[PATH_MAX] = "";
+    char *buffer = NULL;
+
+    GF_ASSERT(snap_dict);
+    GF_ASSERT(dict);
+
+    ret = dict_set_int32_sizen(snap_dict, "sub-cmd", GF_SNAP_DELETE_TYPE_ITER);
+    if (ret) {
+        gf_log("cli", GF_LOG_ERROR,
+               "Could not save command type in snap dictionary");
+        goto out;
+    }
+
+    ret = snprintf(key, sizeof(key), "snapname%zu", index);
+    if (ret < 0) {
+        goto out;
+    }
+
+    ret = dict_get_str(dict, key, &buffer);
+    if (ret) {
+        gf_log("cli", GF_LOG_ERROR, "Failed to get snapname");
+        goto out;
+    }
+
+    ret = dict_set_dynstr_with_alloc(snap_dict, "snapname", buffer);
+    if (ret) {
+        gf_log("cli", GF_LOG_ERROR, "Failed to save snapname");
+        goto out;
+    }
+
+    ret = dict_set_int32_sizen(snap_dict, "type", GF_SNAP_OPTION_TYPE_DELETE);
+    if (ret) {
+        gf_log("cli", GF_LOG_ERROR, "Failed to save command type");
+        goto out;
+    }
+
+    ret = dict_set_dynstr_with_alloc(snap_dict, "cmd-str", "snapshot delete");
+    if (ret) {
+        gf_log("cli", GF_LOG_ERROR, "Could not save command string as delete");
+        goto out;
+    }
+out:
+    return ret;
+}
+
+static int
+cli_populate_req_dict_for_status(dict_t *snap_dict, dict_t *dict, int index)
 {
-        int             ret             =       -1;
-        char            key[PATH_MAX]   =       "";
-        char            *buffer         =       NULL;
-        int             brickcount      =       0;
-        int             i               =       0;
-        int             pid             =       0;
+    int ret = -1;
+    char key[PATH_MAX] = "";
+    char *buffer = NULL;
 
-        GF_ASSERT (dict);
-        GF_ASSERT (key_prefix);
+    GF_ASSERT(snap_dict);
+    GF_ASSERT(dict);
 
-        ret = snprintf (key, sizeof (key), "%s.brickcount", key_prefix);
-        if (ret < 0) {
-                goto out;
-        }
-        ret = dict_get_int32 (dict, key, &brickcount);
-        if (ret) {
-                gf_log ("cli", GF_LOG_ERROR, "Failed to fetch brickcount");
-                goto out;
-        }
+    ret = dict_set_uint32(snap_dict, "sub-cmd", GF_SNAP_STATUS_TYPE_ITER);
+    if (ret) {
+        gf_log("cli", GF_LOG_ERROR, "Could not save command type in snap dict");
+        goto out;
+    }
 
-        for (i = 0 ; i < brickcount ; i++) {
-                ret = snprintf (key, sizeof (key), "%s.brick%d.path",
-                                key_prefix, i);
-                if (ret < 0) {
-                        goto out;
-                }
+    ret = snprintf(key, sizeof(key), "status.snap%d.snapname", index);
+    if (ret < 0) {
+        goto out;
+    }
 
-                ret = dict_get_str (dict, key, &buffer);
-                if (ret) {
-                        gf_log ("cli", GF_LOG_INFO,
-                                "Unable to get Brick Path");
-                        continue;
-                }
-                cli_out ("\n\t%-17s %s   %s", "Brick Path", ":", buffer);
+    ret = dict_get_strn(dict, key, ret, &buffer);
+    if (ret) {
+        gf_log("cli", GF_LOG_ERROR, "Could not get snapname");
+        goto out;
+    }
 
-                ret = snprintf (key, sizeof (key), "%s.brick%d.vgname",
-                                key_prefix, i);
-                if (ret < 0) {
-                        goto out;
-                }
+    ret = dict_set_str_sizen(snap_dict, "snapname", buffer);
+    if (ret) {
+        gf_log("cli", GF_LOG_ERROR, "Could not save snapname in snap dict");
+        goto out;
+    }
 
-                ret = dict_get_str (dict, key, &buffer);
-                if (ret) {
-                        gf_log ("cli", GF_LOG_INFO,
-                                "Unable to get Volume Group");
-                        cli_out ("\t%-17s %s   %s", "Volume Group", ":",
-                                 "N/A");
-                } else
-                        cli_out ("\t%-17s %s   %s", "Volume Group", ":",
-                                 buffer);
-
-                ret = snprintf (key, sizeof (key), "%s.brick%d.status",
-                                key_prefix, i);
-                if (ret < 0) {
-                        goto out;
-                }
+    ret = dict_set_int32_sizen(snap_dict, "type", GF_SNAP_OPTION_TYPE_STATUS);
+    if (ret) {
+        gf_log("cli", GF_LOG_ERROR, "Could not save command type");
+        goto out;
+    }
 
-                ret = dict_get_str (dict, key, &buffer);
-                if (ret) {
-                        gf_log ("cli", GF_LOG_INFO,
-                                "Unable to get Brick Running");
-                        cli_out ("\t%-17s %s   %s", "Brick Running", ":",
-                                 "N/A");
-                } else
-                        cli_out ("\t%-17s %s   %s", "Brick Running", ":",
-                                 buffer);
-
-                ret = snprintf (key, sizeof (key), "%s.brick%d.pid",
-                                key_prefix, i);
-                if (ret < 0) {
-                        goto out;
-                }
+    ret = dict_set_dynstr_with_alloc(snap_dict, "cmd-str", "snapshot status");
+    if (ret) {
+        gf_log("cli", GF_LOG_ERROR, "Could not save command string as status");
+        goto out;
+    }
 
-                ret = dict_get_int32 (dict, key, &pid);
-                if (ret) {
-                        gf_log ("cli", GF_LOG_INFO,
-                                "Unable to get pid");
-                        cli_out ("\t%-17s %s   %s", "Brick PID", ":", "N/A");
-                } else
-                        cli_out ("\t%-17s %s   %d", "Brick PID", ":", pid);
-
-                ret = snprintf (key, sizeof (key), "%s.brick%d.data",
-                                key_prefix, i);
-                if (ret < 0) {
-                        goto out;
-                }
+    ret = dict_set_int32_sizen(snap_dict, "hold_vol_locks", _gf_false);
+    if (ret) {
+        gf_log("cli", GF_LOG_ERROR, "Setting volume lock flag failed");
+        goto out;
+    }
 
-                ret = dict_get_str (dict, key, &buffer);
-                if (ret) {
-                        gf_log ("cli", GF_LOG_INFO,
-                                "Unable to get Data Percent");
-                        cli_out ("\t%-17s %s   %s", "Data Percentage", ":",
-                                 "N/A");
-                } else
-                        cli_out ("\t%-17s %s   %s", "Data Percentage", ":",
-                                 buffer);
-
-                ret = snprintf (key, sizeof (key), "%s.brick%d.lvsize",
-                                key_prefix, i);
-                if (ret < 0) {
-                        goto out;
-                }
-                ret = dict_get_str (dict, key, &buffer);
-                if (ret) {
-                        gf_log ("cli", GF_LOG_INFO, "Unable to get LV Size");
-                        cli_out ("\t%-17s %s   %s", "LV Size", ":", "N/A");
-                } else
-                        cli_out ("\t%-17s %s   %s", "LV Size", ":", buffer);
+out:
+    return ret;
+}
 
+static int
+cli_snapshot_status(dict_t *dict, gf_cli_rsp *rsp, call_frame_t *frame)
+{
+    int ret = -1;
+    int status_cmd = -1;
+    cli_local_t *local = NULL;
+
+    GF_ASSERT(dict);
+    GF_ASSERT(rsp);
+    GF_ASSERT(frame);
+
+    local = ((call_frame_t *)frame)->local;
+    if (!local) {
+        gf_log("cli", GF_LOG_ERROR, "frame->local is NULL");
+        goto out;
+    }
+
+    if (rsp->op_ret) {
+        if (rsp->op_errstr) {
+            ret = dict_set_dynstr_with_alloc(local->dict, "op_err_str",
+                                             rsp->op_errstr);
+            if (ret) {
+                gf_log("cli", GF_LOG_ERROR,
+                       "Failed to set op_errstr in local dictionary");
+                goto out;
+            }
+        }
+        ret = rsp->op_ret;
+        goto out;
+    }
+
+    ret = dict_get_int32_sizen(dict, "sub-cmd", &status_cmd);
+    if (ret) {
+        gf_log("cli", GF_LOG_ERROR, "Could not fetch status type");
+        goto out;
+    }
+
+    if ((status_cmd != GF_SNAP_STATUS_TYPE_SNAP) &&
+        (status_cmd != GF_SNAP_STATUS_TYPE_ITER)) {
+        dict_copy(dict, local->dict);
+        goto out;
+    }
+
+    if (global_state->mode & GLUSTER_MODE_XML) {
+        ret = cli_xml_snapshot_status_single_snap(local, dict, "status.snap0");
+        if (ret) {
+            gf_log("cli", GF_LOG_ERROR,
+                   "Failed to create xml output for snapshot status");
         }
+    } else {
+        ret = cli_get_single_snap_status(dict, "status.snap0");
+        if (ret) {
+            gf_log("cli", GF_LOG_ERROR, "Could not fetch status of snap");
+        }
+    }
 
-        ret = 0;
 out:
-        return ret;
+    return ret;
 }
 
+static int
+gf_cli_generate_snapshot_event(gf_cli_rsp *rsp, dict_t *dict, int32_t type,
+                               char *snap_name, char *volname, char *snap_uuid,
+                               char *clone_name)
+{
+    int ret = -1;
+    int config_command = 0;
+    int32_t delete_cmd = -1;
+    uint64_t hard_limit = 0;
+    uint64_t soft_limit = 0;
+    char *auto_delete = NULL;
+    char *snap_activate = NULL;
+    char msg[PATH_MAX] = {
+        0,
+    };
+    char option[512] = {
+        0,
+    };
+
+    GF_VALIDATE_OR_GOTO("cli", dict, out);
+    GF_VALIDATE_OR_GOTO("cli", rsp, out);
+
+    switch (type) {
+        case GF_SNAP_OPTION_TYPE_CREATE:
+            if (!snap_name) {
+                gf_log("cli", GF_LOG_ERROR, "Failed to get snap name");
+                goto out;
+            }
 
+            if (!volname) {
+                gf_log("cli", GF_LOG_ERROR, "Failed to get volume name");
+                goto out;
+            }
 
-int
-cli_get_single_snap_status (dict_t *dict, char *keyprefix)
-{
-        int             ret             =       -1;
-        char            key[PATH_MAX]   =       "";
-        int             i               =       0;
-        int             volcount        =       0;
-        char            *get_buffer     =       NULL;
-
-        GF_ASSERT (dict);
-        GF_ASSERT (keyprefix);
+            if (rsp->op_ret != 0) {
+                gf_event(EVENT_SNAPSHOT_CREATE_FAILED,
+                         "snapshot_name=%s;volume_name=%s;error=%s", snap_name,
+                         volname,
+                         rsp->op_errstr ? rsp->op_errstr
+                                        : "Please check log file for details");
+                ret = 0;
+                break;
+            }
 
-        ret = snprintf (key, sizeof (key), "%s.snapname", keyprefix);
-        if (ret < 0) {
+            if (!snap_uuid) {
+                gf_log("cli", GF_LOG_ERROR, "Failed to get snap uuid");
                 goto out;
-        }
+            }
 
-        ret = dict_get_str (dict, key, &get_buffer);
-        if (ret) {
-                gf_log ("cli", GF_LOG_ERROR, "Unable to get snapname");
-                goto out;
-        }
-        cli_out ("\nSnap Name : %s", get_buffer);
+            gf_event(EVENT_SNAPSHOT_CREATED,
+                     "snapshot_name=%s;volume_name=%s;snapshot_uuid=%s",
+                     snap_name, volname, snap_uuid);
 
-        ret = snprintf (key, sizeof (key), "%s.uuid", keyprefix);
-        if (ret < 0) {
-                goto out;
-        }
+            ret = 0;
+            break;
 
-        ret = dict_get_str (dict, key, &get_buffer);
-        if (ret) {
-                gf_log ("cli", GF_LOG_ERROR, "Unable to get snap UUID");
+        case GF_SNAP_OPTION_TYPE_ACTIVATE:
+            if (!snap_name) {
+                gf_log("cli", GF_LOG_ERROR, "Failed to get snap name");
                 goto out;
-        }
-        cli_out ("Snap UUID : %s", get_buffer);
+            }
 
-        ret = snprintf (key, sizeof (key), "%s.volcount", keyprefix);
-        if (ret < 0) {
-                goto out;
-        }
+            if (rsp->op_ret != 0) {
+                gf_event(EVENT_SNAPSHOT_ACTIVATE_FAILED,
+                         "snapshot_name=%s;error=%s", snap_name,
+                         rsp->op_errstr ? rsp->op_errstr
+                                        : "Please check log file for details");
+                ret = 0;
+                break;
+            }
 
-        ret = dict_get_int32 (dict, key, &volcount);
-        if (ret) {
-                gf_log ("cli", GF_LOG_ERROR, "Unable to get volume count");
+            if (!snap_uuid) {
+                gf_log("cli", GF_LOG_ERROR, "Failed to get snap uuid");
                 goto out;
-        }
+            }
 
-        for (i = 0 ; i < volcount ; i++) {
-                ret = snprintf (key, sizeof (key), "%s.vol%d", keyprefix, i);
-                if (ret < 0) {
-                        goto out;
-                }
+            gf_event(EVENT_SNAPSHOT_ACTIVATED,
+                     "snapshot_name=%s;snapshot_uuid=%s", snap_name, snap_uuid);
 
-                ret = cli_get_snap_volume_status (dict, key);
-                if (ret) {
-                        gf_log ("cli", GF_LOG_ERROR,
-                                "Could not get snap volume status");
-                        goto out;
-                }
-        }
-out:
-        return ret;
-}
+            ret = 0;
+            break;
 
-int32_t
-cli_populate_req_dict_for_delete (dict_t *snap_dict, dict_t *dict, size_t index)
-{
-        int32_t         ret             = -1;
-        char            key[PATH_MAX]   = "";
-        char            *buffer         = NULL;
-        int             type            = 0;
-        int             snapcount       = 0;
+        case GF_SNAP_OPTION_TYPE_DEACTIVATE:
+            if (!snap_name) {
+                gf_log("cli", GF_LOG_ERROR, "Failed to get snap name");
+                goto out;
+            }
 
-        GF_ASSERT (snap_dict);
-        GF_ASSERT (dict);
+            if (rsp->op_ret != 0) {
+                gf_event(EVENT_SNAPSHOT_DEACTIVATE_FAILED,
+                         "snapshot_name=%s;error=%s", snap_name,
+                         rsp->op_errstr ? rsp->op_errstr
+                                        : "Please check log file for details");
+                ret = 0;
+                break;
+            }
 
-        ret = dict_set_int32 (snap_dict, "sub-cmd",
-                              GF_SNAP_DELETE_TYPE_ITER);
-        if (ret) {
-                gf_log ("cli", GF_LOG_ERROR, "Could not save command "
-                        "type in snap dictionary");
+            if (!snap_uuid) {
+                gf_log("cli", GF_LOG_ERROR, "Failed to get snap uuid");
                 goto out;
-        }
+            }
 
-        ret = snprintf (key, sizeof (key), "snapname%lu", index);
-        if (ret < 0) {
-                goto out;
-        }
+            gf_event(EVENT_SNAPSHOT_DEACTIVATED,
+                     "snapshot_name=%s;snapshot_uuid=%s", snap_name, snap_uuid);
 
-        ret = dict_get_str (dict, key, &buffer);
-        if (ret) {
-                gf_log ("cli", GF_LOG_ERROR, "Failed to get snapname");
-                goto out;
-        }
+            ret = 0;
+            break;
 
-        ret = dict_set_dynstr_with_alloc (snap_dict, "snapname", buffer);
-        if (ret) {
-                gf_log ("cli", GF_LOG_ERROR, "Failed to save snapname");
+        case GF_SNAP_OPTION_TYPE_RESTORE:
+            if (!snap_name) {
+                gf_log("cli", GF_LOG_ERROR, "Failed to get snap name");
                 goto out;
-        }
+            }
 
-        ret = dict_set_int32 (snap_dict, "type", GF_SNAP_OPTION_TYPE_DELETE);
-        if (ret) {
-                gf_log ("cli", GF_LOG_ERROR, "Failed to save command type");
+            if (rsp->op_ret != 0) {
+                gf_event(EVENT_SNAPSHOT_RESTORE_FAILED,
+                         "snapshot_name=%s;error=%s", snap_name,
+                         rsp->op_errstr ? rsp->op_errstr
+                                        : "Please check log file for details");
+                ret = 0;
+                break;
+            }
+
+            if (!snap_uuid) {
+                ret = -1;
+                gf_log("cli", GF_LOG_ERROR, "Failed to get snap uuid");
                 goto out;
-        }
+            }
 
-        ret = dict_set_dynstr_with_alloc (snap_dict, "cmd-str",
-                                       "snapshot delete");
-        if (ret) {
-                gf_log ("cli", GF_LOG_ERROR,
-                        "Could not save command string as delete");
+            if (!volname) {
+                ret = -1;
+                gf_log("cli", GF_LOG_ERROR, "Failed to get volname");
                 goto out;
-        }
-out:
-        return ret;
-}
+            }
 
-int
-cli_populate_req_dict_for_status (dict_t *snap_dict, dict_t *dict, int index) {
-        int             ret             =       -1;
-        char            key[PATH_MAX]   =       "";
-        char            *buffer         =       NULL;
-        int             type            =       0;
-        int             snapcount       =       0;
-
-        GF_ASSERT (snap_dict);
-        GF_ASSERT (dict);
-
-        ret = dict_set_uint32 (snap_dict, "sub-cmd",
-                               GF_SNAP_STATUS_TYPE_SNAP);
-        if (ret) {
-                gf_log ("cli", GF_LOG_ERROR, "Could not save command "
-                        "type in snap dict");
+            gf_event(EVENT_SNAPSHOT_RESTORED,
+                     "snapshot_name=%s;snapshot_uuid=%s;volume_name=%s",
+                     snap_name, snap_uuid, volname);
+
+            ret = 0;
+            break;
+
+        case GF_SNAP_OPTION_TYPE_DELETE:
+            ret = dict_get_int32_sizen(dict, "sub-cmd", &delete_cmd);
+            if (ret) {
+                gf_log("cli", GF_LOG_ERROR, "Could not get sub-cmd");
                 goto out;
-        }
+            }
 
-        ret = snprintf (key, sizeof (key), "status.snap%d.snapname", index);
-         if (ret < 0) {
-                 goto out;
-         }
+            /*
+             * Need not generate any event (success or failure) for delete *
+             * all, as it will trigger individual delete for all snapshots *
+             */
+            if (delete_cmd == GF_SNAP_DELETE_TYPE_ALL) {
+                ret = 0;
+                break;
+            }
 
-        ret = dict_get_str (dict, key, &buffer);
-        if (ret) {
-                gf_log ("cli", GF_LOG_ERROR, "Could not get snapname");
+            if (!snap_name) {
+                gf_log("cli", GF_LOG_ERROR, "Failed to get snap name");
                 goto out;
-        }
+            }
 
-        ret = dict_set_str (snap_dict, "snapname", buffer);
-        if (ret) {
-                gf_log ("cli", GF_LOG_ERROR, "Could not save snapname "
-                        "in snap dict");
+            if (rsp->op_ret != 0) {
+                gf_event(EVENT_SNAPSHOT_DELETE_FAILED,
+                         "snapshot_name=%s;error=%s", snap_name,
+                         rsp->op_errstr ? rsp->op_errstr
+                                        : "Please check log file for details");
+                ret = 0;
+                break;
+            }
+
+            if (!snap_uuid) {
+                ret = -1;
+                gf_log("cli", GF_LOG_ERROR, "Failed to get snap uuid");
                 goto out;
+            }
 
-        }
+            gf_event(EVENT_SNAPSHOT_DELETED,
+                     "snapshot_name=%s;snapshot_uuid=%s", snap_name, snap_uuid);
 
-        ret = dict_set_int32 (snap_dict, "type", GF_SNAP_OPTION_TYPE_STATUS);
-        if (ret) {
-                gf_log ("cli", GF_LOG_ERROR,
-                        "Could not save command type");
+            ret = 0;
+            break;
+
+        case GF_SNAP_OPTION_TYPE_CLONE:
+            if (!clone_name) {
+                gf_log("cli", GF_LOG_ERROR, "Failed to get clone name");
                 goto out;
-        }
+            }
 
-        ret = dict_set_dynstr_with_alloc (snap_dict, "cmd-str",
-                                       "snapshot status");
-        if (ret) {
-                gf_log ("cli", GF_LOG_ERROR,
-                        "Could not save command string as status");
+            if (!snap_name) {
+                gf_log("cli", GF_LOG_ERROR, "Failed to get snapname name");
                 goto out;
-        }
+            }
 
-        ret = dict_set_int32 (snap_dict, "hold_vol_locks", _gf_false);
-        if (ret) {
-                gf_log ("cli", GF_LOG_ERROR,
-                        "Setting volume lock flag failed");
+            if (rsp->op_ret != 0) {
+                gf_event(EVENT_SNAPSHOT_CLONE_FAILED,
+                         "snapshot_name=%s;clone_name=%s;error=%s", snap_name,
+                         clone_name,
+                         rsp->op_errstr ? rsp->op_errstr
+                                        : "Please check log file for details");
+                ret = 0;
+                break;
+            }
+
+            if (!snap_uuid) {
+                ret = -1;
+                gf_log("cli", GF_LOG_ERROR, "Failed to get snap uuid");
                 goto out;
-        }
+            }
 
-out:
-        return ret;
-}
+            gf_event(EVENT_SNAPSHOT_CLONED,
+                     "snapshot_name=%s;clone_name=%s;clone_uuid=%s", snap_name,
+                     clone_name, snap_uuid);
 
-int
-cli_snapshot_status (dict_t *dict, gf_cli_rsp *rsp,
-                             call_frame_t *frame)
-{
-        char            key[PATH_MAX]   =       "";
-        int             ret             =       -1;
-        int             status_cmd      =       -1;
-        cli_local_t     *local          =       NULL;
+            ret = 0;
+            break;
 
-        GF_ASSERT (dict);
-        GF_ASSERT (rsp);
-        GF_ASSERT (frame);
+        case GF_SNAP_OPTION_TYPE_CONFIG:
+            if (rsp->op_ret != 0) {
+                gf_event(EVENT_SNAPSHOT_CONFIG_UPDATE_FAILED, "error=%s",
+                         rsp->op_errstr ? rsp->op_errstr
+                                        : "Please check log file for details");
+                ret = 0;
+                break;
+            }
 
-        local = ((call_frame_t *) frame) -> local;
-        if (!local) {
-                gf_log ("cli", GF_LOG_ERROR, "frame->local is NULL");
+            ret = dict_get_int32_sizen(dict, "config-command", &config_command);
+            if (ret) {
+                gf_log("cli", GF_LOG_ERROR, "Could not fetch config type");
                 goto out;
-        }
+            }
 
-        if (rsp->op_ret) {
-                if (rsp->op_errstr) {
-                        ret = dict_set_dynstr_with_alloc (local->dict,
-                                                          "op_err_str",
-                                                          rsp->op_errstr);
-                        if (ret) {
-                                gf_log ("cli", GF_LOG_ERROR, "Failed to set "
-                                        "op_errstr in local dictionary");
-                                goto out;
-                        }
-                }
-                ret = rsp->op_ret;
-                goto out;
-        }
+            if (config_command == GF_SNAP_CONFIG_DISPLAY) {
+                ret = 0;
+                break;
+            }
 
-        ret = dict_get_int32 (dict, "sub-cmd", &status_cmd);
-        if (ret) {
-                gf_log ("cli", GF_LOG_ERROR, "Could not fetch status type");
-                goto out;
-        }
+            /* These are optional parameters therefore ignore the error */
+            ret = dict_get_uint64(dict, "snap-max-hard-limit", &hard_limit);
+            ret = dict_get_uint64(dict, "snap-max-soft-limit", &soft_limit);
+            ret = dict_get_str_sizen(dict, "auto-delete", &auto_delete);
+            ret = dict_get_str_sizen(dict, "snap-activate-on-create",
+                                     &snap_activate);
 
-        if (status_cmd != GF_SNAP_STATUS_TYPE_SNAP) {
-                dict_copy (dict, local->dict);
+            if (!hard_limit && !soft_limit && !auto_delete && !snap_activate) {
+                ret = -1;
+                gf_log("cli", GF_LOG_ERROR,
+                       "At least one option from "
+                       "snap-max-hard-limit, snap-max-soft-limit, "
+                       "auto-delete and snap-activate-on-create "
+                       "should be set");
                 goto out;
-        }
+            }
 
+            if (hard_limit || soft_limit) {
+                snprintf(option, sizeof(option), "%s=%" PRIu64,
+                         hard_limit ? "hard_limit" : "soft_limit",
+                         hard_limit ? hard_limit : soft_limit);
+            } else if (auto_delete || snap_activate) {
+                snprintf(option, sizeof(option), "%s=%s",
+                         auto_delete ? "auto-delete" : "snap-activate",
+                         auto_delete ? auto_delete : snap_activate);
+            }
 
-        ret = snprintf (key, sizeof (key), "status.snap0");
-        if (ret < 0) {
-                goto out;
-        }
+            volname = NULL;
+            ret = dict_get_str_sizen(dict, "volname", &volname);
 
-        if (global_state->mode & GLUSTER_MODE_XML) {
-                ret = cli_xml_snapshot_status_single_snap (local, dict, key);
-                if (ret) {
-                        gf_log ("cli", GF_LOG_ERROR, "Failed to create "
-                                "xml output for snapshot status");
-                        goto out;
-                }
-        } else {
-                ret = cli_get_single_snap_status (dict, key);
-                if (ret) {
-                        gf_log ("cli", GF_LOG_ERROR, "Could not fetch "
-                                "status of snap");
-                        goto out;
-                }
-        }
+            snprintf(msg, sizeof(msg), "config_type=%s;%s",
+                     volname ? "volume_config" : "system_config", option);
+
+            gf_event(EVENT_SNAPSHOT_CONFIG_UPDATED, "%s", msg);
+
+            ret = 0;
+            break;
+
+        default:
+            gf_log("cli", GF_LOG_WARNING,
+                   "Cannot generate event for unknown type.");
+            ret = 0;
+            goto out;
+    }
 
-        ret = 0;
 out:
-        return ret;
+    return ret;
 }
 
-int
-gf_cli_snapshot_cbk (struct rpc_req *req, struct iovec *iov,
-                     int count, void *myframe)
-{
-        int                   ret                      = -1;
-        gf_cli_rsp            rsp                      = {0, };
-        dict_t               *dict                     = NULL;
-        char                 *snap_name                = NULL;
-        char                 *clone_name               = NULL;
-        int32_t               type                     =  0;
-        call_frame_t         *frame                    = NULL;
-        gf_boolean_t         snap_driven               = _gf_false;
-        int8_t               soft_limit_flag           = -1;
-        char                 *volname                  = NULL;
-
-        if (req->rpc_status == -1) {
-                ret = -1;
-                goto out;
-        }
+/*
+ * Fetch necessary data from dict at one place instead of *
+ * repeating the same code again and again.               *
+ */
+static int
+gf_cli_snapshot_get_data_from_dict(dict_t *dict, char **snap_name,
+                                   char **volname, char **snap_uuid,
+                                   int8_t *soft_limit_flag, char **clone_name)
+{
+    int ret = -1;
 
-        frame = myframe;
+    GF_VALIDATE_OR_GOTO("cli", dict, out);
 
-        ret = xdr_to_generic (*iov, &rsp, (xdrproc_t)xdr_gf_cli_rsp);
-        if (ret < 0) {
-                gf_log (frame->this->name, GF_LOG_ERROR,
-                        "Failed to decode xdr response");
-                goto out;
+    if (snap_name) {
+        ret = dict_get_str_sizen(dict, "snapname", snap_name);
+        if (ret) {
+            gf_log("cli", GF_LOG_DEBUG, "failed to get snapname from dict");
         }
+    }
 
-        dict = dict_new ();
-
-        if (!dict) {
-                ret = -1;
-                goto out;
+    if (volname) {
+        ret = dict_get_str_sizen(dict, "volname1", volname);
+        if (ret) {
+            gf_log("cli", GF_LOG_DEBUG, "failed to get volname1 from dict");
         }
+    }
 
-        ret = dict_unserialize (rsp.dict.dict_val, rsp.dict.dict_len, &dict);
+    if (snap_uuid) {
+        ret = dict_get_str_sizen(dict, "snapuuid", snap_uuid);
+        if (ret) {
+            gf_log("cli", GF_LOG_DEBUG, "failed to get snapuuid from dict");
+        }
+    }
 
-        if (ret)
-                goto out;
+    if (soft_limit_flag) {
+        ret = dict_get_int8(dict, "soft-limit-reach", soft_limit_flag);
+        if (ret) {
+            gf_log("cli", GF_LOG_DEBUG,
+                   "failed to get soft-limit-reach from dict");
+        }
+    }
 
-        ret = dict_get_int32 (dict, "type", &type);
+    if (clone_name) {
+        ret = dict_get_str_sizen(dict, "clonename", clone_name);
         if (ret) {
-                gf_log (frame->this->name, GF_LOG_ERROR, "failed to get type");
-                goto out;
+            gf_log("cli", GF_LOG_DEBUG, "failed to get clonename from dict");
         }
+    }
 
-        /* Snapshot status and delete command is handled separately */
-        if (global_state->mode & GLUSTER_MODE_XML &&
-            GF_SNAP_OPTION_TYPE_STATUS != type &&
-            GF_SNAP_OPTION_TYPE_DELETE != type) {
-                ret = cli_xml_output_snapshot (type, dict, rsp.op_ret,
-                                               rsp.op_errno, rsp.op_errstr);
-                if (ret) {
-                        gf_log ("cli", GF_LOG_ERROR, "Error outputting to xml");
-                }
+    ret = 0;
+out:
+    return ret;
+}
 
-                goto out;
+static int
+gf_cli_snapshot_cbk(struct rpc_req *req, struct iovec *iov, int count,
+                    void *myframe)
+{
+    int ret = -1;
+    gf_cli_rsp rsp = {
+        0,
+    };
+    dict_t *dict = NULL;
+    char *snap_name = NULL;
+    char *clone_name = NULL;
+    int32_t type = 0;
+    call_frame_t *frame = NULL;
+    gf_boolean_t snap_driven = _gf_false;
+    int8_t soft_limit_flag = -1;
+    char *volname = NULL;
+    char *snap_uuid = NULL;
+
+    GF_ASSERT(myframe);
+
+    if (req->rpc_status == -1) {
+        goto out;
+    }
+
+    frame = myframe;
+
+    ret = xdr_to_generic(*iov, &rsp, (xdrproc_t)xdr_gf_cli_rsp);
+    if (ret < 0) {
+        gf_log(frame->this->name, GF_LOG_ERROR, XDR_DECODE_FAIL);
+        goto out;
+    }
+
+    dict = dict_new();
+
+    if (!dict) {
+        ret = -1;
+        goto out;
+    }
+
+    ret = dict_unserialize(rsp.dict.dict_val, rsp.dict.dict_len, &dict);
+
+    if (ret)
+        goto out;
+
+    ret = dict_get_int32_sizen(dict, "type", &type);
+    if (ret) {
+        gf_log(frame->this->name, GF_LOG_ERROR, "failed to get type");
+        goto out;
+    }
+
+    ret = gf_cli_snapshot_get_data_from_dict(
+        dict, &snap_name, &volname, &snap_uuid, &soft_limit_flag, &clone_name);
+    if (ret) {
+        gf_log("cli", GF_LOG_ERROR, "Failed to fetch data from dict.");
+        goto out;
+    }
+
+#if (USE_EVENTS)
+    ret = gf_cli_generate_snapshot_event(&rsp, dict, type, snap_name, volname,
+                                         snap_uuid, clone_name);
+    if (ret) {
+        gf_log("cli", GF_LOG_ERROR, "Failed to generate snapshot event");
+        goto out;
+    }
+#endif
+
+    /* Snapshot status and delete command is handled separately */
+    if (global_state->mode & GLUSTER_MODE_XML &&
+        GF_SNAP_OPTION_TYPE_STATUS != type &&
+        GF_SNAP_OPTION_TYPE_DELETE != type) {
+        ret = cli_xml_output_snapshot(type, dict, rsp.op_ret, rsp.op_errno,
+                                      rsp.op_errstr);
+        if (ret) {
+            gf_log("cli", GF_LOG_ERROR, XML_ERROR);
         }
 
-        switch (type) {
+        goto out;
+    }
+
+    switch (type) {
         case GF_SNAP_OPTION_TYPE_CREATE:
-                if (rsp.op_ret) {
-                        cli_err("snapshot create: failed: %s",
-                                 rsp.op_errstr ? rsp.op_errstr :
-                                 "Please check log file for details");
-                                 ret = rsp.op_ret;
-                                 goto out;
-                }
+            if (rsp.op_ret) {
+                cli_err("snapshot create: failed: %s",
+                        rsp.op_errstr ? rsp.op_errstr
+                                      : "Please check log file for details");
+                ret = rsp.op_ret;
+                goto out;
+            }
 
-                ret = dict_get_str (dict, "snapname", &snap_name);
-                if (ret) {
-                        gf_log ("cli", GF_LOG_ERROR,
-                                "Failed to get snap name");
-                        goto out;
-                }
+            if (!snap_name) {
+                ret = -1;
+                gf_log("cli", GF_LOG_ERROR, "Failed to get snap name");
+                goto out;
+            }
 
-                /* TODO : Instead of using volname1 directly use
-                 * volname$i in loop once snapshot of multiple
-                 * volumes are supported
-                 */
-                ret = dict_get_str (dict, "volname1", &volname);
-                if (ret) {
-                        gf_log ("cli", GF_LOG_ERROR,
-                                "Failed to get volume name");
-                        goto out;
-                }
+            if (!volname) {
+                ret = -1;
+                gf_log("cli", GF_LOG_ERROR, "Failed to get volume name");
+                goto out;
+            }
 
-                cli_out ("snapshot create: success: Snap %s created "
-                                        "successfully", snap_name);
+            cli_out("snapshot create: success: Snap %s created successfully",
+                    snap_name);
 
-                ret = dict_get_int8 (dict, "soft-limit-reach",
-                                    &soft_limit_flag);
-                if (soft_limit_flag == 1) {
-                        cli_out ("Warning: Soft-limit of volume (%s) is "
-                                "reached. Snapshot creation is not possible "
-                                "once hard-limit is reached.", volname);
-                }
-                ret = 0;
-                break;
+            if (soft_limit_flag == 1) {
+                cli_out(
+                    "Warning: Soft-limit of volume (%s) is "
+                    "reached. Snapshot creation is not possible "
+                    "once hard-limit is reached.",
+                    volname);
+            }
+            ret = 0;
+            break;
 
         case GF_SNAP_OPTION_TYPE_CLONE:
-                if (rsp.op_ret) {
-                        cli_err("snapshot clone: failed: %s",
-                                 rsp.op_errstr ? rsp.op_errstr :
-                                 "Please check log file for details");
-                                 ret = rsp.op_ret;
-                                 goto out;
-                }
+            if (rsp.op_ret) {
+                cli_err("snapshot clone: failed: %s",
+                        rsp.op_errstr ? rsp.op_errstr
+                                      : "Please check log file for details");
+                ret = rsp.op_ret;
+                goto out;
+            }
 
-                ret = dict_get_str (dict, "clonename", &clone_name);
-                if (ret) {
-                        gf_log ("cli", GF_LOG_ERROR,
-                                "Failed to get clone name");
-                        goto out;
-                }
+            if (!clone_name) {
+                gf_log("cli", GF_LOG_ERROR, "Failed to get clone name");
+                goto out;
+            }
 
-                ret = dict_get_str (dict, "snapname", &snap_name);
-                if (ret) {
-                        gf_log ("cli", GF_LOG_ERROR,
-                                "Failed to get snapname name");
-                        goto out;
-                }
+            if (!snap_name) {
+                gf_log("cli", GF_LOG_ERROR, "Failed to get snapname name");
+                goto out;
+            }
 
-                cli_out ("snapshot clone: success: Clone %s created "
-                                        "successfully", clone_name);
+            cli_out("snapshot clone: success: Clone %s created successfully",
+                    clone_name);
 
-                ret = 0;
-                break;
+            ret = 0;
+            break;
 
         case GF_SNAP_OPTION_TYPE_RESTORE:
-                /* TODO: Check if rsp.op_ret needs to be checked here. Or is
-                 * it ok to check this in the start of the function where we
-                 * get rsp.*/
-                if (rsp.op_ret) {
-                        cli_err("snapshot restore: failed: %s",
-                                 rsp.op_errstr ? rsp.op_errstr :
-                                 "Please check log file for details");
-                                 ret = rsp.op_ret;
-                                 goto out;
-                }
+            if (rsp.op_ret) {
+                cli_err("snapshot restore: failed: %s",
+                        rsp.op_errstr ? rsp.op_errstr
+                                      : "Please check log file for details");
+                ret = rsp.op_ret;
+                goto out;
+            }
 
-                ret = dict_get_str (dict, "snapname", &snap_name);
-                if (ret) {
-                        gf_log ("cli", GF_LOG_ERROR,
-                                "Failed to get snap name");
-                        goto out;
-                }
+            if (!snap_name) {
+                gf_log("cli", GF_LOG_ERROR, "Failed to get snap name");
+                goto out;
+            }
 
-                cli_out ("Snapshot restore: %s: Snap restored "
-                                 "successfully", snap_name);
+            cli_out("Snapshot restore: %s: Snap restored successfully",
+                    snap_name);
 
-                ret = 0;
-                break;
+            ret = 0;
+            break;
         case GF_SNAP_OPTION_TYPE_ACTIVATE:
-                /* TODO: Check if rsp.op_ret needs to be checked here. Or is
-                 * it ok to check this in the start of the function where we
-                 * get rsp.*/
-                if (rsp.op_ret) {
-                        cli_err("snapshot activate: failed: %s",
-                                 rsp.op_errstr ? rsp.op_errstr :
-                                 "Please check log file for details");
-                                 ret = rsp.op_ret;
-                                 goto out;
-                }
+            if (rsp.op_ret) {
+                cli_err("snapshot activate: failed: %s",
+                        rsp.op_errstr ? rsp.op_errstr
+                                      : "Please check log file for details");
+                ret = rsp.op_ret;
+                goto out;
+            }
 
-                ret = dict_get_str (dict, "snapname", &snap_name);
-                if (ret) {
-                        gf_log ("cli", GF_LOG_ERROR,
-                                "Failed to get snap name");
-                        goto out;
-                }
+            if (!snap_name) {
+                gf_log("cli", GF_LOG_ERROR, "Failed to get snap name");
+                goto out;
+            }
 
-                cli_out ("Snapshot activate: %s: Snap activated "
-                                 "successfully", snap_name);
+            cli_out("Snapshot activate: %s: Snap activated successfully",
+                    snap_name);
 
-                ret = 0;
-                break;
+            ret = 0;
+            break;
 
         case GF_SNAP_OPTION_TYPE_DEACTIVATE:
-                /* TODO: Check if rsp.op_ret needs to be checked here. Or is
-                 * it ok to check this in the start of the function where we
-                 * get rsp.*/
-                if (rsp.op_ret) {
-                        cli_err("snapshot deactivate: failed: %s",
-                                 rsp.op_errstr ? rsp.op_errstr :
-                                 "Please check log file for details");
-                                 ret = rsp.op_ret;
-                                 goto out;
-                }
+            if (rsp.op_ret) {
+                cli_err("snapshot deactivate: failed: %s",
+                        rsp.op_errstr ? rsp.op_errstr
+                                      : "Please check log file for details");
+                ret = rsp.op_ret;
+                goto out;
+            }
 
-                ret = dict_get_str (dict, "snapname", &snap_name);
-                if (ret) {
-                        gf_log ("cli", GF_LOG_ERROR,
-                                "Failed to get snap name");
-                        goto out;
-                }
+            if (!snap_name) {
+                gf_log("cli", GF_LOG_ERROR, "Failed to get snap name");
+                goto out;
+            }
 
-                cli_out ("Snapshot deactivate: %s: Snap deactivated "
-                                 "successfully", snap_name);
+            cli_out("Snapshot deactivate: %s: Snap deactivated successfully",
+                    snap_name);
 
-                ret = 0;
-                break;
+            ret = 0;
+            break;
         case GF_SNAP_OPTION_TYPE_INFO:
-                if (rsp.op_ret) {
-                        cli_err ("Snapshot info : failed: %s",
-                                  rsp.op_errstr ? rsp.op_errstr :
-                                  "Please check log file for details");
-                        ret = rsp.op_ret;
-                        goto out;
-                }
+            if (rsp.op_ret) {
+                cli_err("Snapshot info : failed: %s",
+                        rsp.op_errstr ? rsp.op_errstr
+                                      : "Please check log file for details");
+                ret = rsp.op_ret;
+                goto out;
+            }
 
-                snap_driven = dict_get_str_boolean (dict, "snap-driven",
-                                                    _gf_false);
-                if (snap_driven == _gf_true) {
-                        ret = cli_call_snapshot_info  (dict, snap_driven);
-                        if (ret) {
-                                gf_log ("cli", GF_LOG_ERROR,
-                                        "Snapshot info failed");
-                                goto out;
-                        }
-                } else if (snap_driven == _gf_false) {
-                        ret = cli_get_snaps_in_volume (dict);
-                        if (ret) {
-                                gf_log ("cli", GF_LOG_ERROR,
-                                        "Snapshot info failed");
-                                goto out;
-                        }
+            snap_driven = dict_get_str_boolean(dict, "snap-driven", _gf_false);
+            if (snap_driven == _gf_true) {
+                ret = cli_call_snapshot_info(dict, snap_driven);
+                if (ret) {
+                    gf_log("cli", GF_LOG_ERROR, "Snapshot info failed");
+                    goto out;
                 }
-                break;
-
-        case GF_SNAP_OPTION_TYPE_CONFIG:
-                ret = cli_snapshot_config_display (dict, &rsp);
+            } else if (snap_driven == _gf_false) {
+                ret = cli_get_snaps_in_volume(dict);
                 if (ret) {
-                        gf_log ("cli", GF_LOG_ERROR, "Failed to display "
-                                "snapshot config output.");
-                        goto out;
+                    gf_log("cli", GF_LOG_ERROR, "Snapshot info failed");
+                    goto out;
                 }
-                break;
+            }
+            break;
+
+        case GF_SNAP_OPTION_TYPE_CONFIG:
+            ret = cli_snapshot_config_display(dict, &rsp);
+            if (ret) {
+                gf_log("cli", GF_LOG_ERROR,
+                       "Failed to display snapshot config output.");
+                goto out;
+            }
+            break;
 
         case GF_SNAP_OPTION_TYPE_LIST:
-                if (rsp.op_ret) {
-                        cli_err ("Snapshot list : failed: %s",
-                                 rsp.op_errstr ? rsp.op_errstr :
-                                "Please check log file for details");
-                        ret = rsp.op_ret;
-                        goto out;
-                }
+            if (rsp.op_ret) {
+                cli_err("Snapshot list : failed: %s",
+                        rsp.op_errstr ? rsp.op_errstr
+                                      : "Please check log file for details");
+                ret = rsp.op_ret;
+                goto out;
+            }
 
-                ret = cli_snapshot_list (dict);
-                if (ret) {
-                        gf_log ("cli", GF_LOG_ERROR, "Failed to display "
-                                "snapshot list");
-                        goto out;
-                }
-                break;
+            ret = cli_snapshot_list(dict);
+            if (ret) {
+                gf_log("cli", GF_LOG_ERROR, "Failed to display snapshot list");
+                goto out;
+            }
+            break;
 
         case GF_SNAP_OPTION_TYPE_DELETE:
-                ret = cli_snapshot_remove_reply (&rsp, dict, frame);
-                if (ret) {
-                        gf_log ("cli", GF_LOG_ERROR,
-                                "Failed to delete snap");
-                        goto out;
-                }
-                break;
+            ret = cli_snapshot_remove_reply(&rsp, dict, frame);
+            if (ret) {
+                gf_log("cli", GF_LOG_ERROR, "Failed to delete snap");
+                goto out;
+            }
+            break;
 
         case GF_SNAP_OPTION_TYPE_STATUS:
-                ret = cli_snapshot_status (dict, &rsp, frame);
-                if (ret) {
-                        gf_log ("cli", GF_LOG_ERROR, "Failed to display "
-                                "snapshot status output.");
-                        goto out;
-                }
-                break;
+            ret = cli_snapshot_status(dict, &rsp, frame);
+            if (ret) {
+                gf_log("cli", GF_LOG_ERROR,
+                       "Failed to display snapshot status output.");
+                goto out;
+            }
+            break;
 
         default:
-                cli_err ("Unknown command executed");
-                ret = -1;
-                goto out;
-        }
+            cli_err("Unknown command executed");
+            ret = -1;
+            goto out;
+    }
 out:
-        if (dict)
-                dict_unref (dict);
-        cli_cmd_broadcast_response (ret);
+    if (dict)
+        dict_unref(dict);
+    cli_cmd_broadcast_response(ret);
 
-        free (rsp.dict.dict_val);
-        free (rsp.op_errstr);
+    free(rsp.dict.dict_val);
+    free(rsp.op_errstr);
 
-        return ret;
+    return ret;
 }
 
 int32_t
-gf_cli_snapshot_for_delete (call_frame_t *frame, xlator_t *this,
-                            void *data)
-{
-        gf_cli_req      req                     = {{0,}};
-        dict_t          *options                = NULL;
-        int32_t         ret                     = -1;
-        int32_t         cmd                     = -1;
-        cli_local_t     *local                  = NULL;
-        dict_t          *snap_dict              = NULL;
-        int32_t         snapcount               = 0;
-        int             i                       = 0;
-        char            question[PATH_MAX]      = "";
-        char            *volname                = NULL;
-        gf_answer_t     answer                  = GF_ANSWER_NO;
-
-        GF_VALIDATE_OR_GOTO ("cli", frame,  out);
-        GF_VALIDATE_OR_GOTO ("cli", frame->local, out);
-        GF_VALIDATE_OR_GOTO ("cli", this, out);
-        GF_VALIDATE_OR_GOTO ("cli", data, out);
-
-        local = frame->local;
-
-        options = data;
-
-        ret = dict_get_int32 (local->dict, "sub-cmd", &cmd);
-        if (ret) {
-                gf_log ("cli", GF_LOG_ERROR, "Failed to get "
-                        "sub-cmd");
-                goto out;
-        }
+gf_cli_snapshot_for_delete(call_frame_t *frame, xlator_t *this, void *data)
+{
+    gf_cli_req req = {{
+        0,
+    }};
+    int32_t ret = -1;
+    int32_t cmd = -1;
+    cli_local_t *local = NULL;
+    dict_t *snap_dict = NULL;
+    int32_t snapcount = 0;
+    int i = 0;
+    char question[PATH_MAX] = "";
+    char *volname = NULL;
+    gf_answer_t answer = GF_ANSWER_NO;
+
+    GF_VALIDATE_OR_GOTO("cli", frame, out);
+    GF_VALIDATE_OR_GOTO("cli", frame->local, out);
+    GF_VALIDATE_OR_GOTO("cli", this, out);
+    GF_VALIDATE_OR_GOTO("cli", data, out);
+
+    local = frame->local;
+
+    ret = dict_get_int32_sizen(local->dict, "sub-cmd", &cmd);
+    if (ret) {
+        gf_log("cli", GF_LOG_ERROR, "Failed to get sub-cmd");
+        goto out;
+    }
+
+    /* No need multiple RPCs for individual snapshot delete*/
+    if (cmd == GF_SNAP_DELETE_TYPE_SNAP) {
+        ret = 0;
+        goto out;
+    }
 
-        /* No need multiple RPCs for individual snapshot delete*/
-        if (cmd == GF_SNAP_DELETE_TYPE_SNAP) {
-                ret = 0;
-                goto out;
-        }
+    ret = dict_get_int32_sizen(local->dict, "snapcount", &snapcount);
+    if (ret) {
+        gf_log("cli", GF_LOG_ERROR, "Could not get snapcount");
+        goto out;
+    }
 
-        ret = dict_get_int32 (local->dict, "snapcount",
-                              &snapcount);
+    if (global_state->mode & GLUSTER_MODE_XML) {
+#ifdef HAVE_LIB_XML
+        ret = xmlTextWriterWriteFormatElement(
+            local->writer, (xmlChar *)"snapCount", "%d", snapcount);
         if (ret) {
-                gf_log ("cli", GF_LOG_ERROR, "Could not get "
-                        "snapcount");
-                goto out;
+            gf_log("cli", GF_LOG_ERROR,
+                   "Failed to write xml element \"snapCount\"");
+            goto out;
         }
-
-        if (global_state->mode & GLUSTER_MODE_XML) {
-#ifdef HAVE_LIB_XML
-                ret = xmlTextWriterWriteFormatElement (local->writer,
-                                                (xmlChar *)"snapCount",
-                                                "%d", snapcount);
-                if (ret) {
-                        gf_log ("cli", GF_LOG_ERROR, "Failed to write "
-                                "xml element \"snapCount\"");
-                        goto out;
-                }
 #endif /* HAVE_LIB_XML */
-        } else if (snapcount == 0) {
-                cli_out ("No snapshots present");
-                goto out;
+    } else if (snapcount == 0) {
+        cli_out("No snapshots present");
+        goto out;
+    }
+
+    if (cmd == GF_SNAP_DELETE_TYPE_ALL) {
+        snprintf(question, sizeof(question),
+                 "System contains %d snapshot(s).\nDo you still "
+                 "want to continue and delete them? ",
+                 snapcount);
+    } else {
+        ret = dict_get_str_sizen(local->dict, "volname", &volname);
+        if (ret) {
+            gf_log("cli", GF_LOG_ERROR,
+                   "Failed to fetch volname from local dictionary");
+            goto out;
         }
 
-        if (cmd == GF_SNAP_DELETE_TYPE_ALL) {
-                snprintf (question, sizeof (question), "System contains %d "
-                          "snapshot(s).\nDo you still "
-                          "want to continue and delete them? ",
-                           snapcount);
-        } else {
-                ret = dict_get_str (local->dict, "volname", &volname);
-                if (ret) {
-                        gf_log ("cli", GF_LOG_ERROR, "Failed to fetch "
-                                "volname from local dictionary");
-                        goto out;
-                }
-
-                snprintf (question, sizeof (question), "Volume (%s) contains "
-                          "%d snapshot(s).\nDo you still want to "
-                           "continue and delete them? ", volname,
-                           snapcount);
-        }
+        snprintf(question, sizeof(question),
+                 "Volume (%s) contains %d snapshot(s).\nDo you still want to "
+                 "continue and delete them? ",
+                 volname, snapcount);
+    }
 
-        answer = cli_cmd_get_confirmation (global_state, question);
-        if (GF_ANSWER_NO == answer) {
-                ret = 0;
-                gf_log ("cli", GF_LOG_DEBUG, "User cancelled "
-                        "snapshot delete operation for snap delete");
-                goto out;
-        }
+    answer = cli_cmd_get_confirmation(global_state, question);
+    if (GF_ANSWER_NO == answer) {
+        ret = 0;
+        gf_log("cli", GF_LOG_DEBUG,
+               "User cancelled snapshot delete operation for snap delete");
+        goto out;
+    }
 
-        for (i = 1 ; i <= snapcount ; i++) {
-                ret = -1;
+    for (i = 1; i <= snapcount; i++) {
+        ret = -1;
 
-                snap_dict = dict_new();
-                if (!snap_dict)
-                        goto out;
+        snap_dict = dict_new();
+        if (!snap_dict)
+            goto out;
 
-                ret = cli_populate_req_dict_for_delete (snap_dict,
-                                                       local->dict, i);
-                if (ret) {
-                        gf_log ("cli", GF_LOG_ERROR, "Could not "
-                                "populate snap request dictionary");
-                        goto out;
-                }
+        ret = cli_populate_req_dict_for_delete(snap_dict, local->dict, i);
+        if (ret) {
+            gf_log("cli", GF_LOG_ERROR,
+                   "Could not populate snap request dictionary");
+            goto out;
+        }
 
-                ret = cli_to_glusterd (&req, frame,
-                                gf_cli_snapshot_cbk,
-                                (xdrproc_t) xdr_gf_cli_req, snap_dict,
-                                GLUSTER_CLI_SNAP, this, cli_rpc_prog,
-                                NULL);
-                if (ret) {
-                        /* Fail the operation if deleting one of the
-                         * snapshots is failed
-                         */
-                        gf_log ("cli", GF_LOG_ERROR, "cli_to_glusterd "
-                                "for snapshot delete failed");
-                        goto out;
-                }
-                dict_unref (snap_dict);
-                snap_dict = NULL;
+        ret = cli_to_glusterd(&req, frame, gf_cli_snapshot_cbk,
+                              (xdrproc_t)xdr_gf_cli_req, snap_dict,
+                              GLUSTER_CLI_SNAP, this, cli_rpc_prog, NULL);
+        if (ret) {
+            /* Fail the operation if deleting one of the
+             * snapshots is failed
+             */
+            gf_log("cli", GF_LOG_ERROR,
+                   "cli_to_glusterd for snapshot delete failed");
+            goto out;
         }
+        dict_unref(snap_dict);
+        snap_dict = NULL;
+    }
 
 out:
-        if (snap_dict)
-                dict_unref (snap_dict);
+    if (snap_dict)
+        dict_unref(snap_dict);
+    GF_FREE(req.dict.dict_val);
 
-        return ret;
+    return ret;
 }
 
-int32_t
-gf_cli_snapshot_for_status (call_frame_t *frame, xlator_t *this,
-                            void *data)
-{
+static int32_t
+gf_cli_snapshot_for_status(call_frame_t *frame, xlator_t *this, void *data)
+{
+    gf_cli_req req = {{
+        0,
+    }};
+    int ret = -1;
+    int32_t cmd = -1;
+    cli_local_t *local = NULL;
+    dict_t *snap_dict = NULL;
+    int snapcount = 0;
+    int i = 0;
+
+    GF_VALIDATE_OR_GOTO("cli", frame, out);
+    GF_VALIDATE_OR_GOTO("cli", frame->local, out);
+    GF_VALIDATE_OR_GOTO("cli", this, out);
+    GF_VALIDATE_OR_GOTO("cli", data, out);
+
+    local = frame->local;
+
+    ret = dict_get_int32_sizen(local->dict, "sub-cmd", &cmd);
+    if (ret) {
+        gf_log("cli", GF_LOG_ERROR, "Failed to get sub-cmd");
+        goto out;
+    }
+
+    /* Snapshot status of single snap (i.e. GF_SNAP_STATUS_TYPE_SNAP)
+     * is already handled. Therefore we can return from here.
+     * If want to get status of all snaps in the system or volume then
+     * we should get them one by one.*/
+    if ((cmd == GF_SNAP_STATUS_TYPE_SNAP) ||
+        (cmd == GF_SNAP_STATUS_TYPE_ITER)) {
+        ret = 0;
+        goto out;
+    }
 
-        gf_cli_req        req           = {{0,}};
-        dict_t           *options       = NULL;
-        int               ret           = -1;
-        int32_t           cmd           = -1;
-        cli_local_t      *local         = NULL;
-        dict_t           *snap_dict     = NULL;
-        int              snapcount      =  0;
-        int              i              =  0;
+    ret = dict_get_int32_sizen(local->dict, "status.snapcount", &snapcount);
+    if (ret) {
+        gf_log("cli", GF_LOG_ERROR, "Could not get snapcount");
+        goto out;
+    }
 
-        GF_VALIDATE_OR_GOTO ("cli", frame, out);
-        GF_VALIDATE_OR_GOTO ("cli", frame->local, out);
-        GF_VALIDATE_OR_GOTO ("cli", this, out);
-        GF_VALIDATE_OR_GOTO ("cli", data, out);
+    if (snapcount == 0 && !(global_state->mode & GLUSTER_MODE_XML)) {
+        cli_out("No snapshots present");
+    }
 
-        local = frame->local;
-        options = data;
+    for (i = 0; i < snapcount; i++) {
+        ret = -1;
 
-        ret = dict_get_int32 (local->dict, "sub-cmd", &cmd);
-        if (ret) {
-                gf_log ("cli", GF_LOG_ERROR, "Failed to get sub-cmd");
-                goto out;
-        }
+        snap_dict = dict_new();
+        if (!snap_dict)
+            goto out;
 
-        /* Snapshot status of single snap (i.e. GF_SNAP_STATUS_TYPE_SNAP)
-         * is already handled. Therefore we can return from here.
-         * If want to get status of all snaps in the system or volume then
-         * we should get them one by one.*/
-        if (cmd == GF_SNAP_STATUS_TYPE_SNAP) {
-                ret = 0;
-                goto out;
-        }
-
-        ret = dict_get_int32 (local->dict, "status.snapcount", &snapcount);
+        ret = cli_populate_req_dict_for_status(snap_dict, local->dict, i);
         if (ret) {
-                gf_log ("cli", GF_LOG_ERROR, "Could not get snapcount");
-                goto out;
-        }
-
-        if (snapcount == 0 && !(global_state->mode & GLUSTER_MODE_XML)) {
-                cli_out ("No snapshots present");
+            gf_log("cli", GF_LOG_ERROR,
+                   "Could not populate snap request dictionary");
+            goto out;
         }
 
-        for (i = 0 ; i < snapcount; i++) {
-                ret = -1;
-
-                snap_dict = dict_new();
-                if (!snap_dict)
-                        goto out;
+        ret = cli_to_glusterd(&req, frame, gf_cli_snapshot_cbk,
+                              (xdrproc_t)xdr_gf_cli_req, snap_dict,
+                              GLUSTER_CLI_SNAP, this, cli_rpc_prog, NULL);
 
-                ret = cli_populate_req_dict_for_status (snap_dict,
-                                local->dict, i);
-                if (ret) {
-                        gf_log ("cli", GF_LOG_ERROR, "Could not "
-                                        "populate snap request dictionary");
-                        goto out;
-                }
-
-                ret = cli_to_glusterd (&req, frame,
-                                gf_cli_snapshot_cbk,
-                                (xdrproc_t) xdr_gf_cli_req, snap_dict,
-                                GLUSTER_CLI_SNAP, this, cli_rpc_prog,
-                                NULL);
-
-                /* Ignore the return value and error for snapshot
-                 * status of type "ALL" or "VOL"
-                 *
-                 * Scenario : There might be case where status command
-                 * and delete command might be issued at the same time.
-                 * In that case when status tried to fetch detail of
-                 * snap which has been deleted by concurrent command,
-                 * then it will show snapshot not present. Which will
-                 * not be appropriate.
-                 */
-                dict_unref (snap_dict);
-                snap_dict = NULL;
+        /* Ignore the return value and error for snapshot
+         * status of type "ALL" or "VOL"
+         *
+         * Scenario : There might be case where status command
+         * and delete command might be issued at the same time.
+         * In that case when status tried to fetch detail of
+         * snap which has been deleted by concurrent command,
+         * then it will show snapshot not present. Which will
+         * not be appropriate.
+         */
+        if (ret && (cmd != GF_SNAP_STATUS_TYPE_ALL &&
+                    cmd != GF_SNAP_STATUS_TYPE_VOL)) {
+            gf_log("cli", GF_LOG_ERROR,
+                   "cli_to_glusterd for snapshot status failed");
+            goto out;
         }
+        dict_unref(snap_dict);
+        snap_dict = NULL;
+    }
 
-        ret = 0;
+    ret = 0;
 out:
-        if (snap_dict)
-                dict_unref (snap_dict);
+    if (snap_dict)
+        dict_unref(snap_dict);
+    GF_FREE(req.dict.dict_val);
 
-        return ret;
+    return ret;
 }
 
-int32_t
-gf_cli_snapshot (call_frame_t *frame, xlator_t *this,
-                 void *data)
-{
-        gf_cli_req        req           = {{0,}};
-        dict_t           *options       = NULL;
-        int               ret           = -1;
-        int               tmp_ret       = -1;
-        cli_local_t      *local         = NULL;
-        char             *err_str       = NULL;
-        int              type           =  -1;
-
-        if (!frame || !this || !data)
-                goto out;
-
-        if (!frame->local)
-                goto out;
+static int32_t
+gf_cli_snapshot(call_frame_t *frame, xlator_t *this, void *data)
+{
+    gf_cli_req req = {{
+        0,
+    }};
+    dict_t *options = NULL;
+    int ret = -1;
+    int tmp_ret = -1;
+    cli_local_t *local = NULL;
+    char *err_str = NULL;
+    int type = -1;
 
-        local = frame->local;
+    if (!frame || !frame->local || !this || !data)
+        goto out;
 
-        options = data;
+    local = frame->local;
 
-        ret = dict_get_int32 (local->dict, "type", &type);
+    options = data;
 
-        if (global_state->mode & GLUSTER_MODE_XML) {
-                ret = cli_xml_snapshot_begin_composite_op (local);
-                if (ret) {
-                        gf_log ("cli", GF_LOG_ERROR, "Failed to begin "
-                                "snapshot xml composite op");
-                        goto out;
-                }
+    if (global_state->mode & GLUSTER_MODE_XML) {
+        ret = cli_xml_snapshot_begin_composite_op(local);
+        if (ret) {
+            gf_log("cli", GF_LOG_ERROR,
+                   "Failed to begin snapshot xml composite op");
+            goto out;
         }
+    }
+
+    ret = cli_to_glusterd(&req, frame, gf_cli_snapshot_cbk,
+                          (xdrproc_t)xdr_gf_cli_req, options, GLUSTER_CLI_SNAP,
+                          this, cli_rpc_prog, NULL);
+    if (ret) {
+        gf_log("cli", GF_LOG_ERROR, "cli_to_glusterd for snapshot failed");
+        goto xmlend;
+    }
 
+    ret = dict_get_int32_sizen(local->dict, "type", &type);
 
-        ret = cli_to_glusterd (&req, frame, gf_cli_snapshot_cbk,
-                               (xdrproc_t) xdr_gf_cli_req, options,
-                               GLUSTER_CLI_SNAP, this, cli_rpc_prog,
-                               NULL);
+    if (GF_SNAP_OPTION_TYPE_STATUS == type) {
+        ret = gf_cli_snapshot_for_status(frame, this, data);
         if (ret) {
-                gf_log ("cli", GF_LOG_ERROR, "cli_to_glusterd for "
-                        "snapshot failed");
-                goto xmlend;
+            gf_log("cli", GF_LOG_ERROR,
+                   "cli to glusterd for snapshot status command failed");
         }
 
-        if (GF_SNAP_OPTION_TYPE_STATUS == type) {
-                ret = gf_cli_snapshot_for_status (frame, this, data);
-                if (ret) {
-                        gf_log ("cli", GF_LOG_ERROR, "cli to glusterd "
-                                "for snapshot status command failed");
-                }
+        goto xmlend;
+    }
 
-                goto xmlend;
+    if (GF_SNAP_OPTION_TYPE_DELETE == type) {
+        ret = gf_cli_snapshot_for_delete(frame, this, data);
+        if (ret) {
+            gf_log("cli", GF_LOG_ERROR,
+                   "cli to glusterd for snapshot delete command failed");
         }
 
-        if (GF_SNAP_OPTION_TYPE_DELETE == type) {
-                ret = gf_cli_snapshot_for_delete (frame, this, data);
-                if (ret) {
-                        gf_log ("cli", GF_LOG_ERROR, "cli to glusterd "
-                                "for snapshot delete command failed");
-                }
+        goto xmlend;
+    }
 
-                goto xmlend;
-        }
-
-        ret = 0;
+    ret = 0;
 
 xmlend:
-        if (global_state->mode & GLUSTER_MODE_XML) {
-                ret = cli_xml_snapshot_end_composite_op (local);
-                if (ret) {
-                        gf_log ("cli", GF_LOG_ERROR, "Failed to end "
-                                "snapshot xml composite op");
-                        goto out;
-                }
+    if (global_state->mode & GLUSTER_MODE_XML) {
+        ret = cli_xml_snapshot_end_composite_op(local);
+        if (ret) {
+            gf_log("cli", GF_LOG_ERROR,
+                   "Failed to end snapshot xml composite op");
+            goto out;
         }
+    }
 out:
-        if (ret && local && GF_SNAP_OPTION_TYPE_STATUS == type) {
-                tmp_ret = dict_get_str (local->dict, "op_err_str", &err_str);
-                if (err_str) {
-                        cli_err ("Snapshot Status : failed: %s", err_str);
-                        dict_del (local->dict, "op_err_str");
-                } else {
-                        cli_err ("Snapshot Status : failed: %s", "Please "
-                                "check log file for details");
-                }
+    if (ret && local && GF_SNAP_OPTION_TYPE_STATUS == type) {
+        tmp_ret = dict_get_str_sizen(local->dict, "op_err_str", &err_str);
+        if (tmp_ret || !err_str) {
+            cli_err("Snapshot Status : failed: %s",
+                    "Please check log file for details");
+        } else {
+            cli_err("Snapshot Status : failed: %s", err_str);
+            dict_del_sizen(local->dict, "op_err_str");
         }
+    }
 
-        gf_log ("cli", GF_LOG_DEBUG, "Returning %d", ret);
+    gf_log("cli", GF_LOG_DEBUG, RETURNING, ret);
 
-        GF_FREE (req.dict.dict_val);
+    GF_FREE(req.dict.dict_val);
 
-        if (global_state->mode & GLUSTER_MODE_XML) {
-                /* XML mode handles its own error */
-                ret = 0;
-        }
-        return ret;
+    if (global_state->mode & GLUSTER_MODE_XML) {
+        /* XML mode handles its own error */
+        ret = 0;
+    }
+    return ret;
 }
 
-int32_t
-gf_cli_barrier_volume_cbk (struct rpc_req *req, struct iovec *iov,
-                                  int count, void *myframe)
-{
-        gf_cli_rsp                      rsp = {0,};
-        int                             ret = -1;
-        dict_t                          *dict = NULL;
-
-        if (-1 == req->rpc_status)
-                goto out;
-        ret = xdr_to_generic (*iov, &rsp,
-                              (xdrproc_t)xdr_gf_cli_rsp);
-        if (ret < 0) {
-                gf_log (((call_frame_t *) myframe)->this->name, GF_LOG_ERROR,
-                        "Failed to decode xdr response");
-                goto out;
-        }
-        gf_log ("cli", GF_LOG_DEBUG, "Received response to barrier");
-
-        if (rsp.op_ret) {
-                if (rsp.op_errstr && (strlen (rsp.op_errstr) > 1)) {
-                        cli_err ("volume barrier: command unsuccessful : %s",
-                                 rsp.op_errstr);
-                } else {
-                        cli_err ("volume barrier: command unsuccessful");
-                }
+static int32_t
+gf_cli_barrier_volume_cbk(struct rpc_req *req, struct iovec *iov, int count,
+                          void *myframe)
+{
+    gf_cli_rsp rsp = {
+        0,
+    };
+    int ret = -1;
+
+    GF_ASSERT(myframe);
+
+    if (-1 == req->rpc_status)
+        goto out;
+
+    ret = xdr_to_generic(*iov, &rsp, (xdrproc_t)xdr_gf_cli_rsp);
+    if (ret < 0) {
+        gf_log(((call_frame_t *)myframe)->this->name, GF_LOG_ERROR,
+               XDR_DECODE_FAIL);
+        goto out;
+    }
+    gf_log("cli", GF_LOG_DEBUG, "Received response to barrier");
+
+    if (rsp.op_ret) {
+        if (rsp.op_errstr && (strlen(rsp.op_errstr) > 1)) {
+            cli_err("volume barrier: command unsuccessful : %s", rsp.op_errstr);
         } else {
-                cli_out ("volume barrier: command successful");
+            cli_err("volume barrier: command unsuccessful");
         }
-        ret = rsp.op_ret;
+    } else {
+        cli_out("volume barrier: command successful");
+    }
+    ret = rsp.op_ret;
 
 out:
-        if (dict)
-                dict_unref (dict);
-        free (rsp.op_errstr);
-        free (rsp.dict.dict_val);
-        cli_cmd_broadcast_response (ret);
-        return ret;
+    cli_cmd_broadcast_response(ret);
+    gf_free_xdr_cli_rsp(rsp);
+    return ret;
 }
 
-int
-gf_cli_barrier_volume (call_frame_t *frame, xlator_t *this, void *data)
+static int
+gf_cli_barrier_volume(call_frame_t *frame, xlator_t *this, void *data)
 {
-        gf_cli_req                      req = {{0,}};
-        dict_t                          *options = NULL;
-        int                             ret = -1;
+    gf_cli_req req = {{
+        0,
+    }};
+    dict_t *options = NULL;
+    int ret = -1;
 
-        if (!frame || !this || !data)
-                goto out;
+    options = data;
 
-        options = data;
+    ret = cli_to_glusterd(&req, frame, gf_cli_barrier_volume_cbk,
+                          (xdrproc_t)xdr_gf_cli_req, options,
+                          GLUSTER_CLI_BARRIER_VOLUME, this, cli_rpc_prog, NULL);
+    gf_log("cli", GF_LOG_DEBUG, RETURNING, ret);
 
-        ret = cli_to_glusterd (&req, frame, gf_cli_barrier_volume_cbk,
-                               (xdrproc_t) xdr_gf_cli_req, options,
-                               GLUSTER_CLI_BARRIER_VOLUME, this, cli_rpc_prog,
-                               NULL);
-out:
-        gf_log ("cli", GF_LOG_DEBUG, "Returning %d", ret);
-
-        GF_FREE (req.dict.dict_val);
-        return ret;
+    GF_FREE(req.dict.dict_val);
+    return ret;
 }
 
-int32_t
-gf_cli_get_vol_opt_cbk (struct rpc_req *req, struct iovec *iov, int count,
-                        void *myframe)
-{
-        gf_cli_rsp   rsp  = {0,};
-        int          ret  = -1;
-        dict_t      *dict = NULL;
-        char        *key  = NULL;
-        char        *value = NULL;
-        char         msg[1024] = {0,};
-        int          i     = 0;
-        char         dict_key[50] = {0,};
-
-        if (-1 == req->rpc_status)
-                goto out;
-        ret = xdr_to_generic (*iov, &rsp,
-                              (xdrproc_t)xdr_gf_cli_rsp);
-        if (ret < 0) {
-                gf_log (((call_frame_t *) myframe)->this->name, GF_LOG_ERROR,
-                        "Failed to decode xdr response");
-                goto out;
-        }
-        gf_log ("cli", GF_LOG_DEBUG, "Received response to get volume option");
-
-        if (rsp.op_ret) {
-                if (strcmp (rsp.op_errstr, ""))
-                        snprintf (msg, sizeof (msg), "volume get option: "
-                                  "failed: %s", rsp.op_errstr);
-                else
-                        snprintf (msg, sizeof (msg), "volume get option: "
-                                  "failed");
-
-                if (global_state->mode & GLUSTER_MODE_XML) {
-                        ret = cli_xml_output_str ("volGetopts", msg, rsp.op_ret,
-                                          rsp.op_errno, rsp.op_errstr);
-                        if (ret) {
-                                gf_log ("cli", GF_LOG_ERROR,
-                                        "Error outputting to xml");
-                        }
-                } else {
-                        cli_err ("%s", msg);
-                }
-                ret = rsp.op_ret;
-                goto out_nolog;
-        }
-        dict = dict_new ();
-
-        if (!dict) {
-                ret = -1;
-                goto out;
-        }
-
-        ret = dict_unserialize (rsp.dict.dict_val, rsp.dict.dict_len, &dict);
-        if (ret) {
-                gf_log ("cli", GF_LOG_ERROR,
-                        "Failed rsp_dict unserialization");
-                goto out;
-        }
+static int32_t
+gf_cli_get_vol_opt_cbk(struct rpc_req *req, struct iovec *iov, int count,
+                       void *myframe)
+{
+    gf_cli_rsp rsp = {
+        0,
+    };
+    int ret = -1;
+    dict_t *dict = NULL;
+    char *key = NULL;
+    char *value = NULL;
+    char msg[1024] = {
+        0,
+    };
+    int i = 0;
+    char dict_key[50] = {
+        0,
+    };
+
+    GF_ASSERT(myframe);
+
+    if (-1 == req->rpc_status)
+        goto out;
+
+    ret = xdr_to_generic(*iov, &rsp, (xdrproc_t)xdr_gf_cli_rsp);
+    if (ret < 0) {
+        gf_log(((call_frame_t *)myframe)->this->name, GF_LOG_ERROR,
+               XDR_DECODE_FAIL);
+        goto out;
+    }
+    gf_log("cli", GF_LOG_DEBUG, "Received response to get volume option");
+
+    if (rsp.op_ret) {
+        if (strcmp(rsp.op_errstr, ""))
+            snprintf(msg, sizeof(msg), "volume get option: failed: %s",
+                     rsp.op_errstr);
+        else
+            snprintf(msg, sizeof(msg), "volume get option: failed");
 
         if (global_state->mode & GLUSTER_MODE_XML) {
-                ret = cli_xml_output_vol_getopts (dict, rsp.op_ret,
-                                                  rsp.op_errno,
-                                                  rsp.op_errstr);
-                if (ret) {
-                        gf_log ("cli", GF_LOG_ERROR, "xml output generation "
-                                "failed");
-                        ret = 0;
-                }
-                goto out;
+            ret = cli_xml_output_str("volGetopts", msg, rsp.op_ret,
+                                     rsp.op_errno, rsp.op_errstr);
+            if (ret) {
+                gf_log("cli", GF_LOG_ERROR, XML_ERROR);
+            }
+        } else {
+            cli_err("%s", msg);
         }
-
-        ret = dict_get_int32 (dict, "count", &count);
+        ret = rsp.op_ret;
+        goto out_nolog;
+    }
+    dict = dict_new();
+
+    if (!dict) {
+        ret = -1;
+        goto out;
+    }
+
+    ret = dict_unserialize(rsp.dict.dict_val, rsp.dict.dict_len, &dict);
+    if (ret) {
+        gf_log("cli", GF_LOG_ERROR, DICT_UNSERIALIZE_FAIL);
+        goto out;
+    }
+
+    if (global_state->mode & GLUSTER_MODE_XML) {
+        ret = cli_xml_output_vol_getopts(dict, rsp.op_ret, rsp.op_errno,
+                                         rsp.op_errstr);
         if (ret) {
-                gf_log ("cli", GF_LOG_ERROR, "Failed to retrieve count "
-                        "from the dictionary");
-                goto out;
-        }
-        if (count <= 0) {
-                gf_log ("cli", GF_LOG_ERROR, "Value of count :%d is "
-                        "invalid", count);
-                ret = -1;
-                goto out;
+            gf_log("cli", GF_LOG_ERROR, XML_ERROR);
+            ret = 0;
+        }
+        goto out;
+    }
+
+    ret = dict_get_str_sizen(dict, "warning", &value);
+    if (!ret) {
+        cli_out("%s", value);
+    }
+
+    ret = dict_get_int32_sizen(dict, "count", &count);
+    if (ret) {
+        gf_log("cli", GF_LOG_ERROR,
+               "Failed to retrieve count from the dictionary");
+        goto out;
+    }
+
+    if (count <= 0) {
+        gf_log("cli", GF_LOG_ERROR, "Value of count :%d is invalid", count);
+        ret = -1;
+        goto out;
+    }
+
+    cli_out("%-40s%-40s", "Option", "Value");
+    cli_out("%-40s%-40s", "------", "-----");
+    for (i = 1; i <= count; i++) {
+        ret = snprintf(dict_key, sizeof dict_key, "key%d", i);
+        ret = dict_get_strn(dict, dict_key, ret, &key);
+        if (ret) {
+            gf_log("cli", GF_LOG_ERROR,
+                   "Failed to retrieve %s from the dictionary", dict_key);
+            goto out;
         }
-
-        cli_out ("%-40s%-40s", "Option", "Value");
-        cli_out ("%-40s%-40s", "------", "-----");
-        for (i=1; i<=count; i++) {
-                snprintf (dict_key, sizeof dict_key, "key%d", i);
-                ret = dict_get_str (dict, dict_key, &key);
-                if (ret) {
-                        gf_log ("cli", GF_LOG_ERROR, "Failed to"
-                                " retrieve %s from the "
-                                "dictionary", dict_key);
-                        goto out;
-                }
-                snprintf (dict_key, sizeof dict_key, "value%d", i);
-                ret = dict_get_str (dict, dict_key, &value);
-                if (ret) {
-                        gf_log ("cli", GF_LOG_ERROR, "Failed to "
-                                "retrieve key value for %s from"
-                                "the dictionary", dict_key);
-                        goto out;
-                }
-                cli_out ("%-40s%-40s", key, value);
+        ret = snprintf(dict_key, sizeof dict_key, "value%d", i);
+        ret = dict_get_strn(dict, dict_key, ret, &value);
+        if (ret) {
+            gf_log("cli", GF_LOG_ERROR,
+                   "Failed to retrieve key value for %s from the dictionary",
+                   dict_key);
+            goto out;
         }
+        cli_out("%-40s%-40s", key, value);
+    }
 
 out:
-        if (ret) {
-                cli_out ("volume get option failed. Check the cli/glusterd log "
-                         "file for more details");
-        }
+    if (ret) {
+        cli_out(
+            "volume get option failed. Check the cli/glusterd log "
+            "file for more details");
+    }
 
 out_nolog:
-        if (dict)
-                dict_unref (dict);
-        free (rsp.op_errstr);
-        free (rsp.dict.dict_val);
-        cli_cmd_broadcast_response (ret);
-        return ret;
+    if (dict)
+        dict_unref(dict);
+    cli_cmd_broadcast_response(ret);
+    gf_free_xdr_cli_rsp(rsp);
+    return ret;
 }
 
-int
-gf_cli_get_vol_opt (call_frame_t *frame, xlator_t *this, void *data)
+static int
+gf_cli_get_vol_opt(call_frame_t *frame, xlator_t *this, void *data)
 {
-        gf_cli_req  req     = {{0,}};
-        dict_t     *options = NULL;
-        int         ret     = -1;
+    gf_cli_req req = {{
+        0,
+    }};
+    dict_t *options = NULL;
+    int ret = -1;
 
-        if (!frame || !this || !data)
-                goto out;
-
-        options = data;
+    options = data;
 
-        ret = cli_to_glusterd (&req, frame, gf_cli_get_vol_opt_cbk,
-                               (xdrproc_t)xdr_gf_cli_req, options,
-                               GLUSTER_CLI_GET_VOL_OPT, this, cli_rpc_prog,
-                               NULL);
-out:
-        gf_log ("cli", GF_LOG_DEBUG, "Returning %d", ret);
+    ret = cli_to_glusterd(&req, frame, gf_cli_get_vol_opt_cbk,
+                          (xdrproc_t)xdr_gf_cli_req, options,
+                          GLUSTER_CLI_GET_VOL_OPT, this, cli_rpc_prog, NULL);
+    gf_log("cli", GF_LOG_DEBUG, RETURNING, ret);
 
-        GF_FREE (req.dict.dict_val);
-        return ret;
+    GF_FREE(req.dict.dict_val);
+    return ret;
 }
 
-int
-cli_to_glusterd (gf_cli_req *req, call_frame_t *frame,
-                 fop_cbk_fn_t cbkfn, xdrproc_t xdrproc, dict_t *dict,
-                 int procnum, xlator_t *this, rpc_clnt_prog_t *prog,
-                 struct iobref *iobref)
-{
-        int                ret = 0;
-        size_t             len = 0;
-        char               *cmd = NULL;
-        int                i = 0;
-        const char         **words = NULL;
-        cli_local_t        *local = NULL;
-
-        if (!this || !frame || !dict) {
-                ret = -1;
-                goto out;
-        }
+static int
+add_cli_cmd_timeout_to_dict(dict_t *dict)
+{
+    int ret = 0;
 
-        if (!frame->local) {
-                ret = -1;
-                goto out;
+    if (cli_default_conn_timeout > 120) {
+        ret = dict_set_uint32(dict, "timeout", cli_default_conn_timeout);
+        if (ret) {
+            gf_log("cli", GF_LOG_INFO, "Failed to save timeout to dict");
         }
+    }
+    return ret;
+}
 
-        local = frame->local;
+static int
+cli_to_glusterd(gf_cli_req *req, call_frame_t *frame, fop_cbk_fn_t cbkfn,
+                xdrproc_t xdrproc, dict_t *dict, int procnum, xlator_t *this,
+                rpc_clnt_prog_t *prog, struct iobref *iobref)
+{
+    int ret = 0;
+    size_t len = 0;
+    char *cmd = NULL;
+    int i = 0;
+    const char **words = NULL;
+    cli_local_t *local = NULL;
 
-        if (!local->words) {
-                ret = -1;
-                goto out;
-        }
+    if (!this || !frame || !frame->local || !dict) {
+        ret = -1;
+        goto out;
+    }
 
-        words = local->words;
+    local = frame->local;
 
-        while (words[i])
-                len += strlen (words[i++]) + 1;
+    if (!local->words) {
+        ret = -1;
+        goto out;
+    }
 
-        cmd = GF_CALLOC (1, len, gf_common_mt_char);
+    words = local->words;
 
-        if (!cmd) {
-                ret = -1;
-                goto out;
-        }
+    while (words[i])
+        len += strlen(words[i++]) + 1;
 
-        for (i = 0; words[i]; i++) {
-                strncat (cmd, words[i], strlen (words[i]));
-                if (words[i+1] != NULL)
-                        strncat (cmd, " ", strlen (" "));
-        }
+    cmd = GF_MALLOC(len + 1, gf_common_mt_char);
+    if (!cmd) {
+        ret = -1;
+        goto out;
+    }
+    cmd[0] = '\0';
 
-        cmd [len - 1] = '\0';
+    for (i = 0; words[i]; i++) {
+        strncat(cmd, words[i], len - 1);
+        if (words[i + 1] != NULL)
+            strncat(cmd, " ", len - 1);
+    }
 
-        ret = dict_set_dynstr (dict, "cmd-str", cmd);
-        if (ret)
-                goto out;
+    ret = dict_set_dynstr_sizen(dict, "cmd-str", cmd);
+    if (ret)
+        goto out;
 
-        ret = dict_allocate_and_serialize (dict, &(req->dict).dict_val,
-                                           &(req->dict).dict_len);
+    ret = add_cli_cmd_timeout_to_dict(dict);
 
-        if (ret < 0) {
-                gf_log (this->name, GF_LOG_DEBUG,
-                        "failed to get serialized length of dict");
-                goto out;
-        }
+    ret = dict_allocate_and_serialize(dict, &(req->dict).dict_val,
+                                      &(req->dict).dict_len);
 
-        ret = cli_cmd_submit (NULL, req, frame, prog, procnum, iobref, this,
-                              cbkfn, (xdrproc_t) xdrproc);
+    if (ret < 0) {
+        gf_log(this->name, GF_LOG_DEBUG,
+               "failed to get serialized length of dict");
+        goto out;
+    }
 
+    ret = cli_cmd_submit(NULL, req, frame, prog, procnum, iobref, this, cbkfn,
+                         (xdrproc_t)xdrproc);
 out:
-        return ret;
-
+    return ret;
 }
 
-int
-gf_cli_print_bitrot_scrub_status (dict_t *dict)
-{
-        int            i                = 1;
-        int            j                = 0;
-        int            ret              = -1;
-        int            count            = 0;
-        char           key[256]         = {0,};
-        char           *volname         = NULL;
-        char           *node_name       = NULL;
-        char           *scrub_freq      = NULL;
-        char           *state_scrub     = NULL;
-        char           *scrub_impact    = NULL;
-        char           *bad_file_str    = NULL;
-        char           *scrub_log_file  = NULL;
-        char           *bitrot_log_file = NULL;
-        uint64_t       scrub_files      = 0;
-        uint64_t       unsigned_files   = 0;
-        uint64_t       scrub_time       = 0;
-        uint64_t       days             = 0;
-        uint64_t       hour             = 0;
-        uint64_t       minut            = 0;
-        uint64_t       second           = 0;
-        char          *last_scrub       = NULL;
-        uint64_t       error_count      = 0;
-
-
-        ret = dict_get_str (dict, "volname", &volname);
-        if (ret)
-                gf_log ("cli", GF_LOG_TRACE, "failed to get volume name");
-
-        ret = dict_get_str (dict, "features.scrub", &state_scrub);
-        if (ret)
-                gf_log ("cli", GF_LOG_TRACE, "failed to get scrub state value");
-
-        ret = dict_get_str (dict, "features.scrub-throttle", &scrub_impact);
+static int
+gf_cli_print_bitrot_scrub_status(dict_t *dict)
+{
+    int i = 1;
+    int j = 0;
+    int ret = -1;
+    int count = 0;
+    char key[64] = {
+        0,
+    };
+    char *volname = NULL;
+    char *node_name = NULL;
+    char *scrub_freq = NULL;
+    char *state_scrub = NULL;
+    char *scrub_impact = NULL;
+    char *bad_file_str = NULL;
+    char *scrub_log_file = NULL;
+    char *bitrot_log_file = NULL;
+    uint64_t scrub_files = 0;
+    uint64_t unsigned_files = 0;
+    uint64_t scrub_time = 0;
+    uint64_t days = 0;
+    uint64_t hours = 0;
+    uint64_t minutes = 0;
+    uint64_t seconds = 0;
+    char *last_scrub = NULL;
+    uint64_t error_count = 0;
+    int8_t scrub_running = 0;
+    char *scrub_state_op = NULL;
+
+    ret = dict_get_int32_sizen(dict, "count", &count);
+    if (ret) {
+        gf_log("cli", GF_LOG_ERROR,
+               "failed to get count value from dictionary");
+        goto out;
+    }
+
+    ret = dict_get_str_sizen(dict, "volname", &volname);
+    if (ret)
+        gf_log("cli", GF_LOG_TRACE, "failed to get volume name");
+
+    ret = dict_get_str_sizen(dict, "features.scrub", &state_scrub);
+    if (ret)
+        gf_log("cli", GF_LOG_TRACE, "failed to get scrub state value");
+
+    ret = dict_get_str_sizen(dict, "features.scrub-throttle", &scrub_impact);
+    if (ret)
+        gf_log("cli", GF_LOG_TRACE, "failed to get scrub impact value");
+
+    ret = dict_get_str_sizen(dict, "features.scrub-freq", &scrub_freq);
+    if (ret)
+        gf_log("cli", GF_LOG_TRACE, "failed to get scrub -freq value");
+
+    ret = dict_get_str_sizen(dict, "bitrot_log_file", &bitrot_log_file);
+    if (ret)
+        gf_log("cli", GF_LOG_TRACE, "failed to get bitrot log file location");
+
+    ret = dict_get_str_sizen(dict, "scrub_log_file", &scrub_log_file);
+    if (ret)
+        gf_log("cli", GF_LOG_TRACE, "failed to get scrubber log file location");
+
+    for (i = 1; i <= count; i++) {
+        snprintf(key, sizeof(key), "scrub-running-%d", i);
+        ret = dict_get_int8(dict, key, &scrub_running);
         if (ret)
-                gf_log ("cli", GF_LOG_TRACE, "failed to get scrub impact "
-                        "value");
+            gf_log("cli", GF_LOG_TRACE, "failed to get scrubbed files");
+        if (scrub_running)
+            break;
+    }
 
-        ret = dict_get_str (dict, "features.scrub-freq", &scrub_freq);
-        if (ret)
-                gf_log ("cli", GF_LOG_TRACE, "failed to get scrub -freq value");
+    if (scrub_running)
+        gf_asprintf(&scrub_state_op, "%s (In Progress)", state_scrub);
+    else
+        gf_asprintf(&scrub_state_op, "%s (Idle)", state_scrub);
 
-        ret = dict_get_str (dict, "bitrot_log_file", &bitrot_log_file);
-        if (ret)
-                gf_log ("cli", GF_LOG_TRACE, "failed to get bitrot log file "
-                        "location");
+    cli_out("\n%s: %s\n", "Volume name ", volname);
 
-        ret = dict_get_str (dict, "scrub_log_file", &scrub_log_file);
-        if (ret)
-                gf_log ("cli", GF_LOG_TRACE, "failed to get scrubber log file "
-                        "location");
+    cli_out("%s: %s\n", "State of scrub", scrub_state_op);
 
-        ret = dict_get_int32 (dict, "count", &count);
-        if (ret) {
-                gf_log ("cli", GF_LOG_ERROR, "count not get count value from"
-                        " dictionary");
-                goto out;
-        }
-
-        cli_out ("\n%s: %s\n", "Volume name ", volname);
-
-        cli_out ("%s: %s\n", "State of scrub", state_scrub);
+    cli_out("%s: %s\n", "Scrub impact", scrub_impact);
 
-        cli_out ("%s: %s\n", "Scrub impact", scrub_impact);
+    cli_out("%s: %s\n", "Scrub frequency", scrub_freq);
 
-        cli_out ("%s: %s\n", "Scrub frequency", scrub_freq);
+    cli_out("%s: %s\n", "Bitrot error log location", bitrot_log_file);
 
-        cli_out ("%s: %s\n", "Bitrot error log location", bitrot_log_file);
+    cli_out("%s: %s\n", "Scrubber error log location", scrub_log_file);
 
-        cli_out ("%s: %s\n", "Scrubber error log location", scrub_log_file);
+    for (i = 1; i <= count; i++) {
+        /* Reset the variables to prevent carryover of values */
+        node_name = NULL;
+        last_scrub = NULL;
+        scrub_time = 0;
+        error_count = 0;
+        scrub_files = 0;
+        unsigned_files = 0;
 
+        snprintf(key, sizeof(key), "node-name-%d", i);
+        ret = dict_get_str(dict, key, &node_name);
+        if (ret)
+            gf_log("cli", GF_LOG_TRACE, "failed to get node-name");
 
-        for (i = 1; i <= count; i++) {
-                /* Reset the variables to prevent carryover of values */
-                node_name       = NULL;
-                last_scrub      = NULL;
-                scrub_time      = 0;
-                days            = 0;
-                hour            = 0;
-                minut           = 0;
-                second          = 0;
-                error_count     = 0;
-                scrub_files     = 0;
-                unsigned_files  = 0;
+        snprintf(key, sizeof(key), "scrubbed-files-%d", i);
+        ret = dict_get_uint64(dict, key, &scrub_files);
+        if (ret)
+            gf_log("cli", GF_LOG_TRACE, "failed to get scrubbed files");
 
-                memset (key, 0, 256);
-                snprintf (key, 256, "node-name-%d", i);
-                ret = dict_get_str (dict, key, &node_name);
-                if (ret)
-                        gf_log ("cli", GF_LOG_TRACE, "failed to get node-name");
+        snprintf(key, sizeof(key), "unsigned-files-%d", i);
+        ret = dict_get_uint64(dict, key, &unsigned_files);
+        if (ret)
+            gf_log("cli", GF_LOG_TRACE, "failed to get unsigned files");
 
-                memset (key, 0, 256);
-                snprintf (key, 256, "scrubbed-files-%d", i);
-                ret = dict_get_uint64 (dict, key, &scrub_files);
-                if (ret)
-                        gf_log ("cli", GF_LOG_TRACE, "failed to get scrubbed "
-                                "files");
+        snprintf(key, sizeof(key), "scrub-duration-%d", i);
+        ret = dict_get_uint64(dict, key, &scrub_time);
+        if (ret)
+            gf_log("cli", GF_LOG_TRACE, "failed to get last scrub duration");
 
-                memset (key, 0, 256);
-                snprintf (key, 256, "unsigned-files-%d", i);
-                ret = dict_get_uint64 (dict, key, &unsigned_files);
-                if (ret)
-                        gf_log ("cli", GF_LOG_TRACE, "failed to get unsigned "
-                                "files");
+        snprintf(key, sizeof(key), "last-scrub-time-%d", i);
+        ret = dict_get_str(dict, key, &last_scrub);
+        if (ret)
+            gf_log("cli", GF_LOG_TRACE, "failed to get last scrub time");
+        snprintf(key, sizeof(key), "error-count-%d", i);
+        ret = dict_get_uint64(dict, key, &error_count);
+        if (ret)
+            gf_log("cli", GF_LOG_TRACE, "failed to get error count");
 
-                memset (key, 0, 256);
-                snprintf (key, 256, "scrub-duration-%d", i);
-                ret = dict_get_uint64 (dict, key, &scrub_time);
-                if (ret)
-                        gf_log ("cli", GF_LOG_TRACE, "failed to get last scrub "
-                                "duration");
+        cli_out("\n%s\n",
+                "=========================================================");
 
-                memset (key, 0, 256);
-                snprintf (key, 256, "last-scrub-time-%d", i);
-                ret = dict_get_str (dict, key, &last_scrub);
-                if (ret)
-                        gf_log ("cli", GF_LOG_TRACE, "failed to get last scrub"
-                                " time");
+        cli_out("%s: %s\n", "Node", node_name);
 
-                memset (key, 0, 256);
-                snprintf (key, 256, "error-count-%d", i);
-                ret = dict_get_uint64 (dict, key, &error_count);
-                if (ret)
-                        gf_log ("cli", GF_LOG_TRACE, "failed to get error "
-                                "count");
+        cli_out("%s: %" PRIu64 "\n", "Number of Scrubbed files", scrub_files);
 
-                cli_out ("\n%s\n", "=========================================="
-                         "===============");
+        cli_out("%s: %" PRIu64 "\n", "Number of Skipped files", unsigned_files);
 
-                cli_out ("%s: %s\n", "Node name", node_name);
+        if ((!last_scrub) || !strcmp(last_scrub, ""))
+            cli_out("%s: %s\n", "Last completed scrub time",
+                    "Scrubber pending to complete.");
+        else
+            cli_out("%s: %s\n", "Last completed scrub time", last_scrub);
 
-                cli_out ("%s: %"PRIu64 "\n", "Number of Scrubbed files",
-                          scrub_files);
+        /* Printing last scrub duration time in human readable form*/
+        seconds = scrub_time % 60;
+        minutes = (scrub_time / 60) % 60;
+        hours = (scrub_time / 3600) % 24;
+        days = scrub_time / 86400;
+        cli_out("%s: %" PRIu64 ":%" PRIu64 ":%" PRIu64 ":%" PRIu64 "\n",
+                "Duration of last scrub (D:M:H:M:S)", days, hours, minutes,
+                seconds);
 
-                cli_out ("%s: %"PRIu64 "\n", "Number of Unsigned files",
-                          unsigned_files);
+        cli_out("%s: %" PRIu64 "\n", "Error count", error_count);
 
-                if ((!last_scrub) || !strcmp (last_scrub, ""))
-                        cli_out ("%s: %s\n", "Last completed scrub time",
-                                 "Scrubber pending to complete.");
-                else
-                        cli_out ("%s: %s\n", "Last completed scrub time",
-                                 last_scrub);
-
-                /* Printing last scrub duration time in human readable form*/
-                days       = scrub_time/86400;
-                hour       = (scrub_time%86400)/3600;
-                minut      = (scrub_time%86400%3600)/60;
-                second     = (scrub_time%86400%3600%60);
-                cli_out ("%s: %"PRIu64 ":%"PRIu64 ":%"PRIu64 ":%"PRIu64 "\n",
-                         "Duration of last scrub", days, hour, minut, second);
-
-                cli_out ("%s: %"PRIu64 "\n", "Error count", error_count);
-
-                if (error_count)
-                        cli_out ("%s:\n", "Corrupted object's");
-                /* Printing list of bad file's (Corrupted object's)*/
-                for (j = 0; j < error_count; j++) {
-                        memset (key, 0, 256);
-                        snprintf (key, 256, "quarantine-%d-%d", j, i);
-                        ret = dict_get_str (dict, key, &bad_file_str);
-                        if (!ret) {
-                                cli_out ("%s\n", bad_file_str);
-                        }
+        if (error_count) {
+            cli_out("%s:\n", "Corrupted object's [GFID]");
+            /* Printing list of bad file's (Corrupted object's)*/
+            for (j = 0; j < error_count; j++) {
+                snprintf(key, sizeof(key), "quarantine-%d-%d", j, i);
+                ret = dict_get_str(dict, key, &bad_file_str);
+                if (!ret) {
+                    cli_out("%s\n", bad_file_str);
                 }
+            }
         }
-        cli_out ("%s\n", "=========================================="
-                 "===============");
+    }
+    cli_out("%s\n",
+            "=========================================================");
 
 out:
-        return 0;
+    GF_FREE(scrub_state_op);
+    return 0;
 }
 
-int
-gf_cli_bitrot_cbk (struct rpc_req *req, struct iovec *iov,
-                   int count, void *myframe)
-{
-        int                  ret                       = -1;
-        int                  type                      = 0;
-        gf_cli_rsp           rsp                       = {0, };
-        dict_t               *dict                     = NULL;
-        call_frame_t         *frame                    = NULL;
-
-        if (req->rpc_status == -1) {
-                ret = -1;
-                goto out;
-        }
-
-        frame = myframe;
+static int
+gf_cli_bitrot_cbk(struct rpc_req *req, struct iovec *iov, int count,
+                  void *myframe)
+{
+    int ret = -1;
+    int type = 0;
+    gf_cli_rsp rsp = {
+        0,
+    };
+    dict_t *dict = NULL;
+    char *scrub_cmd = NULL;
+    char *volname = NULL;
+    char *cmd_str = NULL;
+    char *cmd_op = NULL;
+
+    GF_ASSERT(myframe);
+
+    if (req->rpc_status == -1) {
+        goto out;
+    }
+
+    ret = xdr_to_generic(*iov, &rsp, (xdrproc_t)xdr_gf_cli_rsp);
+    if (ret < 0) {
+        gf_log(((call_frame_t *)myframe)->this->name, GF_LOG_ERROR,
+               XDR_DECODE_FAIL);
+        goto out;
+    }
+
+    if (rsp.op_ret) {
+        ret = -1;
+        if (global_state->mode & GLUSTER_MODE_XML)
+            goto xml_output;
 
-        ret = xdr_to_generic (*iov, &rsp, (xdrproc_t)xdr_gf_cli_rsp);
-        if (ret < 0) {
-                gf_log (frame->this->name, GF_LOG_ERROR,
-                        "Failed to decode xdr response");
-                goto out;
-        }
+        if (strcmp(rsp.op_errstr, ""))
+            cli_err("Bitrot command failed : %s", rsp.op_errstr);
+        else
+            cli_err("Bitrot command : failed");
 
-        if (rsp.op_ret) {
-                ret = -1;
-                if (global_state->mode & GLUSTER_MODE_XML)
-                        goto xml_output;
+        goto out;
+    }
 
-                if (strcmp (rsp.op_errstr, ""))
-                        cli_err ("Bitrot command failed : %s", rsp.op_errstr);
-                else
-                        cli_err ("Bitrot command : failed");
+    if (rsp.dict.dict_len) {
+        /* Unserialize the dictionary */
+        dict = dict_new();
 
-                goto out;
+        if (!dict) {
+            ret = -1;
+            goto out;
         }
 
-        if (rsp.dict.dict_len) {
-                /* Unserialize the dictionary */
-                dict = dict_new ();
-
-                if (!dict) {
-                        ret = -1;
-                        goto out;
-                }
-
-                ret = dict_unserialize (rsp.dict.dict_val,
-                                        rsp.dict.dict_len,
-                                        &dict);
+        ret = dict_unserialize(rsp.dict.dict_val, rsp.dict.dict_len, &dict);
 
-                if (ret) {
-                        gf_log ("cli", GF_LOG_ERROR, "failed to unserialize "
-                                "req-buffer to dictionary");
-                        goto out;
-                }
+        if (ret) {
+            gf_log("cli", GF_LOG_ERROR, DICT_UNSERIALIZE_FAIL);
+            goto out;
         }
+    }
 
-        gf_log ("cli", GF_LOG_DEBUG, "Received resp to bit rot command");
+    gf_log("cli", GF_LOG_DEBUG, "Received resp to bit rot command");
 
-        ret = dict_get_int32 (dict, "type", &type);
-        if (ret) {
-                gf_log ("cli", GF_LOG_ERROR, "Failed to get command type");
-                goto out;
-        }
+    ret = dict_get_int32_sizen(dict, "type", &type);
+    if (ret) {
+        gf_log("cli", GF_LOG_ERROR, "Failed to get command type");
+        goto out;
+    }
 
-        if ((type == GF_BITROT_CMD_SCRUB_STATUS) &&
-             !(global_state->mode & GLUSTER_MODE_XML)) {
-                ret = gf_cli_print_bitrot_scrub_status (dict);
-                if (ret) {
-                        gf_log ("cli", GF_LOG_ERROR, "Failed to print bitrot "
-                                "scrub status");
-                }
-                goto out;
-        }
+    if ((type == GF_BITROT_CMD_SCRUB_STATUS) &&
+        !(global_state->mode & GLUSTER_MODE_XML)) {
+        ret = gf_cli_print_bitrot_scrub_status(dict);
+        if (ret) {
+            gf_log("cli", GF_LOG_ERROR, "Failed to print bitrot scrub status");
+        }
+        goto out;
+    }
+
+    /* Ignoring the error, as using dict val for cli output only */
+    ret = dict_get_str_sizen(dict, "volname", &volname);
+    if (ret)
+        gf_log("cli", GF_LOG_TRACE, "failed to get volume name");
+
+    ret = dict_get_str_sizen(dict, "scrub-value", &scrub_cmd);
+    if (ret)
+        gf_log("cli", GF_LOG_TRACE, "Failed to get scrub command");
+
+    ret = dict_get_str_sizen(dict, "cmd-str", &cmd_str);
+    if (ret)
+        gf_log("cli", GF_LOG_TRACE, "failed to get command string");
+
+    if (cmd_str)
+        cmd_op = strrchr(cmd_str, ' ') + 1;
+
+    switch (type) {
+        case GF_BITROT_OPTION_TYPE_ENABLE:
+            cli_out("volume bitrot: success bitrot enabled for volume %s",
+                    volname);
+            ret = 0;
+            goto out;
+        case GF_BITROT_OPTION_TYPE_DISABLE:
+            cli_out("volume bitrot: success bitrot disabled for volume %s",
+                    volname);
+            ret = 0;
+            goto out;
+        case GF_BITROT_CMD_SCRUB_ONDEMAND:
+            cli_out("volume bitrot: scrubber started ondemand for volume %s",
+                    volname);
+            ret = 0;
+            goto out;
+        case GF_BITROT_OPTION_TYPE_SCRUB:
+            if (!strncmp("pause", scrub_cmd, sizeof("pause")))
+                cli_out("volume bitrot: scrubber paused for volume %s",
+                        volname);
+            if (!strncmp("resume", scrub_cmd, sizeof("resume")))
+                cli_out("volume bitrot: scrubber resumed for volume %s",
+                        volname);
+            ret = 0;
+            goto out;
+        case GF_BITROT_OPTION_TYPE_SCRUB_FREQ:
+            cli_out(
+                "volume bitrot: scrub-frequency is set to %s "
+                "successfully for volume %s",
+                cmd_op, volname);
+            ret = 0;
+            goto out;
+        case GF_BITROT_OPTION_TYPE_SCRUB_THROTTLE:
+            cli_out(
+                "volume bitrot: scrub-throttle is set to %s "
+                "successfully for volume %s",
+                cmd_op, volname);
+            ret = 0;
+            goto out;
+    }
 
 xml_output:
-        if (global_state->mode & GLUSTER_MODE_XML) {
-                ret = cli_xml_output_vol_profile (dict, rsp.op_ret,
-                                                  rsp.op_errno,
-                                                  rsp.op_errstr);
-                if (ret)
-                        gf_log ("cli", GF_LOG_ERROR,
-                                "Error outputting to xml");
-                goto out;
-        }
+    if (global_state->mode & GLUSTER_MODE_XML) {
+        ret = cli_xml_output_vol_profile(dict, rsp.op_ret, rsp.op_errno,
+                                         rsp.op_errstr);
+        if (ret)
+            gf_log("cli", GF_LOG_ERROR, XML_ERROR);
+        goto out;
+    }
 
-        if (!rsp.op_ret)
-                cli_out ("volume bitrot: success");
+    if (!rsp.op_ret)
+        cli_out("volume bitrot: success");
 
-        ret = rsp.op_ret;
+    ret = rsp.op_ret;
 
 out:
-        if (dict)
-            dict_unref (dict);
-
-        free (rsp.dict.dict_val);
-        free (rsp.op_errstr);
-
-        cli_cmd_broadcast_response (ret);
+    if (dict)
+        dict_unref(dict);
 
-        return ret;
+    gf_free_xdr_cli_rsp(rsp);
+    cli_cmd_broadcast_response(ret);
+    return ret;
 }
 
-int32_t
-gf_cli_bitrot (call_frame_t *frame, xlator_t *this, void *data)
-{
-        gf_cli_req        req           = { {0,} };
-        dict_t           *options       = NULL;
-        int               ret           = -1;
-
-        if (!frame || !this || !data)
-                goto out;
-
-        options = data;
-
-        ret = cli_to_glusterd (&req, frame, gf_cli_bitrot_cbk,
-                               (xdrproc_t) xdr_gf_cli_req, options,
-                               GLUSTER_CLI_BITROT, this, cli_rpc_prog,
-                               NULL);
-        if (ret) {
-                gf_log ("cli", GF_LOG_ERROR, "cli_to_glusterd for "
-                        "bitrot failed");
-                goto out;
-        }
-
-out:
-        gf_log ("cli", GF_LOG_DEBUG, "Returning %d", ret);
-
-        GF_FREE (req.dict.dict_val);
-
-        return ret;
-}
-
-struct rpc_clnt_procedure gluster_cli_actors[GLUSTER_CLI_MAXVALUE] = {
-        [GLUSTER_CLI_NULL]             = {"NULL", NULL },
-        [GLUSTER_CLI_PROBE]            = {"PROBE_QUERY", gf_cli_probe},
-        [GLUSTER_CLI_DEPROBE]          = {"DEPROBE_QUERY", gf_cli_deprobe},
-        [GLUSTER_CLI_LIST_FRIENDS]     = {"LIST_FRIENDS", gf_cli_list_friends},
-        [GLUSTER_CLI_UUID_RESET]       = {"UUID_RESET", gf_cli3_1_uuid_reset},
-        [GLUSTER_CLI_UUID_GET]       = {"UUID_GET", gf_cli3_1_uuid_get},
-        [GLUSTER_CLI_CREATE_VOLUME]    = {"CREATE_VOLUME", gf_cli_create_volume},
-        [GLUSTER_CLI_DELETE_VOLUME]    = {"DELETE_VOLUME", gf_cli_delete_volume},
-        [GLUSTER_CLI_START_VOLUME]     = {"START_VOLUME", gf_cli_start_volume},
-        [GLUSTER_CLI_STOP_VOLUME]      = {"STOP_VOLUME", gf_cli_stop_volume},
-        [GLUSTER_CLI_RENAME_VOLUME]    = {"RENAME_VOLUME", gf_cli_rename_volume},
-        [GLUSTER_CLI_DEFRAG_VOLUME]    = {"DEFRAG_VOLUME", gf_cli_defrag_volume},
-        [GLUSTER_CLI_GET_VOLUME]       = {"GET_VOLUME", gf_cli_get_volume},
-        [GLUSTER_CLI_GET_NEXT_VOLUME]  = {"GET_NEXT_VOLUME", gf_cli_get_next_volume},
-        [GLUSTER_CLI_SET_VOLUME]       = {"SET_VOLUME", gf_cli_set_volume},
-        [GLUSTER_CLI_ADD_BRICK]        = {"ADD_BRICK", gf_cli_add_brick},
-        [GLUSTER_CLI_REMOVE_BRICK]     = {"REMOVE_BRICK", gf_cli_remove_brick},
-        [GLUSTER_CLI_REPLACE_BRICK]    = {"REPLACE_BRICK", gf_cli_replace_brick},
-        [GLUSTER_CLI_LOG_ROTATE]       = {"LOG ROTATE", gf_cli_log_rotate},
-        [GLUSTER_CLI_GETSPEC]          = {"GETSPEC", gf_cli_getspec},
-        [GLUSTER_CLI_PMAP_PORTBYBRICK] = {"PMAP PORTBYBRICK", gf_cli_pmap_b2p},
-        [GLUSTER_CLI_SYNC_VOLUME]      = {"SYNC_VOLUME", gf_cli_sync_volume},
-        [GLUSTER_CLI_RESET_VOLUME]     = {"RESET_VOLUME", gf_cli_reset_volume},
-        [GLUSTER_CLI_FSM_LOG]          = {"FSM_LOG", gf_cli_fsm_log},
-        [GLUSTER_CLI_GSYNC_SET]        = {"GSYNC_SET", gf_cli_gsync_set},
-        [GLUSTER_CLI_PROFILE_VOLUME]   = {"PROFILE_VOLUME", gf_cli_profile_volume},
-        [GLUSTER_CLI_QUOTA]            = {"QUOTA", gf_cli_quota},
-        [GLUSTER_CLI_TOP_VOLUME]       = {"TOP_VOLUME", gf_cli_top_volume},
-        [GLUSTER_CLI_GETWD]            = {"GETWD", gf_cli_getwd},
-        [GLUSTER_CLI_STATUS_VOLUME]    = {"STATUS_VOLUME", gf_cli_status_volume},
-        [GLUSTER_CLI_STATUS_ALL]       = {"STATUS_ALL", gf_cli_status_volume_all},
-        [GLUSTER_CLI_MOUNT]            = {"MOUNT", gf_cli_mount},
-        [GLUSTER_CLI_UMOUNT]           = {"UMOUNT", gf_cli_umount},
-        [GLUSTER_CLI_HEAL_VOLUME]      = {"HEAL_VOLUME", gf_cli_heal_volume},
-        [GLUSTER_CLI_STATEDUMP_VOLUME] = {"STATEDUMP_VOLUME", gf_cli_statedump_volume},
-        [GLUSTER_CLI_LIST_VOLUME]      = {"LIST_VOLUME", gf_cli_list_volume},
-        [GLUSTER_CLI_CLRLOCKS_VOLUME]  = {"CLEARLOCKS_VOLUME", gf_cli_clearlocks_volume},
-        [GLUSTER_CLI_COPY_FILE]        = {"COPY_FILE", gf_cli_copy_file},
-        [GLUSTER_CLI_SYS_EXEC]         = {"SYS_EXEC", gf_cli_sys_exec},
-        [GLUSTER_CLI_SNAP]             = {"SNAP", gf_cli_snapshot},
-        [GLUSTER_CLI_BARRIER_VOLUME]   = {"BARRIER VOLUME", gf_cli_barrier_volume},
-        [GLUSTER_CLI_GANESHA]          = {"GANESHA", gf_cli_ganesha},
-        [GLUSTER_CLI_GET_VOL_OPT]      = {"GET_VOL_OPT", gf_cli_get_vol_opt},
-        [GLUSTER_CLI_BITROT]           = {"BITROT", gf_cli_bitrot},
-        [GLUSTER_CLI_ATTACH_TIER]      = {"ATTACH_TIER", gf_cli_attach_tier},
-        [GLUSTER_CLI_DETACH_TIER]      = {"DETACH_TIER", gf_cli_detach_tier},
-        [GLUSTER_CLI_TIER]             = {"TIER", gf_cli_tier}
+static int32_t
+gf_cli_bitrot(call_frame_t *frame, xlator_t *this, void *data)
+{
+    gf_cli_req req = {{
+        0,
+    }};
+    dict_t *options = NULL;
+    int ret = -1;
+
+    options = data;
+
+    ret = cli_to_glusterd(&req, frame, gf_cli_bitrot_cbk,
+                          (xdrproc_t)xdr_gf_cli_req, options,
+                          GLUSTER_CLI_BITROT, this, cli_rpc_prog, NULL);
+    if (ret) {
+        gf_log("cli", GF_LOG_ERROR, "cli_to_glusterd for bitrot failed");
+        goto out;
+    }
+
+out:
+    gf_log("cli", GF_LOG_DEBUG, RETURNING, ret);
+
+    GF_FREE(req.dict.dict_val);
+
+    return ret;
+}
+
+static struct rpc_clnt_procedure gluster_cli_actors[GLUSTER_CLI_MAXVALUE] = {
+    [GLUSTER_CLI_NULL] = {"NULL", NULL},
+    [GLUSTER_CLI_PROBE] = {"PROBE_QUERY", gf_cli_probe},
+    [GLUSTER_CLI_DEPROBE] = {"DEPROBE_QUERY", gf_cli_deprobe},
+    [GLUSTER_CLI_LIST_FRIENDS] = {"LIST_FRIENDS", gf_cli_list_friends},
+    [GLUSTER_CLI_UUID_RESET] = {"UUID_RESET", gf_cli3_1_uuid_reset},
+    [GLUSTER_CLI_UUID_GET] = {"UUID_GET", gf_cli3_1_uuid_get},
+    [GLUSTER_CLI_CREATE_VOLUME] = {"CREATE_VOLUME", gf_cli_create_volume},
+    [GLUSTER_CLI_DELETE_VOLUME] = {"DELETE_VOLUME", gf_cli_delete_volume},
+    [GLUSTER_CLI_START_VOLUME] = {"START_VOLUME", gf_cli_start_volume},
+    [GLUSTER_CLI_STOP_VOLUME] = {"STOP_VOLUME", gf_cli_stop_volume},
+    [GLUSTER_CLI_RENAME_VOLUME] = {"RENAME_VOLUME", gf_cli_rename_volume},
+    [GLUSTER_CLI_DEFRAG_VOLUME] = {"DEFRAG_VOLUME", gf_cli_defrag_volume},
+    [GLUSTER_CLI_GET_VOLUME] = {"GET_VOLUME", gf_cli_get_volume},
+    [GLUSTER_CLI_GET_NEXT_VOLUME] = {"GET_NEXT_VOLUME", gf_cli_get_next_volume},
+    [GLUSTER_CLI_SET_VOLUME] = {"SET_VOLUME", gf_cli_set_volume},
+    [GLUSTER_CLI_ADD_BRICK] = {"ADD_BRICK", gf_cli_add_brick},
+    [GLUSTER_CLI_REMOVE_BRICK] = {"REMOVE_BRICK", gf_cli_remove_brick},
+    [GLUSTER_CLI_REPLACE_BRICK] = {"REPLACE_BRICK", gf_cli_replace_brick},
+    [GLUSTER_CLI_LOG_ROTATE] = {"LOG ROTATE", gf_cli_log_rotate},
+    [GLUSTER_CLI_GETSPEC] = {"GETSPEC", gf_cli_getspec},
+    [GLUSTER_CLI_PMAP_PORTBYBRICK] = {"PMAP PORTBYBRICK", gf_cli_pmap_b2p},
+    [GLUSTER_CLI_SYNC_VOLUME] = {"SYNC_VOLUME", gf_cli_sync_volume},
+    [GLUSTER_CLI_RESET_VOLUME] = {"RESET_VOLUME", gf_cli_reset_volume},
+    [GLUSTER_CLI_FSM_LOG] = {"FSM_LOG", gf_cli_fsm_log},
+    [GLUSTER_CLI_GSYNC_SET] = {"GSYNC_SET", gf_cli_gsync_set},
+    [GLUSTER_CLI_PROFILE_VOLUME] = {"PROFILE_VOLUME", gf_cli_profile_volume},
+    [GLUSTER_CLI_QUOTA] = {"QUOTA", gf_cli_quota},
+    [GLUSTER_CLI_TOP_VOLUME] = {"TOP_VOLUME", gf_cli_top_volume},
+    [GLUSTER_CLI_GETWD] = {"GETWD", gf_cli_getwd},
+    [GLUSTER_CLI_STATUS_VOLUME] = {"STATUS_VOLUME", gf_cli_status_volume},
+    [GLUSTER_CLI_STATUS_ALL] = {"STATUS_ALL", gf_cli_status_volume_all},
+    [GLUSTER_CLI_MOUNT] = {"MOUNT", gf_cli_mount},
+    [GLUSTER_CLI_UMOUNT] = {"UMOUNT", gf_cli_umount},
+    [GLUSTER_CLI_HEAL_VOLUME] = {"HEAL_VOLUME", gf_cli_heal_volume},
+    [GLUSTER_CLI_STATEDUMP_VOLUME] = {"STATEDUMP_VOLUME",
+                                      gf_cli_statedump_volume},
+    [GLUSTER_CLI_LIST_VOLUME] = {"LIST_VOLUME", gf_cli_list_volume},
+    [GLUSTER_CLI_CLRLOCKS_VOLUME] = {"CLEARLOCKS_VOLUME",
+                                     gf_cli_clearlocks_volume},
+    [GLUSTER_CLI_COPY_FILE] = {"COPY_FILE", gf_cli_copy_file},
+    [GLUSTER_CLI_SYS_EXEC] = {"SYS_EXEC", gf_cli_sys_exec},
+    [GLUSTER_CLI_SNAP] = {"SNAP", gf_cli_snapshot},
+    [GLUSTER_CLI_BARRIER_VOLUME] = {"BARRIER VOLUME", gf_cli_barrier_volume},
+    [GLUSTER_CLI_GET_VOL_OPT] = {"GET_VOL_OPT", gf_cli_get_vol_opt},
+    [GLUSTER_CLI_BITROT] = {"BITROT", gf_cli_bitrot},
+    [GLUSTER_CLI_GET_STATE] = {"GET_STATE", gf_cli_get_state},
+    [GLUSTER_CLI_RESET_BRICK] = {"RESET_BRICK", gf_cli_reset_brick},
+    [GLUSTER_CLI_GANESHA] = {"GANESHA", gf_cli_ganesha},
 };
 
 struct rpc_clnt_program cli_prog = {
-        .progname  = "Gluster CLI",
-        .prognum   = GLUSTER_CLI_PROGRAM,
-        .progver   = GLUSTER_CLI_VERSION,
-        .numproc   = GLUSTER_CLI_MAXVALUE,
-        .proctable = gluster_cli_actors,
+    .progname = "Gluster CLI",
+    .prognum = GLUSTER_CLI_PROGRAM,
+    .progver = GLUSTER_CLI_VERSION,
+    .numproc = GLUSTER_CLI_MAXVALUE,
+    .proctable = gluster_cli_actors,
 };
 
-struct rpc_clnt_procedure cli_quotad_procs[GF_AGGREGATOR_MAXVALUE] = {
-        [GF_AGGREGATOR_NULL]     = {"NULL", NULL},
-        [GF_AGGREGATOR_LOOKUP]   = {"LOOKUP", NULL},
-        [GF_AGGREGATOR_GETLIMIT]   = {"GETLIMIT", cli_quotad_getlimit},
+static struct rpc_clnt_procedure cli_quotad_procs[GF_AGGREGATOR_MAXVALUE] = {
+    [GF_AGGREGATOR_NULL] = {"NULL", NULL},
+    [GF_AGGREGATOR_LOOKUP] = {"LOOKUP", NULL},
+    [GF_AGGREGATOR_GETLIMIT] = {"GETLIMIT", cli_quotad_getlimit},
 };
 
 struct rpc_clnt_program cli_quotad_clnt = {
-        .progname  = "CLI Quotad client",
-        .prognum   = GLUSTER_AGGREGATOR_PROGRAM,
-        .progver   = GLUSTER_AGGREGATOR_VERSION,
-        .numproc   = GF_AGGREGATOR_MAXVALUE,
-        .proctable = cli_quotad_procs,
+    .progname = "CLI Quotad client",
+    .prognum = GLUSTER_AGGREGATOR_PROGRAM,
+    .progver = GLUSTER_AGGREGATOR_VERSION,
+    .numproc = GF_AGGREGATOR_MAXVALUE,
+    .proctable = cli_quotad_procs,
 };
diff --git a/cli/src/cli-xml-output.c b/cli/src/cli-xml-output.c
index 7140f803071..069de75801c 100644
--- a/cli/src/cli-xml-output.c
+++ b/cli/src/cli-xml-output.c
@@ -10,15 +10,12 @@
 #include <stdlib.h>
 #include "cli.h"
 #include "cli1-xdr.h"
-#include "run.h"
-#include "compat.h"
-#include "syscall.h"
+#include <glusterfs/run.h>
+#include <glusterfs/compat.h>
+#include <glusterfs/syscall.h>
+#include <glusterfs/upcall-utils.h>
 
-
-enum gf_task_types {
-    GF_TASK_TYPE_REBALANCE,
-    GF_TASK_TYPE_REMOVE_BRICK
-};
+enum gf_task_types { GF_TASK_TYPE_REBALANCE, GF_TASK_TYPE_REMOVE_BRICK };
 
 /*
  * IMPORTANT NOTE:
@@ -42,4384 +39,3796 @@ enum gf_task_types {
  *      }
  *      #endif
  *
- *  Following the above formate ensures that all xml related code is compliled
+ *  Following the above format ensures that all xml related code is compiled
  *  only when libxml2 is present, and also keeps the rest of the codebase free
  *  of #if (HAVE_LIB_XML)
  */
 
-
 #if (HAVE_LIB_XML)
 
 #include <libxml/encoding.h>
 #include <libxml/xmlwriter.h>
 
-#define XML_RET_CHECK_AND_GOTO(ret, label)      do {            \
-                if (ret < 0) {                                  \
-                        ret = -1;                               \
-                        goto label;                             \
-                }                                               \
-                else                                            \
-                        ret = 0;                                \
-        }while (0)                                              \
+#define XML_RET_CHECK_AND_GOTO(ret, label)                                     \
+    do {                                                                       \
+        if (ret < 0) {                                                         \
+            ret = -1;                                                          \
+            goto label;                                                        \
+        } else                                                                 \
+            ret = 0;                                                           \
+    } while (0)
 
 int
-cli_begin_xml_output (xmlTextWriterPtr *writer, xmlDocPtr *doc)
+cli_begin_xml_output(xmlTextWriterPtr *writer, xmlDocPtr *doc)
 {
-        int             ret = -1;
+    int ret = -1;
 
-        *writer = xmlNewTextWriterDoc (doc, 0);
-        if (writer == NULL) {
-                ret = -1;
-                goto out;
-        }
+    *writer = xmlNewTextWriterDoc(doc, 0);
+    if (*writer == NULL) {
+        ret = -1;
+        goto out;
+    }
 
-        ret = xmlTextWriterStartDocument (*writer, "1.0", "UTF-8", "yes");
-        XML_RET_CHECK_AND_GOTO (ret, out);
+    ret = xmlTextWriterStartDocument(*writer, "1.0", "UTF-8", "yes");
+    XML_RET_CHECK_AND_GOTO(ret, out);
 
-        /* <cliOutput> */
-        ret = xmlTextWriterStartElement (*writer, (xmlChar *)"cliOutput");
-        XML_RET_CHECK_AND_GOTO (ret, out);
+    /* <cliOutput> */
+    ret = xmlTextWriterStartElement(*writer, (xmlChar *)"cliOutput");
+    XML_RET_CHECK_AND_GOTO(ret, out);
 
 out:
-        gf_log ("cli", GF_LOG_DEBUG, "Returning %d", ret);
-        return ret;
+    gf_log("cli", GF_LOG_DEBUG, "Returning %d", ret);
+    return ret;
 }
 
 int
-cli_end_xml_output (xmlTextWriterPtr writer, xmlDocPtr doc)
+cli_end_xml_output(xmlTextWriterPtr writer, xmlDocPtr doc)
 {
-        int             ret = -1;
+    int ret = -1;
 
-        /* </cliOutput> */
-        ret = xmlTextWriterEndElement (writer);
-        XML_RET_CHECK_AND_GOTO (ret, out);
+    /* </cliOutput> */
+    ret = xmlTextWriterEndElement(writer);
+    XML_RET_CHECK_AND_GOTO(ret, out);
 
-        ret = xmlTextWriterEndDocument (writer);
-        XML_RET_CHECK_AND_GOTO (ret, out);
+    ret = xmlTextWriterEndDocument(writer);
+    XML_RET_CHECK_AND_GOTO(ret, out);
 
+    /* Dump xml document to stdout and pretty format it */
+    xmlSaveFormatFileEnc("-", doc, "UTF-8", 1);
 
-        /* Dump xml document to stdout and pretty format it */
-        xmlSaveFormatFileEnc ("-", doc, "UTF-8", 1);
-
-        xmlFreeTextWriter (writer);
-        xmlFreeDoc (doc);
+    xmlFreeTextWriter(writer);
+    xmlFreeDoc(doc);
 
 out:
-        gf_log ("cli", GF_LOG_DEBUG, "Returning %d", ret);
-        return ret;
+    gf_log("cli", GF_LOG_DEBUG, "Returning %d", ret);
+    return ret;
 }
 
 int
-cli_xml_output_common (xmlTextWriterPtr writer, int op_ret, int op_errno,
-                       char *op_errstr)
+cli_xml_output_common(xmlTextWriterPtr writer, int op_ret, int op_errno,
+                      char *op_errstr)
 {
-        int             ret = -1;
+    int ret = -1;
 
-        ret = xmlTextWriterWriteFormatElement (writer, (xmlChar *)"opRet",
-                                               "%d", op_ret);
-        XML_RET_CHECK_AND_GOTO (ret, out);
+    ret = xmlTextWriterWriteFormatElement(writer, (xmlChar *)"opRet", "%d",
+                                          op_ret);
+    XML_RET_CHECK_AND_GOTO(ret, out);
 
-        ret = xmlTextWriterWriteFormatElement (writer, (xmlChar *)"opErrno",
-                                               "%d", op_errno);
-        XML_RET_CHECK_AND_GOTO (ret, out);
+    ret = xmlTextWriterWriteFormatElement(writer, (xmlChar *)"opErrno", "%d",
+                                          op_errno);
+    XML_RET_CHECK_AND_GOTO(ret, out);
 
-        if (op_errstr)
-                ret = xmlTextWriterWriteFormatElement (writer,
-                                                       (xmlChar *)"opErrstr",
-                                                       "%s", op_errstr);
-        else
-                ret = xmlTextWriterWriteFormatElement (writer,
-                                                       (xmlChar *)"opErrstr",
-                                                       "%s", "");
+    if (op_errstr)
+        ret = xmlTextWriterWriteFormatElement(writer, (xmlChar *)"opErrstr",
+                                              "%s", op_errstr);
+    else
+        ret = xmlTextWriterWriteFormatElement(writer, (xmlChar *)"opErrstr",
+                                              "%s", "");
 
-        XML_RET_CHECK_AND_GOTO (ret, out);
+    XML_RET_CHECK_AND_GOTO(ret, out);
 
 out:
-        gf_log ("cli", GF_LOG_DEBUG, "Returning %d", ret);
-        return ret;
+    gf_log("cli", GF_LOG_DEBUG, "Returning %d", ret);
+    return ret;
 }
 #endif
 
 int
-cli_xml_output_str (char *op, char *str, int op_ret, int op_errno,
-                    char *op_errstr)
+cli_xml_output_str(char *op, char *str, int op_ret, int op_errno,
+                   char *op_errstr)
 {
 #if (HAVE_LIB_XML)
-        int                     ret = -1;
-        xmlTextWriterPtr        writer = NULL;
-        xmlDocPtr               doc = NULL;
+    int ret = -1;
+    xmlTextWriterPtr writer = NULL;
+    xmlDocPtr doc = NULL;
 
-        ret = cli_begin_xml_output (&writer, &doc);
-        if (ret)
-                goto out;
+    ret = cli_begin_xml_output(&writer, &doc);
+    if (ret)
+        goto out;
 
-        ret = cli_xml_output_common (writer, op_ret, op_errno, op_errstr);
-        if (ret)
-                goto out;
+    ret = cli_xml_output_common(writer, op_ret, op_errno, op_errstr);
+    if (ret)
+        goto out;
 
-        if (op) {
-                ret = xmlTextWriterWriteFormatElement (writer,
-                                                       (xmlChar *)"cliOp",
-                                                       "%s", op);
-                XML_RET_CHECK_AND_GOTO (ret, out);
-        }
+    if (op) {
+        ret = xmlTextWriterWriteFormatElement(writer, (xmlChar *)"cliOp", "%s",
+                                              op);
+        XML_RET_CHECK_AND_GOTO(ret, out);
+    }
 
-        if (str) {
-                ret = xmlTextWriterWriteFormatElement (writer,
-                                                       (xmlChar *)"output",
-                                                       "%s", str);
-                XML_RET_CHECK_AND_GOTO (ret, out);
-        }
+    if (str) {
+        ret = xmlTextWriterWriteFormatElement(writer, (xmlChar *)"output", "%s",
+                                              str);
+        XML_RET_CHECK_AND_GOTO(ret, out);
+    }
 
-        ret = cli_end_xml_output (writer, doc);
+    ret = cli_end_xml_output(writer, doc);
 
 out:
-        gf_log ("cli", GF_LOG_DEBUG, "Returning %d", ret);
-        return ret;
+    gf_log("cli", GF_LOG_DEBUG, "Returning %d", ret);
+    return ret;
 #else
-        return 0;
+    return 0;
 #endif
 }
 
 #if (HAVE_LIB_XML)
 int
-cli_xml_output_data_pair (dict_t *this, char *key, data_t *value,
-                          void *data)
+cli_xml_output_data_pair(dict_t *this, char *key, data_t *value, void *data)
 {
-        int                     ret = -1;
-        xmlTextWriterPtr        *writer = NULL;
+    int ret = -1;
+    xmlTextWriterPtr *writer = NULL;
 
-        writer = (xmlTextWriterPtr *)data;
+    writer = (xmlTextWriterPtr *)data;
 
-        ret = xmlTextWriterWriteFormatElement (*writer, (xmlChar *)key,
-                                               "%s", value->data);
+    ret = xmlTextWriterWriteFormatElement(*writer, (xmlChar *)key, "%s",
+                                          value->data);
 
-        return ret;
+    XML_RET_CHECK_AND_GOTO(ret, out);
+out:
+    return ret;
 }
 #endif
 
 int
-cli_xml_output_dict ( char *op, dict_t *dict, int op_ret, int op_errno,
-                      char *op_errstr)
+cli_xml_output_dict(char *op, dict_t *dict, int op_ret, int op_errno,
+                    char *op_errstr)
 {
 #if (HAVE_LIB_XML)
-        int                     ret = -1;
-        xmlTextWriterPtr        writer = NULL;
-        xmlDocPtr               doc = NULL;
+    int ret = -1;
+    xmlTextWriterPtr writer = NULL;
+    xmlDocPtr doc = NULL;
 
-        ret = cli_begin_xml_output (&writer, &doc);
-        if (ret)
-                goto out;
+    ret = cli_begin_xml_output(&writer, &doc);
+    if (ret)
+        goto out;
 
-        ret = cli_xml_output_common (writer, op_ret, op_errno, op_errstr);
-        if (ret)
-                goto out;
+    ret = cli_xml_output_common(writer, op_ret, op_errno, op_errstr);
+    if (ret)
+        goto out;
 
-        /* <"op"> */
-        ret = xmlTextWriterStartElement (writer, (xmlChar *)op);
-        XML_RET_CHECK_AND_GOTO (ret, out);
+    /* <"op"> */
+    ret = xmlTextWriterStartElement(writer, (xmlChar *)op);
+    XML_RET_CHECK_AND_GOTO(ret, out);
 
-        if (dict)
-                dict_foreach (dict, cli_xml_output_data_pair, &writer);
+    if (dict)
+        dict_foreach(dict, cli_xml_output_data_pair, &writer);
 
-        /* </"op"> */
-        ret = xmlTextWriterEndElement (writer);
-        XML_RET_CHECK_AND_GOTO (ret, out);
+    /* </"op"> */
+    ret = xmlTextWriterEndElement(writer);
+    XML_RET_CHECK_AND_GOTO(ret, out);
 
-        ret = cli_end_xml_output (writer, doc);
+    ret = cli_end_xml_output(writer, doc);
 out:
-        gf_log ("cli", GF_LOG_DEBUG, "Returning %d", ret);
-        return ret;
+    gf_log("cli", GF_LOG_DEBUG, "Returning %d", ret);
+    return ret;
 #else
-        return 0;
+    return 0;
 #endif
 }
 
 #if (HAVE_LIB_XML)
 int
-cli_xml_output_vol_status_common (xmlTextWriterPtr writer, dict_t *dict,
-                                  int   brick_index, int *online,
-                                  gf_boolean_t *node_present)
+cli_xml_output_vol_status_common(xmlTextWriterPtr writer, dict_t *dict,
+                                 int brick_index, int *online,
+                                 gf_boolean_t *node_present)
 {
-        int             ret             = -1;
-        char            *hostname       = NULL;
-        char            *path           = NULL;
-        char            *uuid           = NULL;
-        int             port            = 0;
-        int             rdma_port       = 0;
-        int             status          = 0;
-        int             pid             = 0;
-        char            key[1024]       = {0,};
-
-        snprintf (key, sizeof (key), "brick%d.hostname", brick_index);
-        ret = dict_get_str (dict, key, &hostname);
-        if (ret) {
-                *node_present = _gf_false;
-                goto out;
+    int ret = -1;
+    char *hostname = NULL;
+    char *path = NULL;
+    char *uuid = NULL;
+    int port = 0;
+    int rdma_port = 0;
+    int status = 0;
+    int pid = 0;
+    char key[1024] = {
+        0,
+    };
+
+    snprintf(key, sizeof(key), "brick%d.hostname", brick_index);
+    ret = dict_get_str(dict, key, &hostname);
+    if (ret) {
+        *node_present = _gf_false;
+        goto out;
+    }
+    *node_present = _gf_true;
+
+    /* <node>
+     * will be closed in the calling function cli_xml_output_vol_status()*/
+    ret = xmlTextWriterStartElement(writer, (xmlChar *)"node");
+    XML_RET_CHECK_AND_GOTO(ret, out);
+
+    ret = xmlTextWriterWriteFormatElement(writer, (xmlChar *)"hostname", "%s",
+                                          hostname);
+    XML_RET_CHECK_AND_GOTO(ret, out);
+
+    snprintf(key, sizeof(key), "brick%d.path", brick_index);
+    ret = dict_get_str(dict, key, &path);
+    if (ret)
+        goto out;
+    ret = xmlTextWriterWriteFormatElement(writer, (xmlChar *)"path", "%s",
+                                          path);
+    XML_RET_CHECK_AND_GOTO(ret, out);
+
+    snprintf(key, sizeof(key), "brick%d.peerid", brick_index);
+    ret = dict_get_str(dict, key, &uuid);
+    if (ret)
+        goto out;
+    ret = xmlTextWriterWriteFormatElement(writer, (xmlChar *)"peerid", "%s",
+                                          uuid);
+    XML_RET_CHECK_AND_GOTO(ret, out);
+
+    snprintf(key, sizeof(key), "brick%d.status", brick_index);
+    ret = dict_get_int32(dict, key, &status);
+    if (ret)
+        goto out;
+    ret = xmlTextWriterWriteFormatElement(writer, (xmlChar *)"status", "%d",
+                                          status);
+    XML_RET_CHECK_AND_GOTO(ret, out);
+    *online = status;
+
+    snprintf(key, sizeof(key), "brick%d.port", brick_index);
+    ret = dict_get_int32(dict, key, &port);
+    if (ret)
+        goto out;
+
+    snprintf(key, sizeof(key), "brick%d.rdma_port", brick_index);
+    ret = dict_get_int32(dict, key, &rdma_port);
+
+    /* If the process is either offline or doesn't provide a port (shd)
+     * port = "N/A"
+     * else print the port number of the process.
+     */
+
+    /*
+     * Tag 'port' can be removed once console management is started
+     * to support new tag ports.
+     */
+
+    if (*online == 1 && port != 0)
+        ret = xmlTextWriterWriteFormatElement(writer, (xmlChar *)"port", "%d",
+                                              port);
+    else
+        ret = xmlTextWriterWriteFormatElement(writer, (xmlChar *)"port", "%s",
+                                              "N/A");
+    XML_RET_CHECK_AND_GOTO(ret, out);
+    ret = xmlTextWriterStartElement(writer, (xmlChar *)"ports");
+    if (*online == 1 && (port != 0 || rdma_port != 0)) {
+        if (port) {
+            ret = xmlTextWriterWriteFormatElement(writer, (xmlChar *)"tcp",
+                                                  "%d", port);
+        } else {
+            ret = xmlTextWriterWriteFormatElement(writer, (xmlChar *)"tcp",
+                                                  "%s", "N/A");
         }
-        *node_present = _gf_true;
-
-        /* <node>
-         * will be closed in the calling function cli_xml_output_vol_status()*/
-        ret = xmlTextWriterStartElement (writer, (xmlChar *)"node");
-        XML_RET_CHECK_AND_GOTO (ret, out);
-
-        ret = xmlTextWriterWriteFormatElement (writer, (xmlChar *)"hostname",
-                                               "%s", hostname);
-        XML_RET_CHECK_AND_GOTO (ret, out);
-
-        memset (key, 0, sizeof (key));
-        snprintf (key, sizeof (key), "brick%d.path", brick_index);
-        ret = dict_get_str (dict, key, &path);
-        if (ret)
-                goto out;
-        ret = xmlTextWriterWriteFormatElement (writer, (xmlChar *)"path",
-                                               "%s", path);
-        XML_RET_CHECK_AND_GOTO (ret, out);
-
-        memset (key, 0, sizeof (key));
-        snprintf (key, sizeof (key), "brick%d.peerid", brick_index);
-        ret = dict_get_str (dict, key, &uuid);
-        if (ret)
-                goto out;
-        ret = xmlTextWriterWriteFormatElement (writer, (xmlChar *)"peerid",
-                                               "%s", uuid);
-        XML_RET_CHECK_AND_GOTO (ret, out);
-
-        memset (key, 0, sizeof (key));
-        snprintf (key, sizeof (key), "brick%d.status", brick_index);
-        ret = dict_get_int32 (dict, key, &status);
-        if (ret)
-                goto out;
-        ret = xmlTextWriterWriteFormatElement (writer, (xmlChar *)"status",
-                                               "%d", status);
-        XML_RET_CHECK_AND_GOTO (ret, out);
-        *online = status;
-
-        memset (key, 0, sizeof (key));
-        snprintf (key, sizeof (key), "brick%d.port", brick_index);
-        ret = dict_get_int32 (dict, key, &port);
-        if (ret)
-                goto out;
-
-        memset (key, 0, sizeof (key));
-        snprintf (key, sizeof (key), "brick%d.rdma_port", brick_index);
-        ret = dict_get_int32 (dict, key, &rdma_port);
-
-        /* If the process is either offline or doesn't provide a port (shd)
-         * port = "N/A"
-         * else print the port number of the process.
-         */
-
-        /*
-         * Tag 'port' can be removed once console management is started
-         * to support new tag ports.
-         */
-
-        if (*online == 1 && port != 0)
-                 ret = xmlTextWriterWriteFormatElement (writer,
-                                                    (xmlChar *)"port",
-                                                        "%d", port);
-         else
-                 ret = xmlTextWriterWriteFormatElement (writer,
-                                                        (xmlChar *)"port",
-                                                        "%s", "N/A");
-
-        ret = xmlTextWriterStartElement (writer, (xmlChar *)"ports");
-        if (*online == 1 && (port != 0 || rdma_port != 0)) {
-
-                if (port) {
-                ret = xmlTextWriterWriteFormatElement (writer,
-                                                       (xmlChar *)"tcp",
-                                                       "%d", port);
-                } else {
-                ret = xmlTextWriterWriteFormatElement (writer,
-                                                       (xmlChar *)"tcp",
-                                                       "%s", "N/A");
-                }
-
-                if (rdma_port) {
-                ret = xmlTextWriterWriteFormatElement (writer,
-                                                       (xmlChar *)"rdma",
-                                                       "%d", rdma_port);
-                } else {
-                ret = xmlTextWriterWriteFormatElement (writer,
-                                                       (xmlChar *)"rdma",
-                                                       "%s", "N/A");
-                }
-
+        XML_RET_CHECK_AND_GOTO(ret, out);
+        if (rdma_port) {
+            ret = xmlTextWriterWriteFormatElement(writer, (xmlChar *)"rdma",
+                                                  "%d", rdma_port);
         } else {
-                ret = xmlTextWriterWriteFormatElement (writer,
-                                                       (xmlChar *)"tcp",
-                                                       "%s", "N/A");
-                ret = xmlTextWriterWriteFormatElement (writer,
-                                                       (xmlChar *)"rdma",
-                                                       "%s", "N/A");
+            ret = xmlTextWriterWriteFormatElement(writer, (xmlChar *)"rdma",
+                                                  "%s", "N/A");
         }
+        XML_RET_CHECK_AND_GOTO(ret, out);
+    } else {
+        ret = xmlTextWriterWriteFormatElement(writer, (xmlChar *)"tcp", "%s",
+                                              "N/A");
+        XML_RET_CHECK_AND_GOTO(ret, out);
+        ret = xmlTextWriterWriteFormatElement(writer, (xmlChar *)"rdma", "%s",
+                                              "N/A");
+        XML_RET_CHECK_AND_GOTO(ret, out);
+    }
 
-        ret = xmlTextWriterEndElement (writer);
-        XML_RET_CHECK_AND_GOTO (ret, out);
+    ret = xmlTextWriterEndElement(writer);
+    XML_RET_CHECK_AND_GOTO(ret, out);
 
-        memset (key, 0, sizeof (key));
-        snprintf (key, sizeof (key), "brick%d.pid", brick_index);
-        ret = dict_get_int32 (dict, key, &pid);
-        if (ret)
-                goto out;
-        ret = xmlTextWriterWriteFormatElement (writer, (xmlChar *)"pid",
-                                               "%d", pid);
-        XML_RET_CHECK_AND_GOTO (ret, out);
+    snprintf(key, sizeof(key), "brick%d.pid", brick_index);
+    ret = dict_get_int32(dict, key, &pid);
+    if (ret)
+        goto out;
+    ret = xmlTextWriterWriteFormatElement(writer, (xmlChar *)"pid", "%d", pid);
+    XML_RET_CHECK_AND_GOTO(ret, out);
 
 out:
-        gf_log ("cli", GF_LOG_DEBUG, "Returning %d", ret);
-        return ret;
+    gf_log("cli", GF_LOG_DEBUG, "Returning %d", ret);
+    return ret;
 }
 
 int
-cli_xml_output_vol_status_detail (xmlTextWriterPtr writer, dict_t *dict,
-                                  int brick_index)
-{
-        int             ret = -1;
-        uint64_t        size_total = 0;
-        uint64_t        size_free = 0;
-        char            *device = NULL;
-        uint64_t        block_size = 0;
-        char            *mnt_options = NULL;
-        char            *fs_name = NULL;
-        char            *inode_size = NULL;
-        uint64_t        inodes_total = 0;
-        uint64_t        inodes_free = 0;
-        char            key[1024] = {0,};
-
-        snprintf (key, sizeof (key), "brick%d.total", brick_index);
-        ret = dict_get_uint64 (dict, key, &size_total);
-        if (!ret)
-                ret = xmlTextWriterWriteFormatElement (writer,
-                                                       (xmlChar *)"sizeTotal",
-                                                       "%"PRIu64, size_total);
-
-        memset (key, 0, sizeof (key));
-        snprintf (key, sizeof (key), "brick%d.free", brick_index);
-        ret = dict_get_uint64 (dict, key, &size_free);
-        if (!ret)
-                ret = xmlTextWriterWriteFormatElement (writer,
-                                                       (xmlChar *)"sizeFree",
-                                                       "%"PRIu64, size_free);
-
-        memset (key, 0, sizeof (key));
-        snprintf (key, sizeof (key), "brick%d.device", brick_index);
-        ret = dict_get_str (dict, key, &device);
-        if (!ret)
-                ret = xmlTextWriterWriteFormatElement (writer,
-                                                       (xmlChar *)"device",
-                                                       "%s", device);
-
-        memset (key, 0, sizeof (key));
-        snprintf (key, sizeof (key), "brick%d.block_size", brick_index);
-        ret = dict_get_uint64 (dict, key, &block_size);
-        if (!ret)
-                ret = xmlTextWriterWriteFormatElement (writer,
-                                                       (xmlChar *)"blockSize",
-                                                       "%"PRIu64, block_size);
-        XML_RET_CHECK_AND_GOTO (ret, out);
-
-        memset (key, 0, sizeof (key));
-        snprintf (key, sizeof (key), "brick%d.mnt_options", brick_index);
-        ret = dict_get_str (dict, key, &mnt_options);
-        if (!ret)
-                ret = xmlTextWriterWriteFormatElement (writer,
-                                                       (xmlChar *)"mntOptions",
-                                                       "%s", mnt_options);
-
-        memset (key, 0, sizeof (key));
-        snprintf (key, sizeof (key), "brick%d.fs_name", brick_index);
-        ret = dict_get_str (dict, key, &fs_name);
-        if (!ret)
-                ret = xmlTextWriterWriteFormatElement (writer,
-                                                       (xmlChar *)"fsName",
-                                                       "%s", fs_name);
-
-        /* inode details are only available for ext 2/3/4 & xfs */
-        if (!fs_name || !IS_EXT_FS(fs_name) || strcmp (fs_name, "xfs")) {
-                        ret = 0;
-                        goto out;
-        }
-
-        memset (key, 0, sizeof (key));
-        snprintf (key, sizeof (key), "brick%d.inode_size", brick_index);
-        ret = dict_get_str (dict, key, &inode_size);
-        if (!ret)
-                ret = xmlTextWriterWriteFormatElement (writer,
-                                                       (xmlChar *)"inodeSize",
-                                                       "%s", fs_name);
-
-        memset (key, 0, sizeof (key));
-        snprintf (key, sizeof (key), "brick%d.total_inodes", brick_index);
-        ret = dict_get_uint64 (dict, key, &inodes_total);
-        if (!ret)
-                ret = xmlTextWriterWriteFormatElement (writer,
-                                                       (xmlChar *)"inodesTotal",
-                                                       "%"PRIu64, inodes_total);
-
-        memset (key, 0, sizeof (key));
-        snprintf (key, sizeof (key), "brick%d.free_inodes", brick_index);
-        ret = dict_get_uint64 (dict, key, &inodes_free);
-        if (!ret)
-                ret = xmlTextWriterWriteFormatElement (writer,
-                                                       (xmlChar *)"inodesFree",
-                                                       "%"PRIu64, inodes_free);
-
-out:
-        gf_log ("cli", GF_LOG_DEBUG, "Returning %d", ret);
-        return ret;
-}
-
-int
-cli_xml_output_vol_status_mempool (xmlTextWriterPtr writer, dict_t *dict,
-                                   char *prefix)
+cli_xml_output_vol_status_detail(xmlTextWriterPtr writer, dict_t *dict,
+                                 int brick_index)
 {
-        int             ret = -1;
-        int             mempool_count = 0;
-        char            *name = NULL;
-        int             hotcount = 0;
-        int             coldcount = 0;
-        uint64_t        paddedsizeof = 0;
-        uint64_t        alloccount = 0;
-        int             maxalloc = 0;
-        char            key[1024] = {0,};
-        int             i = 0;
-
-        /* <mempool> */
-        ret = xmlTextWriterStartElement (writer, (xmlChar *)"mempool");
-        XML_RET_CHECK_AND_GOTO (ret, out);
-
-        snprintf (key, sizeof (key), "%s.mempool-count", prefix);
-        ret = dict_get_int32 (dict, key, &mempool_count);
-        if (ret)
-                goto out;
-        ret = xmlTextWriterWriteFormatElement (writer, (xmlChar *)"count",
-                                               "%d", mempool_count);
-        XML_RET_CHECK_AND_GOTO (ret, out);
-
-        for (i = 0; i < mempool_count; i++) {
-                /* <pool> */
-                ret = xmlTextWriterStartElement (writer, (xmlChar *)"pool");
-                XML_RET_CHECK_AND_GOTO (ret, out);
-
-                memset (key, 0, sizeof (key));
-                snprintf (key, sizeof (key), "%s.pool%d.name", prefix, i);
-                ret = dict_get_str (dict, key, &name);
-                if (ret)
-                        goto out;
-                ret = xmlTextWriterWriteFormatElement (writer,
-                                                       (xmlChar *)"name",
-                                                       "%s", name);
-                XML_RET_CHECK_AND_GOTO (ret, out);
-
-                memset (key, 0, sizeof (key));
-                snprintf (key, sizeof (key), "%s.pool%d.hotcount", prefix, i);
-                ret = dict_get_int32 (dict, key, &hotcount);
-                if (ret)
-                        goto out;
-                ret = xmlTextWriterWriteFormatElement (writer,
-                                                       (xmlChar *)"hotCount",
-                                                       "%d", hotcount);
-                XML_RET_CHECK_AND_GOTO (ret, out);
-
-                memset (key, 0, sizeof (key));
-                snprintf (key, sizeof (key), "%s.pool%d.coldcount", prefix, i);
-                ret = dict_get_int32 (dict, key, &coldcount);
-                if (ret)
-                        goto out;
-                ret = xmlTextWriterWriteFormatElement (writer,
-                                                       (xmlChar *)"coldCount",
-                                                       "%d", coldcount);
-                XML_RET_CHECK_AND_GOTO (ret, out);
-
-                memset (key, 0, sizeof (key));
-                snprintf (key, sizeof (key), "%s.pool%d.paddedsizeof",
-                          prefix, i);
-                ret = dict_get_uint64 (dict, key, &paddedsizeof);
-                if (ret)
-                        goto out;
-                ret = xmlTextWriterWriteFormatElement
-                        (writer, (xmlChar *)"padddedSizeOf", "%"PRIu64,
-                         paddedsizeof);
-                XML_RET_CHECK_AND_GOTO (ret, out);
-
-                memset (key, 0, sizeof (key));
-                snprintf (key, sizeof (key), "%s.pool%d.alloccount", prefix, i);
-                ret = dict_get_uint64 (dict, key, &alloccount);
-                if (ret)
-                        goto out;
-                ret = xmlTextWriterWriteFormatElement (writer,
-                                                       (xmlChar *)"allocCount",
-                                                       "%"PRIu64, alloccount);
-                XML_RET_CHECK_AND_GOTO (ret, out);
-
-                memset (key, 0, sizeof (key));
-                snprintf (key, sizeof (key), "%s.pool%d.max_alloc", prefix, i);
-                ret = dict_get_int32 (dict, key, &maxalloc);
-                if (ret)
-                        goto out;
-                ret = xmlTextWriterWriteFormatElement (writer,
-                                                       (xmlChar *)"maxAlloc",
-                                                       "%d", maxalloc);
-                XML_RET_CHECK_AND_GOTO (ret, out);
-
-                memset (key, 0, sizeof (key));
-                snprintf (key, sizeof (key), "%s.pool%d.pool-misses", prefix, i);
-                ret = dict_get_uint64 (dict, key, &alloccount);
-                if (ret)
-                        goto out;
-                ret = xmlTextWriterWriteFormatElement (writer,
-                                                       (xmlChar *)"poolMisses",
-                                                       "%"PRIu64, alloccount);
-                XML_RET_CHECK_AND_GOTO (ret, out);
-
-                memset (key, 0, sizeof (key));
-                snprintf (key, sizeof (key), "%s.pool%d.max-stdalloc", prefix, i);
-                ret = dict_get_int32 (dict, key, &maxalloc);
-                if (ret)
-                        goto out;
-                ret = xmlTextWriterWriteFormatElement (writer,
-                                                       (xmlChar *)"maxStdAlloc",
-                                                       "%d", maxalloc);
-                XML_RET_CHECK_AND_GOTO (ret, out);
-
-
-                /* </pool> */
-                ret = xmlTextWriterEndElement (writer);
-                XML_RET_CHECK_AND_GOTO (ret, out);
-        }
-
-        /* </mempool> */
-        ret = xmlTextWriterEndElement (writer);
-        XML_RET_CHECK_AND_GOTO (ret, out);
+    int ret = -1;
+    uint64_t size_total = 0;
+    uint64_t size_free = 0;
+    char *device = NULL;
+    uint64_t block_size = 0;
+    char *mnt_options = NULL;
+    char *fs_name = NULL;
+    char *inode_size = NULL;
+    uint64_t inodes_total = 0;
+    uint64_t inodes_free = 0;
+    char key[1024] = {
+        0,
+    };
+
+    snprintf(key, sizeof(key), "brick%d.total", brick_index);
+    ret = dict_get_uint64(dict, key, &size_total);
+    if (!ret) {
+        ret = xmlTextWriterWriteFormatElement(writer, (xmlChar *)"sizeTotal",
+                                              "%" PRIu64, size_total);
+        XML_RET_CHECK_AND_GOTO(ret, out);
+    }
+    snprintf(key, sizeof(key), "brick%d.free", brick_index);
+    ret = dict_get_uint64(dict, key, &size_free);
+    if (!ret) {
+        ret = xmlTextWriterWriteFormatElement(writer, (xmlChar *)"sizeFree",
+                                              "%" PRIu64, size_free);
+        XML_RET_CHECK_AND_GOTO(ret, out);
+    }
+    snprintf(key, sizeof(key), "brick%d.device", brick_index);
+    ret = dict_get_str(dict, key, &device);
+    if (!ret) {
+        ret = xmlTextWriterWriteFormatElement(writer, (xmlChar *)"device", "%s",
+                                              device);
+        XML_RET_CHECK_AND_GOTO(ret, out);
+    }
+    snprintf(key, sizeof(key), "brick%d.block_size", brick_index);
+    ret = dict_get_uint64(dict, key, &block_size);
+    if (!ret) {
+        ret = xmlTextWriterWriteFormatElement(writer, (xmlChar *)"blockSize",
+                                              "%" PRIu64, block_size);
+        XML_RET_CHECK_AND_GOTO(ret, out);
+    }
+    snprintf(key, sizeof(key), "brick%d.mnt_options", brick_index);
+    ret = dict_get_str(dict, key, &mnt_options);
+    if (!ret) {
+        ret = xmlTextWriterWriteFormatElement(writer, (xmlChar *)"mntOptions",
+                                              "%s", mnt_options);
+        XML_RET_CHECK_AND_GOTO(ret, out);
+    }
+    snprintf(key, sizeof(key), "brick%d.fs_name", brick_index);
+    ret = dict_get_str(dict, key, &fs_name);
+    if (!ret) {
+        ret = xmlTextWriterWriteFormatElement(writer, (xmlChar *)"fsName", "%s",
+                                              fs_name);
+        XML_RET_CHECK_AND_GOTO(ret, out);
+    }
+    snprintf(key, sizeof(key), "brick%d.inode_size", brick_index);
+    ret = dict_get_str(dict, key, &inode_size);
+    if (!ret) {
+        ret = xmlTextWriterWriteFormatElement(writer, (xmlChar *)"inodeSize",
+                                              "%s", fs_name);
+        XML_RET_CHECK_AND_GOTO(ret, out);
+    }
+    snprintf(key, sizeof(key), "brick%d.total_inodes", brick_index);
+    ret = dict_get_uint64(dict, key, &inodes_total);
+    if (!ret) {
+        ret = xmlTextWriterWriteFormatElement(writer, (xmlChar *)"inodesTotal",
+                                              "%" PRIu64, inodes_total);
+        XML_RET_CHECK_AND_GOTO(ret, out);
+    }
+    snprintf(key, sizeof(key), "brick%d.free_inodes", brick_index);
+    ret = dict_get_uint64(dict, key, &inodes_free);
+    if (!ret) {
+        ret = xmlTextWriterWriteFormatElement(writer, (xmlChar *)"inodesFree",
+                                              "%" PRIu64, inodes_free);
+        XML_RET_CHECK_AND_GOTO(ret, out);
+    } else {
+        ret = 0;
+    }
 
 out:
-        gf_log ("cli", GF_LOG_DEBUG, "Returning %d", ret);
-        return ret;
+    gf_log("cli", GF_LOG_DEBUG, "Returning %d", ret);
+    return ret;
 }
 
 int
-cli_xml_output_vol_status_mem (xmlTextWriterPtr writer, dict_t *dict,
-                               int brick_index)
+cli_xml_output_vol_status_mempool(xmlTextWriterPtr writer, dict_t *dict,
+                                  char *prefix)
 {
-        int             ret = -1;
-        int             arena = 0;
-        int             ordblks = 0;
-        int             smblks = 0;
-        int             hblks = 0;
-        int             hblkhd = 0;
-        int             usmblks = 0;
-        int             fsmblks = 0;
-        int             uordblks = 0;
-        int             fordblks = 0;
-        int             keepcost = 0;
-        char            key[1024] = {0,};
-
-        /* <memStatus> */
-        ret = xmlTextWriterStartElement (writer, (xmlChar *)"memStatus");
-        XML_RET_CHECK_AND_GOTO (ret, out);
-
-        /* <mallinfo> */
-        ret = xmlTextWriterStartElement (writer, (xmlChar *)"mallinfo");
-        XML_RET_CHECK_AND_GOTO (ret, out);
-
-        snprintf (key, sizeof (key), "brick%d.mallinfo.arena", brick_index);
-        ret = dict_get_int32 (dict, key, &arena);
-        if (ret)
-                goto out;
-        ret = xmlTextWriterWriteFormatElement (writer, (xmlChar *)"arena",
-                                               "%d", arena);
-        XML_RET_CHECK_AND_GOTO (ret, out);
-
-        memset (key, 0, sizeof (key));
-        snprintf (key, sizeof (key), "brick%d.mallinfo.ordblks", brick_index);
-        ret = dict_get_int32 (dict, key, &ordblks);
-        if (ret)
-                goto out;
-        ret = xmlTextWriterWriteFormatElement (writer, (xmlChar *)"ordblks",
-                                               "%d", ordblks);
-        XML_RET_CHECK_AND_GOTO (ret, out);
+    int ret = -1;
+    int mempool_count = 0;
+    char *name = NULL;
+    int hotcount = 0;
+    int coldcount = 0;
+    uint64_t paddedsizeof = 0;
+    uint64_t alloccount = 0;
+    int maxalloc = 0;
+    char key[1024] = {
+        0,
+    };
+    int i = 0;
+
+    /* <mempool> */
+    ret = xmlTextWriterStartElement(writer, (xmlChar *)"mempool");
+    XML_RET_CHECK_AND_GOTO(ret, out);
+
+    snprintf(key, sizeof(key), "%s.mempool-count", prefix);
+    ret = dict_get_int32(dict, key, &mempool_count);
+    if (ret)
+        goto out;
+    ret = xmlTextWriterWriteFormatElement(writer, (xmlChar *)"count", "%d",
+                                          mempool_count);
+    XML_RET_CHECK_AND_GOTO(ret, out);
+
+    for (i = 0; i < mempool_count; i++) {
+        /* <pool> */
+        ret = xmlTextWriterStartElement(writer, (xmlChar *)"pool");
+        XML_RET_CHECK_AND_GOTO(ret, out);
 
-        memset (key, 0, sizeof (key));
-        snprintf (key, sizeof (key), "brick%d.mallinfo.smblks", brick_index);
-        ret = dict_get_int32 (dict, key, &smblks);
+        snprintf(key, sizeof(key), "%s.pool%d.name", prefix, i);
+        ret = dict_get_str(dict, key, &name);
         if (ret)
-                goto out;
-        ret = xmlTextWriterWriteFormatElement (writer, (xmlChar *)"smblks",
-                                               "%d", smblks);
-        XML_RET_CHECK_AND_GOTO (ret, out);
+            goto out;
+        ret = xmlTextWriterWriteFormatElement(writer, (xmlChar *)"name", "%s",
+                                              name);
+        XML_RET_CHECK_AND_GOTO(ret, out);
 
-        memset (key, 0, sizeof (key));
-        snprintf (key, sizeof (key), "brick%d.mallinfo.hblks", brick_index);
-        ret = dict_get_int32 (dict, key, &hblks);
+        snprintf(key, sizeof(key), "%s.pool%d.hotcount", prefix, i);
+        ret = dict_get_int32(dict, key, &hotcount);
         if (ret)
-                goto out;
-        ret = xmlTextWriterWriteFormatElement (writer, (xmlChar *)"hblks",
-                                               "%d", hblks);
-        XML_RET_CHECK_AND_GOTO (ret, out);
+            goto out;
+        ret = xmlTextWriterWriteFormatElement(writer, (xmlChar *)"hotCount",
+                                              "%d", hotcount);
+        XML_RET_CHECK_AND_GOTO(ret, out);
 
-        memset (key, 0, sizeof (key));
-        snprintf (key, sizeof (key), "brick%d.mallinfo.hblkhd", brick_index);
-        ret = dict_get_int32 (dict, key, &hblkhd);
+        snprintf(key, sizeof(key), "%s.pool%d.coldcount", prefix, i);
+        ret = dict_get_int32(dict, key, &coldcount);
         if (ret)
-                goto out;
-        ret = xmlTextWriterWriteFormatElement (writer, (xmlChar *)"hblkhd",
-                                               "%d", hblkhd);
-        XML_RET_CHECK_AND_GOTO (ret, out);
+            goto out;
+        ret = xmlTextWriterWriteFormatElement(writer, (xmlChar *)"coldCount",
+                                              "%d", coldcount);
+        XML_RET_CHECK_AND_GOTO(ret, out);
 
-        memset (key, 0, sizeof (key));
-        snprintf (key, sizeof (key), "brick%d.mallinfo.usmblks", brick_index);
-        ret = dict_get_int32 (dict, key, &usmblks);
+        snprintf(key, sizeof(key), "%s.pool%d.paddedsizeof", prefix, i);
+        ret = dict_get_uint64(dict, key, &paddedsizeof);
         if (ret)
-                goto out;
-        ret = xmlTextWriterWriteFormatElement (writer, (xmlChar *)"usmblks",
-                                               "%d", usmblks);
-        XML_RET_CHECK_AND_GOTO (ret, out);
+            goto out;
+        ret = xmlTextWriterWriteFormatElement(
+            writer, (xmlChar *)"padddedSizeOf", "%" PRIu64, paddedsizeof);
+        XML_RET_CHECK_AND_GOTO(ret, out);
 
-        memset (key, 0, sizeof (key));
-        snprintf (key, sizeof (key), "brick%d.mallinfo.fsmblks", brick_index);
-        ret = dict_get_int32 (dict, key, &fsmblks);
+        snprintf(key, sizeof(key), "%s.pool%d.alloccount", prefix, i);
+        ret = dict_get_uint64(dict, key, &alloccount);
         if (ret)
-                goto out;
-        ret = xmlTextWriterWriteFormatElement (writer, (xmlChar *)"fsmblks",
-                                               "%d", fsmblks);
-        XML_RET_CHECK_AND_GOTO (ret, out);
+            goto out;
+        ret = xmlTextWriterWriteFormatElement(writer, (xmlChar *)"allocCount",
+                                              "%" PRIu64, alloccount);
+        XML_RET_CHECK_AND_GOTO(ret, out);
 
-        memset (key, 0, sizeof (key));
-        snprintf (key, sizeof (key), "brick%d.mallinfo.uordblks", brick_index);
-        ret = dict_get_int32 (dict, key, &uordblks);
+        snprintf(key, sizeof(key), "%s.pool%d.max_alloc", prefix, i);
+        ret = dict_get_int32(dict, key, &maxalloc);
         if (ret)
-                goto out;
-        ret = xmlTextWriterWriteFormatElement (writer, (xmlChar *)"uordblks",
-                                               "%d", uordblks);
-        XML_RET_CHECK_AND_GOTO (ret, out);
+            goto out;
+        ret = xmlTextWriterWriteFormatElement(writer, (xmlChar *)"maxAlloc",
+                                              "%d", maxalloc);
+        XML_RET_CHECK_AND_GOTO(ret, out);
 
-        memset (key, 0, sizeof (key));
-        snprintf (key, sizeof (key), "brick%d.mallinfo.fordblks", brick_index);
-        ret = dict_get_int32 (dict, key, &fordblks);
+        snprintf(key, sizeof(key), "%s.pool%d.pool-misses", prefix, i);
+        ret = dict_get_uint64(dict, key, &alloccount);
         if (ret)
-                goto out;
-        ret = xmlTextWriterWriteFormatElement (writer, (xmlChar *)"fordblks",
-                                               "%d", fordblks);
-        XML_RET_CHECK_AND_GOTO (ret, out);
+            goto out;
+        ret = xmlTextWriterWriteFormatElement(writer, (xmlChar *)"poolMisses",
+                                              "%" PRIu64, alloccount);
+        XML_RET_CHECK_AND_GOTO(ret, out);
 
-        memset (key, 0, sizeof (key));
-        snprintf (key, sizeof (key), "brick%d.mallinfo.keepcost", brick_index);
-        ret = dict_get_int32 (dict, key, &keepcost);
+        snprintf(key, sizeof(key), "%s.pool%d.max-stdalloc", prefix, i);
+        ret = dict_get_int32(dict, key, &maxalloc);
         if (ret)
-                goto out;
-        ret = xmlTextWriterWriteFormatElement (writer, (xmlChar *)"keepcost",
-                                               "%d", keepcost);
-        XML_RET_CHECK_AND_GOTO (ret, out);
-
-        /* </mallinfo> */
-        ret = xmlTextWriterEndElement (writer);
-        XML_RET_CHECK_AND_GOTO (ret, out);
+            goto out;
+        ret = xmlTextWriterWriteFormatElement(writer, (xmlChar *)"maxStdAlloc",
+                                              "%d", maxalloc);
+        XML_RET_CHECK_AND_GOTO(ret, out);
 
-        memset (key, 0, sizeof (key));
-        snprintf (key, sizeof (key), "brick%d", brick_index);
-        ret = cli_xml_output_vol_status_mempool (writer, dict, key);
-        if (ret)
-                goto out;
+        /* </pool> */
+        ret = xmlTextWriterEndElement(writer);
+        XML_RET_CHECK_AND_GOTO(ret, out);
+    }
 
-        /* </memStatus> */
-        ret = xmlTextWriterEndElement (writer);
-        XML_RET_CHECK_AND_GOTO (ret, out);
+    /* </mempool> */
+    ret = xmlTextWriterEndElement(writer);
+    XML_RET_CHECK_AND_GOTO(ret, out);
 
 out:
-        gf_log ("cli", GF_LOG_DEBUG, "Returning %d", ret);
-        return ret;
+    gf_log("cli", GF_LOG_DEBUG, "Returning %d", ret);
+    return ret;
 }
 
 int
-cli_xml_output_vol_status_clients (xmlTextWriterPtr writer, dict_t *dict,
-                                   int brick_index)
+cli_xml_output_vol_status_mem(xmlTextWriterPtr writer, dict_t *dict,
+                              int brick_index)
 {
-        int             ret = -1;
-        int             client_count = 0;
-        char            *hostname = NULL;
-        uint64_t        bytes_read = 0;
-        uint64_t        bytes_write = 0;
-        char            key[1024] = {0,};
-        int             i = 0;
-
-        /* <clientsStatus> */
-        ret = xmlTextWriterStartElement (writer, (xmlChar *)"clientsStatus");
-        XML_RET_CHECK_AND_GOTO (ret, out);
-
-        snprintf (key, sizeof (key), "brick%d.clientcount", brick_index);
-        ret = dict_get_int32 (dict, key, &client_count);
-        if (ret)
-                goto out;
-        ret = xmlTextWriterWriteFormatElement (writer,
-                                               (xmlChar *)"clientCount",
-                                               "%d", client_count);
-        XML_RET_CHECK_AND_GOTO (ret, out);
-
-        for (i = 0; i < client_count; i++) {
-                /* <client> */
-                ret = xmlTextWriterStartElement (writer, (xmlChar *)"client");
-                XML_RET_CHECK_AND_GOTO (ret, out);
-
-                memset (key, 0, sizeof (key));
-                snprintf (key, sizeof (key), "brick%d.client%d.hostname",
-                          brick_index, i);
-                ret = dict_get_str (dict, key, &hostname);
-                if (ret)
-                        goto out;
-                ret = xmlTextWriterWriteFormatElement (writer,
-                                                       (xmlChar *)"hostname",
-                                                       "%s", hostname);
-                XML_RET_CHECK_AND_GOTO (ret, out);
-
-                memset (key, 0, sizeof (key));
-                snprintf (key, sizeof (key), "brick%d.client%d.bytesread",
-                          brick_index, i);
-                ret = dict_get_uint64 (dict, key, &bytes_read);
-                if (ret)
-                        goto out;
-                ret = xmlTextWriterWriteFormatElement (writer,
-                                                       (xmlChar *)"bytesRead",
-                                                       "%"PRIu64, bytes_read);
-                XML_RET_CHECK_AND_GOTO (ret, out);
-
-                memset (key, 0, sizeof (key));
-                snprintf (key, sizeof (key), "brick%d.client%d.byteswrite",
-                          brick_index, i);
-                ret = dict_get_uint64 (dict, key, &bytes_write);
-                if (ret)
-                        goto out;
-                ret = xmlTextWriterWriteFormatElement (writer,
-                                                       (xmlChar *)"bytesWrite",
-                                                       "%"PRIu64, bytes_write);
-                XML_RET_CHECK_AND_GOTO (ret, out);
-
-                /* </client> */
-                ret = xmlTextWriterEndElement (writer);
-                XML_RET_CHECK_AND_GOTO (ret, out);
-        }
+    int ret = -1;
+    int arena = 0;
+    int ordblks = 0;
+    int smblks = 0;
+    int hblks = 0;
+    int hblkhd = 0;
+    int usmblks = 0;
+    int fsmblks = 0;
+    int uordblks = 0;
+    int fordblks = 0;
+    int keepcost = 0;
+    char key[1024] = {
+        0,
+    };
+
+    /* <memStatus> */
+    ret = xmlTextWriterStartElement(writer, (xmlChar *)"memStatus");
+    XML_RET_CHECK_AND_GOTO(ret, out);
+
+    /* <mallinfo> */
+    ret = xmlTextWriterStartElement(writer, (xmlChar *)"mallinfo");
+    XML_RET_CHECK_AND_GOTO(ret, out);
+
+    snprintf(key, sizeof(key), "brick%d.mallinfo.arena", brick_index);
+    ret = dict_get_int32(dict, key, &arena);
+    if (ret)
+        goto out;
+    ret = xmlTextWriterWriteFormatElement(writer, (xmlChar *)"arena", "%d",
+                                          arena);
+    XML_RET_CHECK_AND_GOTO(ret, out);
+
+    snprintf(key, sizeof(key), "brick%d.mallinfo.ordblks", brick_index);
+    ret = dict_get_int32(dict, key, &ordblks);
+    if (ret)
+        goto out;
+    ret = xmlTextWriterWriteFormatElement(writer, (xmlChar *)"ordblks", "%d",
+                                          ordblks);
+    XML_RET_CHECK_AND_GOTO(ret, out);
+
+    snprintf(key, sizeof(key), "brick%d.mallinfo.smblks", brick_index);
+    ret = dict_get_int32(dict, key, &smblks);
+    if (ret)
+        goto out;
+    ret = xmlTextWriterWriteFormatElement(writer, (xmlChar *)"smblks", "%d",
+                                          smblks);
+    XML_RET_CHECK_AND_GOTO(ret, out);
+
+    snprintf(key, sizeof(key), "brick%d.mallinfo.hblks", brick_index);
+    ret = dict_get_int32(dict, key, &hblks);
+    if (ret)
+        goto out;
+    ret = xmlTextWriterWriteFormatElement(writer, (xmlChar *)"hblks", "%d",
+                                          hblks);
+    XML_RET_CHECK_AND_GOTO(ret, out);
+
+    snprintf(key, sizeof(key), "brick%d.mallinfo.hblkhd", brick_index);
+    ret = dict_get_int32(dict, key, &hblkhd);
+    if (ret)
+        goto out;
+    ret = xmlTextWriterWriteFormatElement(writer, (xmlChar *)"hblkhd", "%d",
+                                          hblkhd);
+    XML_RET_CHECK_AND_GOTO(ret, out);
+
+    snprintf(key, sizeof(key), "brick%d.mallinfo.usmblks", brick_index);
+    ret = dict_get_int32(dict, key, &usmblks);
+    if (ret)
+        goto out;
+    ret = xmlTextWriterWriteFormatElement(writer, (xmlChar *)"usmblks", "%d",
+                                          usmblks);
+    XML_RET_CHECK_AND_GOTO(ret, out);
+
+    snprintf(key, sizeof(key), "brick%d.mallinfo.fsmblks", brick_index);
+    ret = dict_get_int32(dict, key, &fsmblks);
+    if (ret)
+        goto out;
+    ret = xmlTextWriterWriteFormatElement(writer, (xmlChar *)"fsmblks", "%d",
+                                          fsmblks);
+    XML_RET_CHECK_AND_GOTO(ret, out);
+
+    snprintf(key, sizeof(key), "brick%d.mallinfo.uordblks", brick_index);
+    ret = dict_get_int32(dict, key, &uordblks);
+    if (ret)
+        goto out;
+    ret = xmlTextWriterWriteFormatElement(writer, (xmlChar *)"uordblks", "%d",
+                                          uordblks);
+    XML_RET_CHECK_AND_GOTO(ret, out);
+
+    snprintf(key, sizeof(key), "brick%d.mallinfo.fordblks", brick_index);
+    ret = dict_get_int32(dict, key, &fordblks);
+    if (ret)
+        goto out;
+    ret = xmlTextWriterWriteFormatElement(writer, (xmlChar *)"fordblks", "%d",
+                                          fordblks);
+    XML_RET_CHECK_AND_GOTO(ret, out);
+
+    snprintf(key, sizeof(key), "brick%d.mallinfo.keepcost", brick_index);
+    ret = dict_get_int32(dict, key, &keepcost);
+    if (ret)
+        goto out;
+    ret = xmlTextWriterWriteFormatElement(writer, (xmlChar *)"keepcost", "%d",
+                                          keepcost);
+    XML_RET_CHECK_AND_GOTO(ret, out);
+
+    /* </mallinfo> */
+    ret = xmlTextWriterEndElement(writer);
+    XML_RET_CHECK_AND_GOTO(ret, out);
+
+    snprintf(key, sizeof(key), "brick%d", brick_index);
+    ret = cli_xml_output_vol_status_mempool(writer, dict, key);
+    if (ret)
+        goto out;
+
+    /* </memStatus> */
+    ret = xmlTextWriterEndElement(writer);
+    XML_RET_CHECK_AND_GOTO(ret, out);
 
-        /* </clientsStatus> */
-        ret = xmlTextWriterEndElement (writer);
-        XML_RET_CHECK_AND_GOTO (ret, out);
 out:
-        gf_log ("cli", GF_LOG_DEBUG, "Returning %d", ret);
-        return ret;
+    gf_log("cli", GF_LOG_DEBUG, "Returning %d", ret);
+    return ret;
 }
 
 int
-cli_xml_output_vol_status_inode_entry (xmlTextWriterPtr writer, dict_t *dict,
-                                       char *prefix)
+cli_xml_output_vol_status_clients(xmlTextWriterPtr writer, dict_t *dict,
+                                  int brick_index)
 {
-        int             ret = -1;
-        char            *gfid = NULL;
-        uint64_t        nlookup = 0;
-        uint32_t        ref = 0;
-        int             ia_type = 0;
-        char            key[1024] = {0,};
-
-        /* <inode> */
-        ret = xmlTextWriterStartElement (writer, (xmlChar *)"inode");
-        XML_RET_CHECK_AND_GOTO (ret, out);
-
-        snprintf (key, sizeof (key), "%s.gfid", prefix);
-        ret = dict_get_str (dict, key, &gfid);
+    int ret = -1;
+    int client_count = 0;
+    char *hostname = NULL;
+    uint64_t bytes_read = 0;
+    uint64_t bytes_write = 0;
+    uint32_t opversion = 0;
+    char key[1024] = {
+        0,
+    };
+    int i = 0;
+
+    /* <clientsStatus> */
+    ret = xmlTextWriterStartElement(writer, (xmlChar *)"clientsStatus");
+    XML_RET_CHECK_AND_GOTO(ret, out);
+
+    snprintf(key, sizeof(key), "brick%d.clientcount", brick_index);
+    ret = dict_get_int32(dict, key, &client_count);
+    if (ret)
+        goto out;
+    ret = xmlTextWriterWriteFormatElement(writer, (xmlChar *)"clientCount",
+                                          "%d", client_count);
+    XML_RET_CHECK_AND_GOTO(ret, out);
+
+    for (i = 0; i < client_count; i++) {
+        /* <client> */
+        ret = xmlTextWriterStartElement(writer, (xmlChar *)"client");
+        XML_RET_CHECK_AND_GOTO(ret, out);
+
+        snprintf(key, sizeof(key), "brick%d.client%d.hostname", brick_index, i);
+        ret = dict_get_str(dict, key, &hostname);
         if (ret)
-                goto out;
-        ret = xmlTextWriterWriteFormatElement (writer, (xmlChar *)"gfid",
-                                               "%s", gfid);
-        XML_RET_CHECK_AND_GOTO (ret, out);
+            goto out;
+        ret = xmlTextWriterWriteFormatElement(writer, (xmlChar *)"hostname",
+                                              "%s", hostname);
+        XML_RET_CHECK_AND_GOTO(ret, out);
 
-        memset (key,0, sizeof (key));
-        snprintf (key, sizeof (key), "%s.nlookup", prefix);
-        ret = dict_get_uint64 (dict, key, &nlookup);
+        snprintf(key, sizeof(key), "brick%d.client%d.bytesread", brick_index,
+                 i);
+        ret = dict_get_uint64(dict, key, &bytes_read);
         if (ret)
-                goto out;
-        ret = xmlTextWriterWriteFormatElement (writer, (xmlChar *)"nLookup",
-                                               "%"PRIu64, nlookup);
-        XML_RET_CHECK_AND_GOTO (ret, out);
+            goto out;
+        ret = xmlTextWriterWriteFormatElement(writer, (xmlChar *)"bytesRead",
+                                              "%" PRIu64, bytes_read);
+        XML_RET_CHECK_AND_GOTO(ret, out);
 
-        memset (key,0, sizeof (key));
-        snprintf (key, sizeof (key), "%s.ref", prefix);
-        ret = dict_get_uint32 (dict, key, &ref);
+        snprintf(key, sizeof(key), "brick%d.client%d.byteswrite", brick_index,
+                 i);
+        ret = dict_get_uint64(dict, key, &bytes_write);
         if (ret)
-                goto out;
-        ret = xmlTextWriterWriteFormatElement (writer, (xmlChar *)"ref",
-                                               "%"PRIu32, ref);
-        XML_RET_CHECK_AND_GOTO (ret, out);
+            goto out;
+        ret = xmlTextWriterWriteFormatElement(writer, (xmlChar *)"bytesWrite",
+                                              "%" PRIu64, bytes_write);
+        XML_RET_CHECK_AND_GOTO(ret, out);
 
-        memset (key,0, sizeof (key));
-        snprintf (key, sizeof (key), "%s.ia_type", prefix);
-        ret = dict_get_int32 (dict, key, &ia_type);
+        snprintf(key, sizeof(key), "brick%d.client%d.opversion", brick_index,
+                 i);
+        ret = dict_get_uint32(dict, key, &opversion);
         if (ret)
-                goto out;
-        ret = xmlTextWriterWriteFormatElement (writer, (xmlChar *)"iaType",
-                                               "%d", ia_type);
-        XML_RET_CHECK_AND_GOTO (ret, out);
+            goto out;
+        ret = xmlTextWriterWriteFormatElement(writer, (xmlChar *)"opVersion",
+                                              "%" PRIu32, opversion);
+        XML_RET_CHECK_AND_GOTO(ret, out);
 
-        /* </inode> */
-        ret = xmlTextWriterEndElement (writer);
-        XML_RET_CHECK_AND_GOTO (ret, out);
+        /* </client> */
+        ret = xmlTextWriterEndElement(writer);
+        XML_RET_CHECK_AND_GOTO(ret, out);
+    }
 
+    /* </clientsStatus> */
+    ret = xmlTextWriterEndElement(writer);
+    XML_RET_CHECK_AND_GOTO(ret, out);
 out:
-        gf_log ("cli", GF_LOG_DEBUG, "Returning %d", ret);
-        return ret;
+    gf_log("cli", GF_LOG_DEBUG, "Returning %d", ret);
+    return ret;
 }
 
 int
-cli_xml_output_vol_status_itable (xmlTextWriterPtr writer, dict_t *dict,
-                                  char *prefix)
+cli_xml_output_vol_status_inode_entry(xmlTextWriterPtr writer, dict_t *dict,
+                                      char *prefix)
 {
-        int             ret = -1;
-        uint32_t        active_size = 0;
-        uint32_t        lru_size = 0;
-        uint32_t        purge_size = 0;
-        char            key[1024] = {0,};
-        int             i = 0;
-
-        snprintf (key, sizeof (key), "%s.active_size", prefix);
-        ret = dict_get_uint32 (dict, key, &active_size);
-        if (ret)
+    int ret = -1;
+    char *gfid = NULL;
+    uint64_t nlookup = 0;
+    uint32_t ref = 0;
+    int ia_type = 0;
+    char key[1024] = {
+        0,
+    };
+
+    /* <inode> */
+    ret = xmlTextWriterStartElement(writer, (xmlChar *)"inode");
+    XML_RET_CHECK_AND_GOTO(ret, out);
+
+    snprintf(key, sizeof(key), "%s.gfid", prefix);
+    ret = dict_get_str(dict, key, &gfid);
+    if (ret)
+        goto out;
+    ret = xmlTextWriterWriteFormatElement(writer, (xmlChar *)"gfid", "%s",
+                                          gfid);
+    XML_RET_CHECK_AND_GOTO(ret, out);
+
+    snprintf(key, sizeof(key), "%s.nlookup", prefix);
+    ret = dict_get_uint64(dict, key, &nlookup);
+    if (ret)
+        goto out;
+    ret = xmlTextWriterWriteFormatElement(writer, (xmlChar *)"nLookup",
+                                          "%" PRIu64, nlookup);
+    XML_RET_CHECK_AND_GOTO(ret, out);
+
+    snprintf(key, sizeof(key), "%s.ref", prefix);
+    ret = dict_get_uint32(dict, key, &ref);
+    if (ret)
+        goto out;
+    ret = xmlTextWriterWriteFormatElement(writer, (xmlChar *)"ref", "%" PRIu32,
+                                          ref);
+    XML_RET_CHECK_AND_GOTO(ret, out);
+
+    snprintf(key, sizeof(key), "%s.ia_type", prefix);
+    ret = dict_get_int32(dict, key, &ia_type);
+    if (ret)
+        goto out;
+    ret = xmlTextWriterWriteFormatElement(writer, (xmlChar *)"iaType", "%d",
+                                          ia_type);
+    XML_RET_CHECK_AND_GOTO(ret, out);
+
+    /* </inode> */
+    ret = xmlTextWriterEndElement(writer);
+    XML_RET_CHECK_AND_GOTO(ret, out);
+
+out:
+    gf_log("cli", GF_LOG_DEBUG, "Returning %d", ret);
+    return ret;
+}
+
+int
+cli_xml_output_vol_status_itable(xmlTextWriterPtr writer, dict_t *dict,
+                                 char *prefix)
+{
+    int ret = -1;
+    uint32_t active_size = 0;
+    uint32_t lru_size = 0;
+    uint32_t purge_size = 0;
+    char key[1024] = {
+        0,
+    };
+    int i = 0;
+
+    snprintf(key, sizeof(key), "%s.active_size", prefix);
+    ret = dict_get_uint32(dict, key, &active_size);
+    if (ret)
+        goto out;
+    ret = xmlTextWriterWriteFormatElement(writer, (xmlChar *)"activeSize",
+                                          "%" PRIu32, active_size);
+    XML_RET_CHECK_AND_GOTO(ret, out);
+    if (active_size != 0) {
+        /* <active> */
+        ret = xmlTextWriterStartElement(writer, (xmlChar *)"active");
+        XML_RET_CHECK_AND_GOTO(ret, out);
+
+        for (i = 0; i < active_size; i++) {
+            snprintf(key, sizeof(key), "%s.active%d", prefix, i);
+            ret = cli_xml_output_vol_status_inode_entry(writer, dict, key);
+            if (ret)
                 goto out;
-        ret = xmlTextWriterWriteFormatElement (writer, (xmlChar *)"activeSize",
-                                               "%"PRIu32, active_size);
-        XML_RET_CHECK_AND_GOTO (ret, out);
-        if (active_size != 0) {
-                /* <active> */
-                ret = xmlTextWriterStartElement (writer, (xmlChar *)"active");
-                XML_RET_CHECK_AND_GOTO (ret, out);
-
-                for (i = 0; i < active_size; i++) {
-                        memset (key, 0, sizeof (key));
-                        snprintf (key, sizeof (key), "%s.active%d", prefix, i);
-                        ret = cli_xml_output_vol_status_inode_entry
-                                (writer, dict, key);
-                        if (ret)
-                                goto out;
-                }
-                /* </active> */
-                ret = xmlTextWriterEndElement (writer);
-                XML_RET_CHECK_AND_GOTO (ret, out);
         }
+        /* </active> */
+        ret = xmlTextWriterEndElement(writer);
+        XML_RET_CHECK_AND_GOTO(ret, out);
+    }
+
+    snprintf(key, sizeof(key), "%s.lru_size", prefix);
+    ret = dict_get_uint32(dict, key, &lru_size);
+    if (ret)
+        goto out;
+    ret = xmlTextWriterWriteFormatElement(writer, (xmlChar *)"lruSize",
+                                          "%" PRIu32, lru_size);
+    XML_RET_CHECK_AND_GOTO(ret, out);
+    if (lru_size != 0) {
+        /* <lru> */
+        ret = xmlTextWriterStartElement(writer, (xmlChar *)"lru");
+        XML_RET_CHECK_AND_GOTO(ret, out);
 
-        memset (key, 0, sizeof (key));
-        snprintf (key, sizeof (key), "%s.lru_size", prefix);
-        ret = dict_get_uint32 (dict, key, &lru_size);
-        if (ret)
+        for (i = 0; i < lru_size; i++) {
+            snprintf(key, sizeof(key), "%s.lru%d", prefix, i);
+            ret = cli_xml_output_vol_status_inode_entry(writer, dict, key);
+            if (ret)
                 goto out;
-        ret = xmlTextWriterWriteFormatElement (writer, (xmlChar *)"lruSize",
-                                               "%"PRIu32, lru_size);
-        XML_RET_CHECK_AND_GOTO (ret, out);
-        if (lru_size != 0) {
-                /* <lru> */
-                ret = xmlTextWriterStartElement (writer, (xmlChar *)"lru");
-                XML_RET_CHECK_AND_GOTO (ret, out);
-
-                for (i = 0; i < lru_size; i++) {
-                        memset (key, 0, sizeof (key));
-                        snprintf (key, sizeof (key), "%s.lru%d", prefix, i);
-                        ret = cli_xml_output_vol_status_inode_entry
-                                (writer, dict, key);
-                        if (ret)
-                                goto out;
-                }
-                /* </lru> */
-                ret = xmlTextWriterEndElement (writer);
-                XML_RET_CHECK_AND_GOTO (ret, out);
         }
+        /* </lru> */
+        ret = xmlTextWriterEndElement(writer);
+        XML_RET_CHECK_AND_GOTO(ret, out);
+    }
+
+    snprintf(key, sizeof(key), "%s.purge_size", prefix);
+    ret = dict_get_uint32(dict, key, &purge_size);
+    if (ret)
+        goto out;
+    ret = xmlTextWriterWriteFormatElement(writer, (xmlChar *)"purgeSize",
+                                          "%" PRIu32, purge_size);
+    XML_RET_CHECK_AND_GOTO(ret, out);
+    if (purge_size != 0) {
+        /* <purge> */
+        ret = xmlTextWriterStartElement(writer, (xmlChar *)"purge");
+        XML_RET_CHECK_AND_GOTO(ret, out);
 
-        memset (key, 0, sizeof (key));
-        snprintf (key, sizeof (key), "%s.purge_size", prefix);
-        ret = dict_get_uint32 (dict, key, &purge_size);
-        if (ret)
+        for (i = 0; i < purge_size; i++) {
+            snprintf(key, sizeof(key), "%s.purge%d", prefix, i);
+            ret = cli_xml_output_vol_status_inode_entry(writer, dict, key);
+            if (ret)
                 goto out;
-        ret = xmlTextWriterWriteFormatElement (writer, (xmlChar *)"purgeSize",
-                                               "%"PRIu32, purge_size);
-        XML_RET_CHECK_AND_GOTO (ret, out);
-        if (purge_size != 0) {
-                /* <purge> */
-                ret = xmlTextWriterStartElement (writer, (xmlChar *)"purge");
-                XML_RET_CHECK_AND_GOTO (ret, out);
-
-                for (i = 0; i < purge_size; i++) {
-                        memset (key, 0, sizeof (key));
-                        snprintf (key, sizeof (key), "%s.purge%d", prefix, i);
-                        ret = cli_xml_output_vol_status_inode_entry
-                                (writer, dict, key);
-                        if (ret)
-                                goto out;
-                }
-                /* </purge> */
-                ret = xmlTextWriterEndElement (writer);
-                XML_RET_CHECK_AND_GOTO (ret, out);
         }
+        /* </purge> */
+        ret = xmlTextWriterEndElement(writer);
+        XML_RET_CHECK_AND_GOTO(ret, out);
+    }
 
 out:
-        gf_log ("cli", GF_LOG_DEBUG, "Returning %d", ret);
-        return ret;
+    gf_log("cli", GF_LOG_DEBUG, "Returning %d", ret);
+    return ret;
 }
 
 int
-cli_xml_output_vol_status_inode (xmlTextWriterPtr writer, dict_t *dict,
-                                 int brick_index)
+cli_xml_output_vol_status_inode(xmlTextWriterPtr writer, dict_t *dict,
+                                int brick_index)
 {
-        int             ret = -1;
-        int             conn_count = 0;
-        char            key[1024] = {0,};
-        int             i = 0;
-
-        /* <inodeStatus> */
-        ret = xmlTextWriterStartElement (writer, (xmlChar *)"inodeStatus");
-        XML_RET_CHECK_AND_GOTO (ret, out);
+    int ret = -1;
+    int conn_count = 0;
+    char key[1024] = {
+        0,
+    };
+    int i = 0;
+
+    /* <inodeStatus> */
+    ret = xmlTextWriterStartElement(writer, (xmlChar *)"inodeStatus");
+    XML_RET_CHECK_AND_GOTO(ret, out);
+
+    snprintf(key, sizeof(key), "brick%d.conncount", brick_index);
+    ret = dict_get_int32(dict, key, &conn_count);
+    if (ret)
+        goto out;
+    ret = xmlTextWriterWriteFormatElement(writer, (xmlChar *)"connections",
+                                          "%d", conn_count);
+    XML_RET_CHECK_AND_GOTO(ret, out);
+
+    for (i = 0; i < conn_count; i++) {
+        /* <connection> */
+        ret = xmlTextWriterStartElement(writer, (xmlChar *)"connection");
+        XML_RET_CHECK_AND_GOTO(ret, out);
 
-        snprintf (key, sizeof (key), "brick%d.conncount", brick_index);
-        ret = dict_get_int32 (dict, key, &conn_count);
+        snprintf(key, sizeof(key), "brick%d.conn%d.itable", brick_index, i);
+        ret = cli_xml_output_vol_status_itable(writer, dict, key);
         if (ret)
-                goto out;
-        ret = xmlTextWriterWriteFormatElement (writer, (xmlChar *)"connections",
-                                               "%d", conn_count);
-        XML_RET_CHECK_AND_GOTO (ret, out);
-
-        for (i = 0; i < conn_count; i++) {
-                /* <connection> */
-                ret = xmlTextWriterStartElement (writer,
-                                                 (xmlChar *)"connection");
-                XML_RET_CHECK_AND_GOTO (ret, out);
-
-                memset (key, 0, sizeof (key));
-                snprintf (key, sizeof (key), "brick%d.conn%d.itable",
-                          brick_index, i);
-                ret = cli_xml_output_vol_status_itable (writer, dict, key);
-                if (ret)
-                        goto out;
+            goto out;
 
-                /* </connection> */
-                ret = xmlTextWriterEndElement (writer);
-                XML_RET_CHECK_AND_GOTO (ret, out);
-        }
+        /* </connection> */
+        ret = xmlTextWriterEndElement(writer);
+        XML_RET_CHECK_AND_GOTO(ret, out);
+    }
 
-        /* </inodeStatus> */
-        ret= xmlTextWriterEndElement (writer);
-        XML_RET_CHECK_AND_GOTO (ret, out);
+    /* </inodeStatus> */
+    ret = xmlTextWriterEndElement(writer);
+    XML_RET_CHECK_AND_GOTO(ret, out);
 
 out:
-        gf_log ("cli", GF_LOG_DEBUG, "Returning %d", ret);
-        return ret;
+    gf_log("cli", GF_LOG_DEBUG, "Returning %d", ret);
+    return ret;
 }
 
 int
-cli_xml_output_vol_status_fdtable (xmlTextWriterPtr writer, dict_t *dict,
-                                   char *prefix)
+cli_xml_output_vol_status_fdtable(xmlTextWriterPtr writer, dict_t *dict,
+                                  char *prefix)
 {
-        int             ret = -1;
-        int             refcount = 0;
-        uint32_t        maxfds = 0;
-        int             firstfree = 0;
-        int             openfds = 0;
-        int             fd_pid = 0;
-        int             fd_refcount = 0;
-        int             fd_flags = 0;
-        char            key[1024] = {0,};
-        int             i = 0;
-
-        /* <fdTable> */
-        ret = xmlTextWriterStartElement (writer, (xmlChar *)"fdTable");
-        XML_RET_CHECK_AND_GOTO (ret, out);
-
-        snprintf (key, sizeof (key), "%s.refcount", prefix);
-        ret = dict_get_int32 (dict, key, &refcount);
+    int ret = -1;
+    int refcount = 0;
+    uint32_t maxfds = 0;
+    int firstfree = 0;
+    int openfds = 0;
+    int fd_pid = 0;
+    int fd_refcount = 0;
+    int fd_flags = 0;
+    char key[1024] = {
+        0,
+    };
+    int i = 0;
+
+    /* <fdTable> */
+    ret = xmlTextWriterStartElement(writer, (xmlChar *)"fdTable");
+    XML_RET_CHECK_AND_GOTO(ret, out);
+
+    snprintf(key, sizeof(key), "%s.refcount", prefix);
+    ret = dict_get_int32(dict, key, &refcount);
+    if (ret)
+        goto out;
+    ret = xmlTextWriterWriteFormatElement(writer, (xmlChar *)"refCount", "%d",
+                                          refcount);
+    XML_RET_CHECK_AND_GOTO(ret, out);
+
+    snprintf(key, sizeof(key), "%s.maxfds", prefix);
+    ret = dict_get_uint32(dict, key, &maxfds);
+    if (ret)
+        goto out;
+    ret = xmlTextWriterWriteFormatElement(writer, (xmlChar *)"maxFds",
+                                          "%" PRIu32, maxfds);
+    XML_RET_CHECK_AND_GOTO(ret, out);
+
+    snprintf(key, sizeof(key), "%s.firstfree", prefix);
+    ret = dict_get_int32(dict, key, &firstfree);
+    if (ret)
+        goto out;
+    ret = xmlTextWriterWriteFormatElement(writer, (xmlChar *)"firstFree", "%d",
+                                          firstfree);
+    XML_RET_CHECK_AND_GOTO(ret, out);
+
+    snprintf(key, sizeof(key), "%s.openfds", prefix);
+    ret = dict_get_int32(dict, key, &openfds);
+    if (ret)
+        goto out;
+    ret = xmlTextWriterWriteFormatElement(writer, (xmlChar *)"openFds", "%d",
+                                          openfds);
+    XML_RET_CHECK_AND_GOTO(ret, out);
+
+    for (i = 0; i < maxfds; i++) {
+        snprintf(key, sizeof(key), "%s.fdentry%d.pid", prefix, i);
+        ret = dict_get_int32(dict, key, &fd_pid);
         if (ret)
-                goto out;
-        ret = xmlTextWriterWriteFormatElement (writer, (xmlChar *)"refCount",
-                                               "%d", refcount);
-        XML_RET_CHECK_AND_GOTO (ret, out);
+            continue;
 
-        memset (key, 0, sizeof (key));
-        snprintf (key, sizeof (key), "%s.maxfds", prefix);
-        ret = dict_get_uint32 (dict, key, &maxfds);
+        snprintf(key, sizeof(key), "%s.fdentry%d.refcount", prefix, i);
+        ret = dict_get_int32(dict, key, &fd_refcount);
         if (ret)
-                goto out;
-        ret = xmlTextWriterWriteFormatElement (writer, (xmlChar *)"maxFds",
-                                               "%"PRIu32, maxfds);
-        XML_RET_CHECK_AND_GOTO (ret, out);
+            continue;
 
-        memset (key, 0, sizeof (key));
-        snprintf (key, sizeof (key), "%s.firstfree", prefix);
-        ret = dict_get_int32 (dict, key, &firstfree);
+        snprintf(key, sizeof(key), "%s.fdentry%d.flags", prefix, i);
+        ret = dict_get_int32(dict, key, &fd_flags);
         if (ret)
-                goto out;
-        ret = xmlTextWriterWriteFormatElement (writer, (xmlChar *)"firstFree",
-                                               "%d", firstfree);
-        XML_RET_CHECK_AND_GOTO (ret, out);
+            continue;
 
-        memset (key, 0, sizeof (key));
-        snprintf (key, sizeof (key), "%s.openfds", prefix);
-        ret = dict_get_int32 (dict, key, &openfds);
-        if (ret)
-                goto out;
-        ret = xmlTextWriterWriteFormatElement (writer, (xmlChar *)"openFds",
-                                               "%d", openfds);
-        XML_RET_CHECK_AND_GOTO (ret, out);
-
-        for (i = 0; i < maxfds; i++) {
-                memset (key, 0, sizeof (key));
-                snprintf (key, sizeof (key), "%s.fdentry%d.pid", prefix, i);
-                ret = dict_get_int32 (dict, key, &fd_pid);
-                if (ret)
-                        continue;
+        /* <fd> */
+        ret = xmlTextWriterStartElement(writer, (xmlChar *)"fd");
+        XML_RET_CHECK_AND_GOTO(ret, out);
 
-                memset (key, 0, sizeof (key));
-                snprintf (key, sizeof (key), "%s.fdentry%d.refcount",
-                          prefix, i);
-                ret = dict_get_int32 (dict, key, &fd_refcount);
-                if (ret)
-                        continue;
+        ret = xmlTextWriterWriteFormatElement(writer, (xmlChar *)"entry", "%d",
+                                              i + 1);
+        XML_RET_CHECK_AND_GOTO(ret, out);
 
-                memset (key, 0, sizeof (key));
-                snprintf (key, sizeof (key), "%s.fdentry%d.flags", prefix, i);
-                ret = dict_get_int32 (dict, key, &fd_flags);
-                if (ret)
-                        continue;
-
-                /* <fd> */
-                ret = xmlTextWriterStartElement (writer, (xmlChar *)"fd");
-                XML_RET_CHECK_AND_GOTO (ret, out);
-
-                ret = xmlTextWriterWriteFormatElement (writer,
-                                                       (xmlChar *)"entry",
-                                                       "%d", i+1);
-                XML_RET_CHECK_AND_GOTO (ret, out);
-
-                ret = xmlTextWriterWriteFormatElement (writer,
-                                                       (xmlChar *)"pid",
-                                                       "%d", fd_pid);
-                XML_RET_CHECK_AND_GOTO (ret, out);
-
-                ret = xmlTextWriterWriteFormatElement (writer,
-                                                       (xmlChar *)"refCount",
-                                                       "%d", fd_refcount);
-                XML_RET_CHECK_AND_GOTO (ret, out);
-
-                ret = xmlTextWriterWriteFormatElement (writer,
-                                                       (xmlChar *)"flags",
-                                                       "%d", fd_flags);
-                XML_RET_CHECK_AND_GOTO (ret, out);
-
-                /* </fd> */
-                ret = xmlTextWriterEndElement (writer);
-                XML_RET_CHECK_AND_GOTO (ret, out);
-        }
+        ret = xmlTextWriterWriteFormatElement(writer, (xmlChar *)"pid", "%d",
+                                              fd_pid);
+        XML_RET_CHECK_AND_GOTO(ret, out);
 
-        /* </fdTable> */
-        ret = xmlTextWriterEndElement (writer);
-        XML_RET_CHECK_AND_GOTO (ret, out);
+        ret = xmlTextWriterWriteFormatElement(writer, (xmlChar *)"refCount",
+                                              "%d", fd_refcount);
+        XML_RET_CHECK_AND_GOTO(ret, out);
+
+        ret = xmlTextWriterWriteFormatElement(writer, (xmlChar *)"flags", "%d",
+                                              fd_flags);
+        XML_RET_CHECK_AND_GOTO(ret, out);
+
+        /* </fd> */
+        ret = xmlTextWriterEndElement(writer);
+        XML_RET_CHECK_AND_GOTO(ret, out);
+    }
+
+    /* </fdTable> */
+    ret = xmlTextWriterEndElement(writer);
+    XML_RET_CHECK_AND_GOTO(ret, out);
 
 out:
-        gf_log ("cli", GF_LOG_DEBUG, "Returning %d", ret);
-        return ret;
+    gf_log("cli", GF_LOG_DEBUG, "Returning %d", ret);
+    return ret;
 }
 
 int
-cli_xml_output_vol_status_fd (xmlTextWriterPtr writer, dict_t *dict,
-                              int brick_index)
+cli_xml_output_vol_status_fd(xmlTextWriterPtr writer, dict_t *dict,
+                             int brick_index)
 {
-        int             ret = -1;
-        int             conn_count = 0;
-        char            key[1024] = {0,};
-        int             i = 0;
-
-        /* <fdStatus> */
-        ret = xmlTextWriterStartElement (writer, (xmlChar *)"fdStatus");
-        XML_RET_CHECK_AND_GOTO (ret, out);
+    int ret = -1;
+    int conn_count = 0;
+    char key[1024] = {
+        0,
+    };
+    int i = 0;
+
+    /* <fdStatus> */
+    ret = xmlTextWriterStartElement(writer, (xmlChar *)"fdStatus");
+    XML_RET_CHECK_AND_GOTO(ret, out);
+
+    snprintf(key, sizeof(key), "brick%d.conncount", brick_index);
+    ret = dict_get_int32(dict, key, &conn_count);
+    if (ret)
+        goto out;
+    ret = xmlTextWriterWriteFormatElement(writer, (xmlChar *)"connections",
+                                          "%d", conn_count);
+    XML_RET_CHECK_AND_GOTO(ret, out);
+
+    for (i = 0; i < conn_count; i++) {
+        /* <connection> */
+        ret = xmlTextWriterStartElement(writer, (xmlChar *)"connection");
+        XML_RET_CHECK_AND_GOTO(ret, out);
 
-        snprintf (key, sizeof (key), "brick%d.conncount", brick_index);
-        ret = dict_get_int32 (dict, key, &conn_count);
+        snprintf(key, sizeof(key), "brick%d.conn%d.fdtable", brick_index, i);
+        ret = cli_xml_output_vol_status_fdtable(writer, dict, key);
         if (ret)
-                goto out;
-        ret = xmlTextWriterWriteFormatElement (writer, (xmlChar *)"connections",
-                                               "%d", conn_count);
-        XML_RET_CHECK_AND_GOTO (ret, out);
-
-        for (i = 0; i < conn_count; i++) {
-                /* <connection> */
-                ret = xmlTextWriterStartElement (writer,
-                                                 (xmlChar *)"connection");
-                XML_RET_CHECK_AND_GOTO (ret, out);
-
-                memset (key, 0, sizeof (key));
-                snprintf (key, sizeof (key), "brick%d.conn%d.fdtable",
-                          brick_index, i);
-                ret = cli_xml_output_vol_status_fdtable (writer, dict, key);
-                if (ret)
-                        goto out;
+            goto out;
 
-                /* </connection> */
-                ret = xmlTextWriterEndElement (writer);
-                XML_RET_CHECK_AND_GOTO (ret, out);
-        }
+        /* </connection> */
+        ret = xmlTextWriterEndElement(writer);
+        XML_RET_CHECK_AND_GOTO(ret, out);
+    }
 
-        /* </fdStatus> */
-        ret = xmlTextWriterEndElement (writer);
-        XML_RET_CHECK_AND_GOTO (ret, out);
+    /* </fdStatus> */
+    ret = xmlTextWriterEndElement(writer);
+    XML_RET_CHECK_AND_GOTO(ret, out);
 
 out:
-        gf_log ("cli", GF_LOG_DEBUG, "Returning %d", ret);
-        return ret;
+    gf_log("cli", GF_LOG_DEBUG, "Returning %d", ret);
+    return ret;
 }
 
 int
-cli_xml_output_vol_status_callframe (xmlTextWriterPtr writer, dict_t *dict,
-                                     char *prefix)
+cli_xml_output_vol_status_callframe(xmlTextWriterPtr writer, dict_t *dict,
+                                    char *prefix)
 {
-        int             ret = -1;
-        int             ref_count = 0;
-        char            *translator = NULL;
-        int             complete = 0;
-        char            *parent = NULL;
-        char            *wind_from = NULL;
-        char            *wind_to = NULL;
-        char            *unwind_from = NULL;
-        char            *unwind_to = NULL;
-        char            key[1024] = {0,};
-
-        /* <callFrame> */
-        ret = xmlTextWriterStartElement (writer, (xmlChar *)"callFrame");
-        XML_RET_CHECK_AND_GOTO (ret, out);
-
-        snprintf (key, sizeof (key), "%s.refcount", prefix);
-        ret = dict_get_int32 (dict, key, &ref_count);
-        if (ret)
-                goto out;
-        ret = xmlTextWriterWriteFormatElement (writer, (xmlChar *)"refCount",
-                                               "%d", ref_count);
-        XML_RET_CHECK_AND_GOTO (ret, out);
-
-        memset (key, 0, sizeof (key));
-        snprintf (key, sizeof (key), "%s.translator", prefix);
-        ret = dict_get_str (dict, key, &translator);
-        if (ret)
-                goto out;
-        ret = xmlTextWriterWriteFormatElement (writer, (xmlChar *)"translator",
-                                               "%s", translator);
-        XML_RET_CHECK_AND_GOTO (ret, out);
-
-        memset (key, 0, sizeof (key));
-        snprintf (key, sizeof (key), "%s.complete", prefix);
-        ret = dict_get_int32 (dict, key, &complete);
-        if (ret)
-                goto out;
-        ret = xmlTextWriterWriteFormatElement (writer, (xmlChar *)"complete",
-                                               "%d", complete);
-        XML_RET_CHECK_AND_GOTO (ret ,out);
-
-        memset (key, 0, sizeof (key));
-        snprintf (key, sizeof (key), "%s.parent", prefix);
-        ret = dict_get_str (dict, key, &parent);
-        if (!ret) {
-                ret = xmlTextWriterWriteFormatElement (writer,
-                                                       (xmlChar *)"parent",
-                                                       "%s", parent);
-                XML_RET_CHECK_AND_GOTO (ret, out);
-        }
+    int ret = -1;
+    int ref_count = 0;
+    char *translator = NULL;
+    int complete = 0;
+    char *parent = NULL;
+    char *wind_from = NULL;
+    char *wind_to = NULL;
+    char *unwind_from = NULL;
+    char *unwind_to = NULL;
+    char key[1024] = {
+        0,
+    };
+
+    /* <callFrame> */
+    ret = xmlTextWriterStartElement(writer, (xmlChar *)"callFrame");
+    XML_RET_CHECK_AND_GOTO(ret, out);
+
+    snprintf(key, sizeof(key), "%s.refcount", prefix);
+    ret = dict_get_int32(dict, key, &ref_count);
+    if (ret)
+        goto out;
+    ret = xmlTextWriterWriteFormatElement(writer, (xmlChar *)"refCount", "%d",
+                                          ref_count);
+    XML_RET_CHECK_AND_GOTO(ret, out);
+
+    snprintf(key, sizeof(key), "%s.translator", prefix);
+    ret = dict_get_str(dict, key, &translator);
+    if (ret)
+        goto out;
+    ret = xmlTextWriterWriteFormatElement(writer, (xmlChar *)"translator", "%s",
+                                          translator);
+    XML_RET_CHECK_AND_GOTO(ret, out);
+
+    snprintf(key, sizeof(key), "%s.complete", prefix);
+    ret = dict_get_int32(dict, key, &complete);
+    if (ret)
+        goto out;
+    ret = xmlTextWriterWriteFormatElement(writer, (xmlChar *)"complete", "%d",
+                                          complete);
+    XML_RET_CHECK_AND_GOTO(ret, out);
+
+    snprintf(key, sizeof(key), "%s.parent", prefix);
+    ret = dict_get_str(dict, key, &parent);
+    if (!ret) {
+        ret = xmlTextWriterWriteFormatElement(writer, (xmlChar *)"parent", "%s",
+                                              parent);
+        XML_RET_CHECK_AND_GOTO(ret, out);
+    }
 
-        memset (key, 0, sizeof (key));
-        snprintf (key, sizeof (key), "%s.windfrom", prefix);
-        ret = dict_get_str (dict, key, &wind_from);
-        if (!ret) {
-                ret = xmlTextWriterWriteFormatElement (writer,
-                                                       (xmlChar *)"windFrom",
-                                                       "%s", wind_from);
-                XML_RET_CHECK_AND_GOTO (ret, out);
-        }
+    snprintf(key, sizeof(key), "%s.windfrom", prefix);
+    ret = dict_get_str(dict, key, &wind_from);
+    if (!ret) {
+        ret = xmlTextWriterWriteFormatElement(writer, (xmlChar *)"windFrom",
+                                              "%s", wind_from);
+        XML_RET_CHECK_AND_GOTO(ret, out);
+    }
 
-        memset (key, 0, sizeof (key));
-        snprintf (key, sizeof (key), "%s.windto", prefix);
-        ret = dict_get_str (dict, key, &wind_to);
-        if (!ret) {
-                ret = xmlTextWriterWriteFormatElement (writer,
-                                                       (xmlChar *)"windTo",
-                                                       "%s", wind_to);
-                XML_RET_CHECK_AND_GOTO (ret, out);
-        }
+    snprintf(key, sizeof(key), "%s.windto", prefix);
+    ret = dict_get_str(dict, key, &wind_to);
+    if (!ret) {
+        ret = xmlTextWriterWriteFormatElement(writer, (xmlChar *)"windTo", "%s",
+                                              wind_to);
+        XML_RET_CHECK_AND_GOTO(ret, out);
+    }
 
-        memset (key, 0, sizeof (key));
-        snprintf (key, sizeof (key), "%s.unwindfrom", prefix);
-        ret = dict_get_str (dict, key, &unwind_from);
-        if (!ret) {
-                ret = xmlTextWriterWriteFormatElement (writer,
-                                                       (xmlChar *)"unwindFrom",
-                                                       "%s", unwind_from);
-                XML_RET_CHECK_AND_GOTO (ret, out);
-        }
+    snprintf(key, sizeof(key), "%s.unwindfrom", prefix);
+    ret = dict_get_str(dict, key, &unwind_from);
+    if (!ret) {
+        ret = xmlTextWriterWriteFormatElement(writer, (xmlChar *)"unwindFrom",
+                                              "%s", unwind_from);
+        XML_RET_CHECK_AND_GOTO(ret, out);
+    }
 
-        memset (key, 0, sizeof (key));
-        snprintf (key, sizeof (key), "%s.unwindto", prefix);
-        ret = dict_get_str (dict, key, &unwind_to);
-        if (!ret) {
-                ret = xmlTextWriterWriteFormatElement (writer,
-                                                       (xmlChar *)"unwindTo",
-                                                       "%s", unwind_to);
-                XML_RET_CHECK_AND_GOTO (ret, out);
-        }
+    snprintf(key, sizeof(key), "%s.unwindto", prefix);
+    ret = dict_get_str(dict, key, &unwind_to);
+    if (!ret) {
+        ret = xmlTextWriterWriteFormatElement(writer, (xmlChar *)"unwindTo",
+                                              "%s", unwind_to);
+        XML_RET_CHECK_AND_GOTO(ret, out);
+    }
 
-        /* </callFrame> */
-        ret = xmlTextWriterEndElement (writer);
-        XML_RET_CHECK_AND_GOTO (ret, out);
+    /* </callFrame> */
+    ret = xmlTextWriterEndElement(writer);
+    XML_RET_CHECK_AND_GOTO(ret, out);
 out:
-        gf_log ("cli", GF_LOG_DEBUG, "Returning %d", ret);
-        return ret;
+    gf_log("cli", GF_LOG_DEBUG, "Returning %d", ret);
+    return ret;
 }
 
 int
-cli_xml_output_vol_status_callstack (xmlTextWriterPtr writer, dict_t *dict,
-                                     char *prefix)
+cli_xml_output_vol_status_callstack(xmlTextWriterPtr writer, dict_t *dict,
+                                    char *prefix)
 {
-        int             ret = -1;
-        int             uid = 0;
-        int             gid = 0;
-        int             pid = 0;
-        uint64_t        unique = 0;
-        int             frame_count = 0;
-        char            key[1024] = {0,};
-        int             i = 0;
-
-        /* <callStack> */
-        ret = xmlTextWriterStartElement (writer, (xmlChar *)"callStack");
-        XML_RET_CHECK_AND_GOTO (ret, out);
-
-        snprintf (key, sizeof (key), "%s.uid", prefix);
-        ret = dict_get_int32 (dict, key, &uid);
-        if (ret)
-                goto out;
-        ret = xmlTextWriterWriteFormatElement (writer, (xmlChar *)"uid",
-                                               "%d", uid);
-        XML_RET_CHECK_AND_GOTO (ret, out);
-
-        memset (key, 0, sizeof (key));
-        snprintf (key, sizeof (key), "%s.gid", prefix);
-        ret = dict_get_int32 (dict, key, &gid);
-        if (ret)
-                goto out;
-        ret = xmlTextWriterWriteFormatElement (writer, (xmlChar *)"gid",
-                                               "%d", gid);
-        XML_RET_CHECK_AND_GOTO (ret, out);
-
-        memset (key, 0, sizeof (key));
-        snprintf (key, sizeof (key), "%s.pid", prefix);
-        ret = dict_get_int32 (dict, key, &pid);
-        if (ret)
-                goto out;
-        ret = xmlTextWriterWriteFormatElement (writer, (xmlChar *)"pid",
-                                               "%d", pid);
-        XML_RET_CHECK_AND_GOTO (ret, out);
-
-        memset (key, 0, sizeof (key));
-        snprintf (key, sizeof (key), "%s.unique", prefix);
-        ret = dict_get_uint64 (dict, key, &unique);
+    int ret = -1;
+    int uid = 0;
+    int gid = 0;
+    int pid = 0;
+    uint64_t unique = 0;
+    int frame_count = 0;
+    char key[1024] = {
+        0,
+    };
+    int i = 0;
+
+    /* <callStack> */
+    ret = xmlTextWriterStartElement(writer, (xmlChar *)"callStack");
+    XML_RET_CHECK_AND_GOTO(ret, out);
+
+    snprintf(key, sizeof(key), "%s.uid", prefix);
+    ret = dict_get_int32(dict, key, &uid);
+    if (ret)
+        goto out;
+    ret = xmlTextWriterWriteFormatElement(writer, (xmlChar *)"uid", "%d", uid);
+    XML_RET_CHECK_AND_GOTO(ret, out);
+
+    snprintf(key, sizeof(key), "%s.gid", prefix);
+    ret = dict_get_int32(dict, key, &gid);
+    if (ret)
+        goto out;
+    ret = xmlTextWriterWriteFormatElement(writer, (xmlChar *)"gid", "%d", gid);
+    XML_RET_CHECK_AND_GOTO(ret, out);
+
+    snprintf(key, sizeof(key), "%s.pid", prefix);
+    ret = dict_get_int32(dict, key, &pid);
+    if (ret)
+        goto out;
+    ret = xmlTextWriterWriteFormatElement(writer, (xmlChar *)"pid", "%d", pid);
+    XML_RET_CHECK_AND_GOTO(ret, out);
+
+    snprintf(key, sizeof(key), "%s.unique", prefix);
+    ret = dict_get_uint64(dict, key, &unique);
+    if (ret)
+        goto out;
+    ret = xmlTextWriterWriteFormatElement(writer, (xmlChar *)"unique",
+                                          "%" PRIu64, unique);
+    XML_RET_CHECK_AND_GOTO(ret, out);
+
+    snprintf(key, sizeof(key), "%s.count", prefix);
+    ret = dict_get_int32(dict, key, &frame_count);
+    if (ret)
+        goto out;
+    ret = xmlTextWriterWriteFormatElement(writer, (xmlChar *)"frameCount", "%d",
+                                          frame_count);
+    XML_RET_CHECK_AND_GOTO(ret, out);
+
+    for (i = 0; i < frame_count; i++) {
+        snprintf(key, sizeof(key), "%s.frame%d", prefix, i);
+        ret = cli_xml_output_vol_status_callframe(writer, dict, key);
         if (ret)
-                goto out;
-        ret = xmlTextWriterWriteFormatElement (writer, (xmlChar *)"unique",
-                                               "%"PRIu64, unique);
-        XML_RET_CHECK_AND_GOTO (ret, out);
-
-        memset (key, 0, sizeof (key));
-        snprintf (key, sizeof (key), "%s.count", prefix);
-        ret = dict_get_int32 (dict, key, &frame_count);
-        if (ret)
-                goto out;
-        ret = xmlTextWriterWriteFormatElement (writer, (xmlChar *)"frameCount",
-                                               "%d", frame_count);
-        XML_RET_CHECK_AND_GOTO (ret, out);
-
-        for (i = 0; i < frame_count; i++) {
-                memset (key, 0, sizeof (key));
-                snprintf (key, sizeof (key), "%s.frame%d", prefix, i);
-                ret = cli_xml_output_vol_status_callframe (writer, dict,
-                                                           key);
-                if (ret)
-                        goto out;
-        }
+            goto out;
+    }
 
-        /* </callStack> */
-        ret = xmlTextWriterEndElement (writer);
-        XML_RET_CHECK_AND_GOTO (ret, out);
+    /* </callStack> */
+    ret = xmlTextWriterEndElement(writer);
+    XML_RET_CHECK_AND_GOTO(ret, out);
 out:
-        gf_log ("cli", GF_LOG_DEBUG, "Returning %d", ret);
-        return ret;
+    gf_log("cli", GF_LOG_DEBUG, "Returning %d", ret);
+    return ret;
 }
 
 int
-cli_xml_output_vol_status_callpool (xmlTextWriterPtr writer, dict_t *dict,
-                                    int brick_index)
+cli_xml_output_vol_status_callpool(xmlTextWriterPtr writer, dict_t *dict,
+                                   int brick_index)
 {
-        int             ret = -1;
-        int             call_count = 0;
-        char            key[1024] = {0,};
-        int             i = 0;
-
-        /* <callpoolStatus> */
-        ret = xmlTextWriterStartElement (writer, (xmlChar *)"callpoolStatus");
-        XML_RET_CHECK_AND_GOTO (ret, out);
-
-        snprintf (key, sizeof (key), "brick%d.callpool.count", brick_index);
-        ret = dict_get_int32 (dict, key, &call_count);
+    int ret = -1;
+    int call_count = 0;
+    char key[1024] = {
+        0,
+    };
+    int i = 0;
+
+    /* <callpoolStatus> */
+    ret = xmlTextWriterStartElement(writer, (xmlChar *)"callpoolStatus");
+    XML_RET_CHECK_AND_GOTO(ret, out);
+
+    snprintf(key, sizeof(key), "brick%d.callpool.count", brick_index);
+    ret = dict_get_int32(dict, key, &call_count);
+    if (ret)
+        goto out;
+    ret = xmlTextWriterWriteFormatElement(writer, (xmlChar *)"count", "%d",
+                                          call_count);
+    XML_RET_CHECK_AND_GOTO(ret, out);
+
+    for (i = 0; i < call_count; i++) {
+        snprintf(key, sizeof(key), "brick%d.callpool.stack%d", brick_index, i);
+        ret = cli_xml_output_vol_status_callstack(writer, dict, key);
         if (ret)
-                goto out;
-        ret = xmlTextWriterWriteFormatElement (writer, (xmlChar *)"count",
-                                               "%d", call_count);
-
-        for (i = 0; i < call_count; i++) {
-                memset (key, 0, sizeof (key));
-                snprintf (key, sizeof (key), "brick%d.callpool.stack%d",
-                          brick_index, i);
-                ret = cli_xml_output_vol_status_callstack (writer, dict,
-                                                           key);
-                if (ret)
-                        goto out;
-        }
+            goto out;
+    }
 
-        /* </callpoolStatus> */
-        ret = xmlTextWriterEndElement (writer);
-        XML_RET_CHECK_AND_GOTO (ret, out);
+    /* </callpoolStatus> */
+    ret = xmlTextWriterEndElement(writer);
+    XML_RET_CHECK_AND_GOTO(ret, out);
 
 out:
-        gf_log ("cli", GF_LOG_DEBUG, "Returning %d", ret);
-        return ret;
+    gf_log("cli", GF_LOG_DEBUG, "Returning %d", ret);
+    return ret;
 }
 #endif
 
 int
-cli_xml_output_vol_status_begin (cli_local_t *local, int op_ret, int op_errno,
-                                 char *op_errstr)
+cli_xml_output_vol_status_begin(cli_local_t *local, int op_ret, int op_errno,
+                                char *op_errstr)
 {
 #if (HAVE_LIB_XML)
-        int                     ret = -1;
+    int ret = -1;
 
-        ret = cli_begin_xml_output (&(local->writer), &(local->doc));
-        XML_RET_CHECK_AND_GOTO (ret, out);
+    ret = cli_begin_xml_output(&(local->writer), &(local->doc));
+    XML_RET_CHECK_AND_GOTO(ret, out);
 
-        ret = cli_xml_output_common (local->writer, op_ret, op_errno,
-                                     op_errstr);
-        XML_RET_CHECK_AND_GOTO (ret, out);
+    ret = cli_xml_output_common(local->writer, op_ret, op_errno, op_errstr);
+    XML_RET_CHECK_AND_GOTO(ret, out);
 
-        /* <volStatus> */
-        ret = xmlTextWriterStartElement (local->writer,
-                                         (xmlChar *) "volStatus");
-        XML_RET_CHECK_AND_GOTO (ret, out);
+    /* <volStatus> */
+    ret = xmlTextWriterStartElement(local->writer, (xmlChar *)"volStatus");
+    XML_RET_CHECK_AND_GOTO(ret, out);
 
-        /* <volumes> */
-        ret = xmlTextWriterStartElement (local->writer, (xmlChar *)"volumes");
-        XML_RET_CHECK_AND_GOTO (ret, out);
+    /* <volumes> */
+    ret = xmlTextWriterStartElement(local->writer, (xmlChar *)"volumes");
+    XML_RET_CHECK_AND_GOTO(ret, out);
 
 out:
-        gf_log ("cli", GF_LOG_DEBUG, "Returning %d", ret);
-        return ret;
+    gf_log("cli", GF_LOG_DEBUG, "Returning %d", ret);
+    return ret;
 #else
-        return 0;
+    return 0;
 #endif
 }
 
 int
-cli_xml_output_vol_status_end (cli_local_t *local)
+cli_xml_output_vol_status_end(cli_local_t *local)
 {
 #if (HAVE_LIB_XML)
-        int     ret = -1;
+    int ret = -1;
 
-        /* </volumes> */
-        ret = xmlTextWriterEndElement (local->writer);
-        XML_RET_CHECK_AND_GOTO (ret, out);
+    /* </volumes> */
+    ret = xmlTextWriterEndElement(local->writer);
+    XML_RET_CHECK_AND_GOTO(ret, out);
 
-        /* </volStatus> */
-        ret = xmlTextWriterEndElement (local->writer);
-        XML_RET_CHECK_AND_GOTO(ret, out);
+    /* </volStatus> */
+    ret = xmlTextWriterEndElement(local->writer);
+    XML_RET_CHECK_AND_GOTO(ret, out);
 
-        ret = cli_end_xml_output (local->writer, local->doc);
+    ret = cli_end_xml_output(local->writer, local->doc);
 out:
-        gf_log ("cli", GF_LOG_DEBUG, "Returning %d", ret);
-        return ret;
+    gf_log("cli", GF_LOG_DEBUG, "Returning %d", ret);
+    return ret;
 #else
-        return 0;
+    return 0;
 #endif
 }
 
 #if (HAVE_LIB_XML)
 int
-cli_xml_output_remove_brick_task_params (xmlTextWriterPtr writer, dict_t *dict,
-                                         char *prefix)
+cli_xml_output_remove_brick_task_params(xmlTextWriterPtr writer, dict_t *dict,
+                                        char *prefix)
 {
-        int             ret = -1;
-        char            key[1024] = {0,};
-        int             count = 0;
-        int             i = 0;
-        char            *brick = NULL;
-
-        /* <params> */
-        ret = xmlTextWriterStartElement (writer, (xmlChar *)"params");
-        XML_RET_CHECK_AND_GOTO (ret, out);
-
-        snprintf (key, sizeof (key), "%s.count", prefix);
-        ret = dict_get_int32 (dict, key, &count);
+    int ret = -1;
+    char key[1024] = {
+        0,
+    };
+    int count = 0;
+    int i = 0;
+    char *brick = NULL;
+
+    /* <params> */
+    ret = xmlTextWriterStartElement(writer, (xmlChar *)"params");
+    XML_RET_CHECK_AND_GOTO(ret, out);
+
+    snprintf(key, sizeof(key), "%s.count", prefix);
+    ret = dict_get_int32(dict, key, &count);
+    if (ret)
+        goto out;
+
+    for (i = 1; i <= count; i++) {
+        snprintf(key, sizeof(key), "%s.brick%d", prefix, i);
+        ret = dict_get_str(dict, key, &brick);
         if (ret)
-                goto out;
-
-        for (i = 1; i <= count; i++) {
-                memset (key, 0, sizeof (key));
-                snprintf (key, sizeof (key), "%s.brick%d", prefix, i);
-                ret = dict_get_str (dict, key, &brick);
-                if (ret)
-                        goto out;
-                ret = xmlTextWriterWriteFormatElement (writer,
-                                                       (xmlChar *)"brick",
-                                                       "%s", brick);
-                XML_RET_CHECK_AND_GOTO (ret, out);
-                brick = NULL;
-        }
-
-        /* </param> */
-        ret = xmlTextWriterEndElement (writer);
+            goto out;
+        ret = xmlTextWriterWriteFormatElement(writer, (xmlChar *)"brick", "%s",
+                                              brick);
+        XML_RET_CHECK_AND_GOTO(ret, out);
+        brick = NULL;
+    }
 
+    /* </param> */
+    ret = xmlTextWriterEndElement(writer);
+    XML_RET_CHECK_AND_GOTO(ret, out);
 out:
-        gf_log ("cli", GF_LOG_DEBUG, "Returning %d", ret);
-        return ret;
+    gf_log("cli", GF_LOG_DEBUG, "Returning %d", ret);
+    return ret;
 }
 
 int
-cli_xml_output_vol_status_tasks (cli_local_t *local, dict_t *dict) {
-        int                     ret = -1;
-        char                    *task_type = NULL;
-        char                    *task_id_str = NULL;
-        int                     status = 0;
-        int                     tasks = 0;
-        char                    key[1024] = {0,};
-        int                     i = 0;
-
-        /* <tasks> */
-        ret = xmlTextWriterStartElement (local->writer, (xmlChar *)"tasks");
-        XML_RET_CHECK_AND_GOTO (ret, out);
-
-        ret = dict_get_int32 (dict, "tasks", &tasks);
+cli_xml_output_vol_status_tasks(cli_local_t *local, dict_t *dict)
+{
+    int ret = -1;
+    char *task_type = NULL;
+    char *task_id_str = NULL;
+    int status = 0;
+    int tasks = 0;
+    char key[1024] = {
+        0,
+    };
+    int i = 0;
+
+    /* <tasks> */
+    ret = xmlTextWriterStartElement(local->writer, (xmlChar *)"tasks");
+    XML_RET_CHECK_AND_GOTO(ret, out);
+
+    ret = dict_get_int32(dict, "tasks", &tasks);
+    if (ret)
+        goto out;
+
+    for (i = 0; i < tasks; i++) {
+        /* <task> */
+        ret = xmlTextWriterStartElement(local->writer, (xmlChar *)"task");
+        XML_RET_CHECK_AND_GOTO(ret, out);
+
+        snprintf(key, sizeof(key), "task%d.type", i);
+        ret = dict_get_str(dict, key, &task_type);
         if (ret)
-                goto out;
+            goto out;
+        ret = xmlTextWriterWriteFormatElement(local->writer, (xmlChar *)"type",
+                                              "%s", task_type);
+        XML_RET_CHECK_AND_GOTO(ret, out);
+
+        snprintf(key, sizeof(key), "task%d.id", i);
+        ret = dict_get_str(dict, key, &task_id_str);
+        if (ret)
+            goto out;
+        ret = xmlTextWriterWriteFormatElement(local->writer, (xmlChar *)"id",
+                                              "%s", task_id_str);
+        XML_RET_CHECK_AND_GOTO(ret, out);
 
-        for (i = 0; i < tasks; i++) {
-                /* <task> */
-                ret = xmlTextWriterStartElement (local->writer,
-                                                 (xmlChar *)"task");
-                XML_RET_CHECK_AND_GOTO (ret, out);
+        snprintf(key, sizeof(key), "task%d.status", i);
+        ret = dict_get_int32(dict, key, &status);
+        if (ret)
+            goto out;
+        ret = xmlTextWriterWriteFormatElement(
+            local->writer, (xmlChar *)"status", "%d", status);
+        XML_RET_CHECK_AND_GOTO(ret, out);
 
-                memset (key, 0, sizeof (key));
-                snprintf (key, sizeof (key), "task%d.type", i);
-                ret = dict_get_str (dict, key, &task_type);
-                if (ret)
-                        goto out;
-                ret = xmlTextWriterWriteFormatElement (local->writer,
-                                                       (xmlChar *)"type",
-                                                       "%s", task_type);
-                XML_RET_CHECK_AND_GOTO (ret, out);
-
-                memset (key, 0, sizeof (key));
-                snprintf (key, sizeof (key), "task%d.id", i);
-                ret = dict_get_str (dict, key, &task_id_str);
-                if (ret)
-                        goto out;
-                ret = xmlTextWriterWriteFormatElement (local->writer,
-                                                       (xmlChar *)"id",
-                                                       "%s", task_id_str);
-                XML_RET_CHECK_AND_GOTO (ret, out);
-
-                memset (key, 0, sizeof (key));
-                snprintf (key, sizeof (key), "task%d.status", i);
-                ret = dict_get_int32 (dict, key, &status);
-                if (ret)
-                        goto out;
-                ret = xmlTextWriterWriteFormatElement (local->writer,
-                                                       (xmlChar *)"status",
-                                                       "%d", status);
-                XML_RET_CHECK_AND_GOTO (ret, out);
-
-                ret = xmlTextWriterWriteFormatElement (local->writer,
-                                                       (xmlChar *)"statusStr",
-                                                       "%s",
-                                             cli_vol_task_status_str[status]);
-
-                XML_RET_CHECK_AND_GOTO (ret, out);
-
-                memset (key, 0, sizeof (key));
-                snprintf (key, sizeof (key), "task%d", i);
-                if (!strcmp (task_type, "Remove brick")) {
-                        ret = cli_xml_output_remove_brick_task_params
-                                (local->writer, dict, key);
-                        if (ret)
-                                goto out;
-                }
+        ret = xmlTextWriterWriteFormatElement(local->writer,
+                                              (xmlChar *)"statusStr", "%s",
+                                              cli_vol_task_status_str[status]);
 
+        XML_RET_CHECK_AND_GOTO(ret, out);
 
-                /* </task> */
-                ret = xmlTextWriterEndElement (local->writer);
-                XML_RET_CHECK_AND_GOTO (ret, out);
+        snprintf(key, sizeof(key), "task%d", i);
+        if (!strcmp(task_type, "Remove brick")) {
+            ret = cli_xml_output_remove_brick_task_params(local->writer, dict,
+                                                          key);
+            if (ret)
+                goto out;
         }
 
-        /* </tasks> */
-        ret = xmlTextWriterEndElement (local->writer);
+        /* </task> */
+        ret = xmlTextWriterEndElement(local->writer);
+        XML_RET_CHECK_AND_GOTO(ret, out);
+    }
 
+    /* </tasks> */
+    ret = xmlTextWriterEndElement(local->writer);
+    XML_RET_CHECK_AND_GOTO(ret, out);
 out:
-        gf_log ("cli", GF_LOG_DEBUG, "Returning %d", ret);
-        return ret;
+    gf_log("cli", GF_LOG_DEBUG, "Returning %d", ret);
+    return ret;
 }
 
 #endif
 
 int
-cli_xml_output_vol_status_tasks_detail (cli_local_t *local, dict_t *dict)
+cli_xml_output_vol_status_tasks_detail(cli_local_t *local, dict_t *dict)
 {
 #if (HAVE_LIB_XML)
-        int    ret     = -1;
-        char  *volname = NULL;
+    int ret = -1;
+    char *volname = NULL;
 
-        /*<volume>*/
-        ret = xmlTextWriterStartElement (local->writer, (xmlChar *)"volume");
-        XML_RET_CHECK_AND_GOTO (ret, out);
+    /*<volume>*/
+    ret = xmlTextWriterStartElement(local->writer, (xmlChar *)"volume");
+    XML_RET_CHECK_AND_GOTO(ret, out);
 
-        ret = dict_get_str (dict, "volname", &volname);
-        if (ret)
-                goto out;
-        ret = xmlTextWriterWriteFormatElement (local->writer,
-                                               (xmlChar *)"volName", "%s",
-                                               volname);
-        XML_RET_CHECK_AND_GOTO (ret, out);
+    ret = dict_get_str(dict, "volname", &volname);
+    if (ret)
+        goto out;
+    ret = xmlTextWriterWriteFormatElement(local->writer, (xmlChar *)"volName",
+                                          "%s", volname);
+    XML_RET_CHECK_AND_GOTO(ret, out);
 
-        ret = cli_xml_output_vol_status_tasks (local, dict);
-        if (ret)
-                goto out;
+    ret = cli_xml_output_vol_status_tasks(local, dict);
+    if (ret)
+        goto out;
 
-        /* </volume> */
-        ret = xmlTextWriterEndElement (local->writer);
-        XML_RET_CHECK_AND_GOTO (ret, out);
+    /* </volume> */
+    ret = xmlTextWriterEndElement(local->writer);
+    XML_RET_CHECK_AND_GOTO(ret, out);
 
 out:
-        return ret;
+    return ret;
 #else
-        return 0;
+    return 0;
 #endif
 }
 
 int
-cli_xml_output_vol_status (cli_local_t *local, dict_t *dict)
+cli_xml_output_vol_status(cli_local_t *local, dict_t *dict)
 {
 #if (HAVE_LIB_XML)
-        int                     ret = -1;
-        char                    *volname = NULL;
-        int                     brick_count = 0;
-        int                     brick_index_max = -1;
-        int                     other_count = 0;
-        int                     index_max = 0;
-        uint32_t                cmd = GF_CLI_STATUS_NONE;
-        int                     online = 0;
-        gf_boolean_t            node_present = _gf_true;
-        int                     i;
-        int                     type            = -1;
-        int                     hot_brick_count = -1;
-
-        /* <volume> */
-        ret = xmlTextWriterStartElement (local->writer, (xmlChar *)"volume");
-        XML_RET_CHECK_AND_GOTO (ret, out);
-
-        ret = dict_get_str (dict, "volname", &volname);
-        if (ret)
-                goto out;
-        ret = xmlTextWriterWriteFormatElement (local->writer,
-                                               (xmlChar *)"volName", "%s",
-                                               volname);
-        XML_RET_CHECK_AND_GOTO (ret, out);
-
-        ret = dict_get_int32 (dict, "count", &brick_count);
-        if (ret)
-                goto out;
-        ret = xmlTextWriterWriteFormatElement (local->writer,
-                                               (xmlChar *)"nodeCount", "%d",
-                                               brick_count);
-        if (ret)
-                goto out;
-
-        ret = dict_get_uint32 (dict, "cmd", &cmd);
-        if (ret)
-                goto out;
-
-        ret = dict_get_int32 (dict, "brick-index-max", &brick_index_max);
-        if (ret)
-                goto out;
-        ret = dict_get_int32 (dict, "other-count", &other_count);
-        if (ret)
-                goto out;
-
-        index_max = brick_index_max + other_count;
-
-        ret = dict_get_int32 (dict, "type", &type);
-        if (ret)
+    int ret = -1;
+    char *volname = NULL;
+    int brick_count = 0;
+    int brick_index_max = -1;
+    int other_count = 0;
+    int index_max = 0;
+    uint32_t cmd = GF_CLI_STATUS_NONE;
+    int online = 0;
+    gf_boolean_t node_present = _gf_true;
+    int i;
+    int type = -1;
+
+    /* <volume> */
+    ret = xmlTextWriterStartElement(local->writer, (xmlChar *)"volume");
+    XML_RET_CHECK_AND_GOTO(ret, out);
+
+    ret = dict_get_str(dict, "volname", &volname);
+    if (ret)
+        goto out;
+    ret = xmlTextWriterWriteFormatElement(local->writer, (xmlChar *)"volName",
+                                          "%s", volname);
+    XML_RET_CHECK_AND_GOTO(ret, out);
+
+    ret = dict_get_int32(dict, "count", &brick_count);
+    if (ret)
+        goto out;
+    ret = xmlTextWriterWriteFormatElement(local->writer, (xmlChar *)"nodeCount",
+                                          "%d", brick_count);
+
+    XML_RET_CHECK_AND_GOTO(ret, out);
+
+    ret = dict_get_uint32(dict, "cmd", &cmd);
+    if (ret)
+        goto out;
+
+    ret = dict_get_int32(dict, "brick-index-max", &brick_index_max);
+    if (ret)
+        goto out;
+    ret = dict_get_int32(dict, "other-count", &other_count);
+    if (ret)
+        goto out;
+
+    index_max = brick_index_max + other_count;
+
+    ret = dict_get_int32(dict, "type", &type);
+    if (ret)
+        goto out;
+
+    for (i = 0; i <= index_max; i++) {
+        ret = cli_xml_output_vol_status_common(local->writer, dict, i, &online,
+                                               &node_present);
+        if (ret) {
+            if (node_present)
                 goto out;
-
-        if (type == GF_CLUSTER_TYPE_TIER) {
-                ret = dict_get_int32 (dict, "hot_brick_count",
-                                      &hot_brick_count);
-                if (ret)
-                        goto out;
-
-                ret = xmlTextWriterStartElement
-                        (local->writer, (xmlChar *)"hotBricks");
-                XML_RET_CHECK_AND_GOTO (ret, out);
-
+            else
+                continue;
         }
-        for (i = 0; i <= index_max; i++) {
 
-                if (type == GF_CLUSTER_TYPE_TIER && i == hot_brick_count) {
+        switch (cmd & GF_CLI_STATUS_MASK) {
+            case GF_CLI_STATUS_DETAIL:
+                ret = cli_xml_output_vol_status_detail(local->writer, dict, i);
+                if (ret)
+                    goto out;
+                break;
 
-                        /* </hotBricks>*/
-                        ret = xmlTextWriterEndElement (local->writer);
-                        XML_RET_CHECK_AND_GOTO (ret, out);
-                        ret = xmlTextWriterStartElement (local->writer,
-                                        (xmlChar *)"coldBricks");
-                        XML_RET_CHECK_AND_GOTO (ret, out);
-                }
-                ret = cli_xml_output_vol_status_common (local->writer, dict, i,
-                                                        &online, &node_present);
-                if (ret) {
-                        if (node_present)
-                                goto out;
-                        else
-                                continue;
+            case GF_CLI_STATUS_MEM:
+                if (online) {
+                    ret = cli_xml_output_vol_status_mem(local->writer, dict, i);
+                    if (ret)
+                        goto out;
                 }
+                break;
 
-                switch (cmd & GF_CLI_STATUS_MASK) {
-                case GF_CLI_STATUS_DETAIL:
-                        ret = cli_xml_output_vol_status_detail (local->writer,
-                                                                dict, i);
-                        if (ret)
-                                goto out;
-                        break;
-
-                case GF_CLI_STATUS_MEM:
-                        if (online) {
-                                ret = cli_xml_output_vol_status_mem
-                                        (local->writer, dict, i);
-                                if (ret)
-                                        goto out;
-                        }
-                        break;
-
-                case GF_CLI_STATUS_CLIENTS:
-                        if (online) {
-                                ret = cli_xml_output_vol_status_clients
-                                        (local->writer, dict, i);
-                                if (ret)
-                                        goto out;
-                        }
-                        break;
-
-                case GF_CLI_STATUS_INODE:
-                        if (online) {
-                                ret = cli_xml_output_vol_status_inode
-                                        (local->writer, dict, i);
-                                if (ret)
-                                        goto out;
-                        }
-                        break;
-
-                case GF_CLI_STATUS_FD:
-                        if (online) {
-                                ret = cli_xml_output_vol_status_fd
-                                        (local->writer, dict, i);
-                                if (ret)
-                                        goto out;
-                        }
-                        break;
-
-                case GF_CLI_STATUS_CALLPOOL:
-                        if (online) {
-                                ret = cli_xml_output_vol_status_callpool
-                                        (local->writer, dict, i);
-                                if (ret)
-                                        goto out;
-                        }
-                        break;
-                default:
-                        break;
-
+            case GF_CLI_STATUS_CLIENTS:
+                if (online) {
+                    ret = cli_xml_output_vol_status_clients(local->writer, dict,
+                                                            i);
+                    if (ret)
+                        goto out;
                 }
+                break;
 
-                /* </node>  was opened in cli_xml_output_vol_status_common()*/
-                ret = xmlTextWriterEndElement (local->writer);
-                XML_RET_CHECK_AND_GOTO (ret, out);
-
-        }
+            case GF_CLI_STATUS_INODE:
+                if (online) {
+                    ret = cli_xml_output_vol_status_inode(local->writer, dict,
+                                                          i);
+                    if (ret)
+                        goto out;
+                }
+                break;
 
-        /* </coldBricks>*/
-        if (type == GF_CLUSTER_TYPE_TIER && i >= hot_brick_count) {
-                ret = xmlTextWriterEndElement (local->writer);
-                XML_RET_CHECK_AND_GOTO (ret, out);
-        }
+            case GF_CLI_STATUS_FD:
+                if (online) {
+                    ret = cli_xml_output_vol_status_fd(local->writer, dict, i);
+                    if (ret)
+                        goto out;
+                }
+                break;
 
-        /* Tasks are only present when a normal volume status call is done on a
-         * single volume or on all volumes
-         */
-        if (((cmd & GF_CLI_STATUS_MASK) == GF_CLI_STATUS_NONE) &&
-            (cmd & (GF_CLI_STATUS_VOL|GF_CLI_STATUS_ALL))) {
-                ret = cli_xml_output_vol_status_tasks (local, dict);
-                if (ret)
+            case GF_CLI_STATUS_CALLPOOL:
+                if (online) {
+                    ret = cli_xml_output_vol_status_callpool(local->writer,
+                                                             dict, i);
+                    if (ret)
                         goto out;
+                }
+                break;
+            default:
+                break;
         }
 
-        /* </volume> */
-        ret = xmlTextWriterEndElement (local->writer);
-        XML_RET_CHECK_AND_GOTO (ret, out);
+        /* </node>  was opened in cli_xml_output_vol_status_common()*/
+        ret = xmlTextWriterEndElement(local->writer);
+        XML_RET_CHECK_AND_GOTO(ret, out);
+    }
+
+    /* Tasks are only present when a normal volume status call is done on a
+     * single volume or on all volumes
+     */
+    if (((cmd & GF_CLI_STATUS_MASK) == GF_CLI_STATUS_NONE) &&
+        (cmd & (GF_CLI_STATUS_VOL | GF_CLI_STATUS_ALL))) {
+        ret = cli_xml_output_vol_status_tasks(local, dict);
+        if (ret)
+            goto out;
+    }
+
+    /* </volume> */
+    ret = xmlTextWriterEndElement(local->writer);
+    XML_RET_CHECK_AND_GOTO(ret, out);
 
 out:
-        gf_log ("cli", GF_LOG_DEBUG, "Returning %d", ret);
-        return ret;
+    gf_log("cli", GF_LOG_DEBUG, "Returning %d", ret);
+    return ret;
 #else
-        return 0;
+    return 0;
 #endif
 }
 
 #if (HAVE_LIB_XML)
 int
-cli_xml_output_vol_top_rw_perf (xmlTextWriterPtr writer, dict_t *dict,
-                                int brick_index, int member_index)
+cli_xml_output_vol_top_rw_perf(xmlTextWriterPtr writer, dict_t *dict,
+                               int brick_index, int member_index)
 {
-        int        ret = -1;
-        char      *filename = NULL;
-        uint64_t   throughput = 0;
-        long int   time_sec = 0;
-        long int   time_usec = 0;
-        char       timestr[256] = {0,};
-        char       key[1024] = {0,};
-
-        /* <file> */
-        ret = xmlTextWriterStartElement (writer, (xmlChar *)"file");
-        XML_RET_CHECK_AND_GOTO (ret, out);
-
-        snprintf (key, sizeof (key), "%d-filename-%d", brick_index,
-                  member_index);
-        ret = dict_get_str (dict, key, &filename);
-        if (ret)
-                goto out;
-        ret = xmlTextWriterWriteFormatElement (writer, (xmlChar *)"filename",
-                                               "%s", filename);
-        XML_RET_CHECK_AND_GOTO (ret, out);
-
-        memset (key, 0, sizeof (key));
-        snprintf (key, sizeof (key), "%d-value-%d", brick_index, member_index);
-        ret = dict_get_uint64 (dict, key, &throughput);
-        if (ret)
-                goto out;
-        ret = xmlTextWriterWriteFormatElement (writer, (xmlChar *)"count",
-                                               "%"PRIu64, throughput);
-        XML_RET_CHECK_AND_GOTO (ret, out);
-
-        memset (key, 0, sizeof (key));
-        snprintf (key, sizeof (key), "%d-time-sec-%d", brick_index,
-                  member_index);
-        ret = dict_get_int32 (dict, key, (int32_t *)&time_sec);
-        if (ret)
-                goto out;
-
-        memset (key, 0, sizeof (key));
-        snprintf (key, sizeof (key), "%d-time-usec-%d", brick_index,
-                  member_index);
-        ret = dict_get_int32 (dict, key, (int32_t *)&time_usec);
-        if (ret)
-                goto out;
-
-        gf_time_fmt (timestr, sizeof timestr, time_sec, gf_timefmt_FT);
-        snprintf (timestr + strlen (timestr),
-                  sizeof timestr - strlen (timestr),
-                  ".%"GF_PRI_SUSECONDS, time_usec);
-        ret = xmlTextWriterWriteFormatElement (writer, (xmlChar *)"time",
-                                               "%s", timestr);
-        XML_RET_CHECK_AND_GOTO (ret, out);
-
-        /* </file> */
-        ret = xmlTextWriterEndElement (writer);
-        XML_RET_CHECK_AND_GOTO (ret, out);
+    int ret = -1;
+    char *filename = NULL;
+    uint64_t throughput = 0;
+    struct timeval tv = {
+        0,
+    };
+    char timestr[GF_TIMESTR_SIZE] = {
+        0,
+    };
+    char key[1024] = {
+        0,
+    };
+
+    /* <file> */
+    ret = xmlTextWriterStartElement(writer, (xmlChar *)"file");
+    XML_RET_CHECK_AND_GOTO(ret, out);
+
+    snprintf(key, sizeof(key), "%d-filename-%d", brick_index, member_index);
+    ret = dict_get_str(dict, key, &filename);
+    if (ret)
+        goto out;
+    ret = xmlTextWriterWriteFormatElement(writer, (xmlChar *)"filename", "%s",
+                                          filename);
+    XML_RET_CHECK_AND_GOTO(ret, out);
+
+    snprintf(key, sizeof(key), "%d-value-%d", brick_index, member_index);
+    ret = dict_get_uint64(dict, key, &throughput);
+    if (ret)
+        goto out;
+    ret = xmlTextWriterWriteFormatElement(writer, (xmlChar *)"count",
+                                          "%" PRIu64, throughput);
+    XML_RET_CHECK_AND_GOTO(ret, out);
+
+    snprintf(key, sizeof(key), "%d-time-sec-%d", brick_index, member_index);
+    ret = dict_get_int32(dict, key, (int32_t *)&tv.tv_sec);
+    if (ret)
+        goto out;
+
+    snprintf(key, sizeof(key), "%d-time-usec-%d", brick_index, member_index);
+    ret = dict_get_int32(dict, key, (int32_t *)&tv.tv_usec);
+    if (ret)
+        goto out;
+
+    gf_time_fmt_tv(timestr, sizeof timestr, &tv, gf_timefmt_FT);
+    ret = xmlTextWriterWriteFormatElement(writer, (xmlChar *)"time", "%s",
+                                          timestr);
+    XML_RET_CHECK_AND_GOTO(ret, out);
+
+    /* </file> */
+    ret = xmlTextWriterEndElement(writer);
+    XML_RET_CHECK_AND_GOTO(ret, out);
 
 out:
-        gf_log ("cli", GF_LOG_DEBUG, "Returning %d", ret);
-        return ret;
+    gf_log("cli", GF_LOG_DEBUG, "Returning %d", ret);
+    return ret;
 }
 
 int
-cli_xml_output_vol_top_other (xmlTextWriterPtr writer, dict_t *dict,
-                                int brick_index, int member_index)
+cli_xml_output_vol_top_other(xmlTextWriterPtr writer, dict_t *dict,
+                             int brick_index, int member_index)
 {
-        int             ret = -1;
-        char            *filename = NULL;
-        uint64_t        count = 0;
-        char            key[1024] = {0,};
-
-        /* <file> */
-        ret = xmlTextWriterStartElement (writer, (xmlChar *)"file");
-        XML_RET_CHECK_AND_GOTO (ret, out);
-
-        snprintf (key, sizeof (key), "%d-filename-%d", brick_index,
-                  member_index);
-        ret = dict_get_str (dict, key, &filename);
-        if (ret)
-                goto out;
-        ret = xmlTextWriterWriteFormatElement (writer, (xmlChar *)"filename",
-                                               "%s", filename);
-        XML_RET_CHECK_AND_GOTO (ret, out);
-
-        memset (key, 0, sizeof (key));
-        snprintf (key, sizeof (key), "%d-value-%d", brick_index, member_index);
-        ret = dict_get_uint64 (dict, key, &count);
-        if (ret)
-                goto out;
-        ret = xmlTextWriterWriteFormatElement (writer, (xmlChar *)"count",
-                                               "%"PRIu64, count);
-        XML_RET_CHECK_AND_GOTO (ret, out);
-
-        /* </file> */
-        ret = xmlTextWriterEndElement (writer);
-        XML_RET_CHECK_AND_GOTO (ret, out);
+    int ret = -1;
+    char *filename = NULL;
+    uint64_t count = 0;
+    char key[1024] = {
+        0,
+    };
+
+    /* <file> */
+    ret = xmlTextWriterStartElement(writer, (xmlChar *)"file");
+    XML_RET_CHECK_AND_GOTO(ret, out);
+
+    snprintf(key, sizeof(key), "%d-filename-%d", brick_index, member_index);
+    ret = dict_get_str(dict, key, &filename);
+    if (ret)
+        goto out;
+    ret = xmlTextWriterWriteFormatElement(writer, (xmlChar *)"filename", "%s",
+                                          filename);
+    XML_RET_CHECK_AND_GOTO(ret, out);
+
+    snprintf(key, sizeof(key), "%d-value-%d", brick_index, member_index);
+    ret = dict_get_uint64(dict, key, &count);
+    if (ret)
+        goto out;
+    ret = xmlTextWriterWriteFormatElement(writer, (xmlChar *)"count",
+                                          "%" PRIu64, count);
+    XML_RET_CHECK_AND_GOTO(ret, out);
+
+    /* </file> */
+    ret = xmlTextWriterEndElement(writer);
+    XML_RET_CHECK_AND_GOTO(ret, out);
 
 out:
-        gf_log ("cli", GF_LOG_DEBUG, "Returning %d", ret);
-        return ret;
+    gf_log("cli", GF_LOG_DEBUG, "Returning %d", ret);
+    return ret;
 }
 #endif
 
 int
-cli_xml_output_vol_top (dict_t *dict, int op_ret, int op_errno,
-                        char *op_errstr)
+cli_xml_output_vol_top(dict_t *dict, int op_ret, int op_errno, char *op_errstr)
 {
 #if (HAVE_LIB_XML)
-        int                     ret = -1;
-        xmlTextWriterPtr        writer = NULL;
-        xmlDocPtr               doc = NULL;
-        int                     brick_count = 0;
-        int                     top_op = GF_CLI_TOP_NONE;
-        char                    *brick_name = NULL;
-        int                     members = 0;
-        uint64_t                current_open = 0;
-        uint64_t                max_open = 0;
-        char                    *max_open_time = NULL;
-        double                  throughput = 0.0;
-        double                  time_taken = 0.0;
-        char                    key[1024] = {0,};
-        int                     i = 0;
-        int                     j = 0;
-
-        ret = cli_begin_xml_output (&writer, &doc);
+    int ret = -1;
+    xmlTextWriterPtr writer = NULL;
+    xmlDocPtr doc = NULL;
+    int brick_count = 0;
+    int top_op = GF_CLI_TOP_NONE;
+    char *brick_name = NULL;
+    int members = 0;
+    uint64_t current_open = 0;
+    uint64_t max_open = 0;
+    char *max_open_time = NULL;
+    double throughput = 0.0;
+    double time_taken = 0.0;
+    char key[1024] = {
+        0,
+    };
+    int i = 0;
+    int j = 0;
+
+    ret = cli_begin_xml_output(&writer, &doc);
+    if (ret)
+        goto out;
+
+    ret = cli_xml_output_common(writer, op_ret, op_errno, op_errstr);
+    if (ret)
+        goto out;
+
+    /* <volTop> */
+    ret = xmlTextWriterStartElement(writer, (xmlChar *)"volTop");
+    XML_RET_CHECK_AND_GOTO(ret, out);
+
+    ret = dict_get_int32(dict, "count", &brick_count);
+    if (ret)
+        goto out;
+    ret = xmlTextWriterWriteFormatElement(writer, (xmlChar *)"brickCount", "%d",
+                                          brick_count);
+    XML_RET_CHECK_AND_GOTO(ret, out);
+
+    ret = dict_get_int32(dict, "1-top-op", &top_op);
+    if (ret)
+        goto out;
+    ret = xmlTextWriterWriteFormatElement(writer, (xmlChar *)"topOp", "%d",
+                                          top_op);
+    XML_RET_CHECK_AND_GOTO(ret, out);
+
+    while (i < brick_count) {
+        i++;
+
+        /* <brick> */
+        ret = xmlTextWriterStartElement(writer, (xmlChar *)"brick");
+        XML_RET_CHECK_AND_GOTO(ret, out);
+
+        snprintf(key, sizeof(key), "%d-brick", i);
+        ret = dict_get_str(dict, key, &brick_name);
         if (ret)
-                goto out;
+            goto out;
+        ret = xmlTextWriterWriteFormatElement(writer, (xmlChar *)"name", "%s",
+                                              brick_name);
+        XML_RET_CHECK_AND_GOTO(ret, out);
 
-        ret = cli_xml_output_common (writer, op_ret, op_errno, op_errstr);
+        snprintf(key, sizeof(key), "%d-members", i);
+        ret = dict_get_int32(dict, key, &members);
         if (ret)
-                goto out;
+            goto out;
+        ret = xmlTextWriterWriteFormatElement(writer, (xmlChar *)"members",
+                                              "%d", members);
+        XML_RET_CHECK_AND_GOTO(ret, out);
 
-        /* <volTop> */
-        ret = xmlTextWriterStartElement (writer, (xmlChar *)"volTop");
-        XML_RET_CHECK_AND_GOTO (ret, out);
+        switch (top_op) {
+            case GF_CLI_TOP_OPEN:
+                snprintf(key, sizeof(key), "%d-current-open", i);
+                ret = dict_get_uint64(dict, key, &current_open);
+                if (ret)
+                    goto out;
+                ret = xmlTextWriterWriteFormatElement(
+                    writer, (xmlChar *)"currentOpen", "%" PRIu64, current_open);
+                XML_RET_CHECK_AND_GOTO(ret, out);
 
-        ret = dict_get_int32 (dict, "count", &brick_count);
-        if (ret)
-                goto out;
-        ret = xmlTextWriterWriteFormatElement (writer, (xmlChar *)"brickCount",
-                                               "%d", brick_count);
-        XML_RET_CHECK_AND_GOTO (ret, out);
+                snprintf(key, sizeof(key), "%d-max-open", i);
+                ret = dict_get_uint64(dict, key, &max_open);
+                if (ret)
+                    goto out;
+                ret = xmlTextWriterWriteFormatElement(
+                    writer, (xmlChar *)"maxOpen", "%" PRIu64, max_open);
+                XML_RET_CHECK_AND_GOTO(ret, out);
 
-        ret = dict_get_int32 (dict, "1-top-op", &top_op);
-        if (ret)
-                goto out;
-        ret = xmlTextWriterWriteFormatElement (writer, (xmlChar *)"topOp",
-                                               "%d", top_op);
-        XML_RET_CHECK_AND_GOTO (ret, out);
+                snprintf(key, sizeof(key), "%d-max-openfd-time", i);
+                ret = dict_get_str(dict, key, &max_open_time);
+                if (ret)
+                    goto out;
+                ret = xmlTextWriterWriteFormatElement(
+                    writer, (xmlChar *)"maxOpenTime", "%s", max_open_time);
+                XML_RET_CHECK_AND_GOTO(ret, out);
 
-        while (i < brick_count) {
-                i++;
+            case GF_CLI_TOP_READ:
+            case GF_CLI_TOP_WRITE:
+            case GF_CLI_TOP_OPENDIR:
+            case GF_CLI_TOP_READDIR:
 
-                /* <brick> */
-                ret = xmlTextWriterStartElement (writer, (xmlChar *)"brick");
-                XML_RET_CHECK_AND_GOTO (ret, out);
+                break;
 
-                memset (key, 0, sizeof (key));
-                snprintf (key, sizeof (key), "%d-brick", i);
-                ret = dict_get_str (dict, key, &brick_name);
-                if (ret)
-                        goto out;
-                ret = xmlTextWriterWriteFormatElement (writer,
-                                                       (xmlChar *)"name",
-                                                       "%s", brick_name);
-                XML_RET_CHECK_AND_GOTO (ret, out);
-
-                memset (key, 0, sizeof (key));
-                snprintf (key , sizeof (key), "%d-members", i);
-                ret = dict_get_int32 (dict, key, &members);
-                if (ret)
-                        goto out;
-                ret = xmlTextWriterWriteFormatElement (writer,
-                                                       (xmlChar *)"members",
-                                                       "%d", members);
-                XML_RET_CHECK_AND_GOTO (ret, out);
-
-                switch (top_op) {
-                case GF_CLI_TOP_OPEN:
-                        memset (key, 0, sizeof (key));
-                        snprintf (key, sizeof (key), "%d-current-open", i);
-                        ret = dict_get_uint64 (dict, key, &current_open);
-                        if (ret)
-                                goto out;
-                        ret = xmlTextWriterWriteFormatElement
-                                (writer, (xmlChar *)"currentOpen", "%"PRIu64,
-                                 current_open);
-                        XML_RET_CHECK_AND_GOTO (ret, out);
-
-                        memset (key, 0, sizeof (key));
-                        snprintf (key, sizeof (key), "%d-max-open", i);
-                        ret = dict_get_uint64 (dict, key, &max_open);
-                        if (ret)
-                                goto out;
-                        ret = xmlTextWriterWriteFormatElement
-                                (writer, (xmlChar *)"maxOpen", "%"PRIu64,
-                                 max_open);
-                        XML_RET_CHECK_AND_GOTO (ret, out);
-
-                        memset (key, 0, sizeof (key));
-                        snprintf (key, sizeof (key), "%d-max-openfd-time", i);
-                        ret = dict_get_str (dict, key, &max_open_time);
-                        if (ret)
-                                goto out;
-                        ret = xmlTextWriterWriteFormatElement
-                                (writer, (xmlChar *)"maxOpenTime", "%s",
-                                 max_open_time);
-                        XML_RET_CHECK_AND_GOTO (ret, out);
-
-                case GF_CLI_TOP_READ:
-                case GF_CLI_TOP_WRITE:
-                case GF_CLI_TOP_OPENDIR:
-                case GF_CLI_TOP_READDIR:
-
-                        break;
-
-                case GF_CLI_TOP_READ_PERF:
-                case GF_CLI_TOP_WRITE_PERF:
-                        memset (key, 0, sizeof (key));
-                        snprintf (key, sizeof (key), "%d-throughput", i);
-                        ret = dict_get_double (dict, key, &throughput);
-                        if (!ret) {
-                                memset (key, 0, sizeof (key));
-                                snprintf (key, sizeof (key), "%d-time", i);
-                                ret = dict_get_double (dict, key, &time_taken);
-                        }
-
-                        if (!ret) {
-                                ret = xmlTextWriterWriteFormatElement
-                                        (writer, (xmlChar *)"throughput",
-                                         "%f", throughput);
-                                XML_RET_CHECK_AND_GOTO (ret, out);
-
-                                ret = xmlTextWriterWriteFormatElement
-                                        (writer, (xmlChar *)"timeTaken",
-                                         "%f", time_taken);
-                        }
-
-                        break;
-
-                default:
-                        ret = -1;
-                        goto out;
+            case GF_CLI_TOP_READ_PERF:
+            case GF_CLI_TOP_WRITE_PERF:
+                snprintf(key, sizeof(key), "%d-throughput", i);
+                ret = dict_get_double(dict, key, &throughput);
+                if (!ret) {
+                    snprintf(key, sizeof(key), "%d-time", i);
+                    ret = dict_get_double(dict, key, &time_taken);
                 }
 
-                for (j = 1; j <= members; j++) {
-                        if (top_op == GF_CLI_TOP_READ_PERF ||
-                            top_op == GF_CLI_TOP_WRITE_PERF) {
-                                ret = cli_xml_output_vol_top_rw_perf
-                                        (writer, dict, i, j);
-                        } else {
-                                ret = cli_xml_output_vol_top_other
-                                        (writer, dict, i, j);
-                        }
-                        if (ret)
-                                goto out;
+                if (!ret) {
+                    ret = xmlTextWriterWriteFormatElement(
+                        writer, (xmlChar *)"throughput", "%f", throughput);
+                    XML_RET_CHECK_AND_GOTO(ret, out);
+
+                    ret = xmlTextWriterWriteFormatElement(
+                        writer, (xmlChar *)"timeTaken", "%f", time_taken);
+                    XML_RET_CHECK_AND_GOTO(ret, out);
                 }
 
+                break;
 
-                /* </brick> */
-                ret = xmlTextWriterEndElement (writer);
-                XML_RET_CHECK_AND_GOTO (ret, out);
+            default:
+                ret = -1;
+                goto out;
         }
 
-        /* </volTop> */
-        ret = xmlTextWriterEndElement (writer);
-        XML_RET_CHECK_AND_GOTO (ret, out);
-        ret = cli_end_xml_output (writer, doc);
+        for (j = 1; j <= members; j++) {
+            if (top_op == GF_CLI_TOP_READ_PERF ||
+                top_op == GF_CLI_TOP_WRITE_PERF) {
+                ret = cli_xml_output_vol_top_rw_perf(writer, dict, i, j);
+            } else {
+                ret = cli_xml_output_vol_top_other(writer, dict, i, j);
+            }
+            if (ret)
+                goto out;
+        }
+
+        /* </brick> */
+        ret = xmlTextWriterEndElement(writer);
+        XML_RET_CHECK_AND_GOTO(ret, out);
+    }
+
+    /* </volTop> */
+    ret = xmlTextWriterEndElement(writer);
+    XML_RET_CHECK_AND_GOTO(ret, out);
+    ret = cli_end_xml_output(writer, doc);
 
 out:
-        gf_log ("cli", GF_LOG_DEBUG, "Returning %d", ret);
-        return ret;
+    gf_log("cli", GF_LOG_DEBUG, "Returning %d", ret);
+    return ret;
 #else
-        return 0;
+    return 0;
 #endif
 }
 
 #if (HAVE_LIB_XML)
 int
-cli_xml_output_vol_profile_stats (xmlTextWriterPtr writer, dict_t *dict,
-                                  int brick_index, int interval)
+cli_xml_output_vol_profile_stats(xmlTextWriterPtr writer, dict_t *dict,
+                                 int brick_index, int interval)
 {
-        int                     ret = -1;
-        uint64_t                read_count = 0;
-        uint64_t                write_count = 0;
-        uint64_t                hits = 0;
-        double                  avg_latency = 0.0;
-        double                  max_latency = 0.0;
-        double                  min_latency = 0.0;
-        uint64_t                duration = 0;
-        uint64_t                total_read = 0;
-        uint64_t                total_write = 0;
-        char                    key[1024] = {0};
-        int                     i = 0;
-
-        /* <cumulativeStats> || <intervalStats> */
-        if (interval == -1)
-                ret = xmlTextWriterStartElement (writer,
-                                                 (xmlChar *)"cumulativeStats");
-        else
-                ret = xmlTextWriterStartElement (writer,
-                                                 (xmlChar *)"intervalStats");
-        XML_RET_CHECK_AND_GOTO (ret, out);
-
-        /* <blockStats> */
-        ret = xmlTextWriterStartElement (writer, (xmlChar *)"blockStats");
-        XML_RET_CHECK_AND_GOTO (ret, out);
-
-        for (i = 0; i < 32; i++) {
-                /* <block> */
-                ret = xmlTextWriterStartElement (writer, (xmlChar *)"block");
-                XML_RET_CHECK_AND_GOTO (ret, out);
-
-                ret = xmlTextWriterWriteFormatElement
-                        (writer, (xmlChar *)"size", "%"PRIu32, (1 << i));
-                XML_RET_CHECK_AND_GOTO (ret, out);
-
-                memset (key, 0, sizeof (key));
-                snprintf (key, sizeof (key), "%d-%d-read-%d", brick_index,
-                          interval, (1 << i));
-                ret = dict_get_uint64 (dict, key, &read_count);
-                if (ret)
-                        read_count = 0;
-                ret = xmlTextWriterWriteFormatElement
-                        (writer, (xmlChar *)"reads", "%"PRIu64, read_count);
-                XML_RET_CHECK_AND_GOTO (ret, out);
-
-                memset (key, 0, sizeof (key));
-                snprintf (key, sizeof (key), "%d-%d-write-%d", brick_index,
-                          interval, (1 << i));
-                ret = dict_get_uint64 (dict, key, &write_count);
-                if (ret)
-                        write_count = 0;
-                ret = xmlTextWriterWriteFormatElement
-                        (writer, (xmlChar *)"writes", "%"PRIu64, write_count);
-                XML_RET_CHECK_AND_GOTO (ret, out);
-
-                /* </block> */
-                ret = xmlTextWriterEndElement (writer);
-                XML_RET_CHECK_AND_GOTO (ret, out);
-        }
+    int ret = -1;
+    uint64_t read_count = 0;
+    uint64_t write_count = 0;
+    uint64_t hits = 0;
+    double avg_latency = 0.0;
+    double max_latency = 0.0;
+    double min_latency = 0.0;
+    uint64_t duration = 0;
+    uint64_t total_read = 0;
+    uint64_t total_write = 0;
+    char key[1024] = {0};
+    int i = 0;
+
+    /* <cumulativeStats> || <intervalStats> */
+    if (interval == -1)
+        ret = xmlTextWriterStartElement(writer, (xmlChar *)"cumulativeStats");
+    else
+        ret = xmlTextWriterStartElement(writer, (xmlChar *)"intervalStats");
+    XML_RET_CHECK_AND_GOTO(ret, out);
+
+    /* <blockStats> */
+    ret = xmlTextWriterStartElement(writer, (xmlChar *)"blockStats");
+    XML_RET_CHECK_AND_GOTO(ret, out);
+
+    for (i = 0; i < 32; i++) {
+        /* <block> */
+        ret = xmlTextWriterStartElement(writer, (xmlChar *)"block");
+        XML_RET_CHECK_AND_GOTO(ret, out);
 
-        /* </blockStats> */
-        ret = xmlTextWriterEndElement (writer);
-        XML_RET_CHECK_AND_GOTO (ret, out);
+        ret = xmlTextWriterWriteFormatElement(writer, (xmlChar *)"size",
+                                              "%" PRIu32, (1U << i));
+        XML_RET_CHECK_AND_GOTO(ret, out);
 
-        /* <fopStats> */
-        ret = xmlTextWriterStartElement (writer, (xmlChar *)"fopStats");
-        XML_RET_CHECK_AND_GOTO (ret, out);
+        snprintf(key, sizeof(key), "%d-%d-read-%" PRIu32, brick_index, interval,
+                 (1U << i));
+        ret = dict_get_uint64(dict, key, &read_count);
+        if (ret)
+            read_count = 0;
+        ret = xmlTextWriterWriteFormatElement(writer, (xmlChar *)"reads",
+                                              "%" PRIu64, read_count);
+        XML_RET_CHECK_AND_GOTO(ret, out);
 
-        for (i = 0; i < GF_FOP_MAXVALUE; i++) {
-                memset (key, 0, sizeof (key));
-                snprintf (key, sizeof (key), "%d-%d-%d-hits", brick_index,
-                          interval, i);
-                ret = dict_get_uint64 (dict, key, &hits);
-                if (ret)
-                        goto cont;
+        snprintf(key, sizeof(key), "%d-%d-write-%" PRIu32, brick_index,
+                 interval, (1U << i));
+        ret = dict_get_uint64(dict, key, &write_count);
+        if (ret)
+            write_count = 0;
+        ret = xmlTextWriterWriteFormatElement(writer, (xmlChar *)"writes",
+                                              "%" PRIu64, write_count);
+        XML_RET_CHECK_AND_GOTO(ret, out);
 
-                memset (key, 0, sizeof (key));
-                snprintf (key, sizeof (key), "%d-%d-%d-avglatency", brick_index,
-                          interval, i);
-                ret = dict_get_double (dict, key, &avg_latency);
-                if (ret)
-                        goto cont;
+        /* </block> */
+        ret = xmlTextWriterEndElement(writer);
+        XML_RET_CHECK_AND_GOTO(ret, out);
+    }
 
-                memset (key, 0, sizeof (key));
-                snprintf (key, sizeof (key), "%d-%d-%d-minlatency", brick_index,
-                          interval, i);
-                ret = dict_get_double (dict, key, &min_latency);
-                if (ret)
-                        goto cont;
+    /* </blockStats> */
+    ret = xmlTextWriterEndElement(writer);
+    XML_RET_CHECK_AND_GOTO(ret, out);
 
-                memset (key, 0, sizeof (key));
-                snprintf (key, sizeof (key), "%d-%d-%d-maxlatency", brick_index,
-                          interval, i);
-                ret = dict_get_double (dict, key, &max_latency);
-                if (ret)
-                        goto cont;
+    /* <fopStats> */
+    ret = xmlTextWriterStartElement(writer, (xmlChar *)"fopStats");
+    XML_RET_CHECK_AND_GOTO(ret, out);
+
+    for (i = 0; i < GF_FOP_MAXVALUE; i++) {
+        snprintf(key, sizeof(key), "%d-%d-%d-hits", brick_index, interval, i);
+        ret = dict_get_uint64(dict, key, &hits);
+        if (ret)
+            goto cont;
 
-                /* <fop> */
-                ret = xmlTextWriterStartElement (writer, (xmlChar *)"fop");
-                XML_RET_CHECK_AND_GOTO (ret, out);
+        snprintf(key, sizeof(key), "%d-%d-%d-avglatency", brick_index, interval,
+                 i);
+        ret = dict_get_double(dict, key, &avg_latency);
+        if (ret)
+            goto cont;
 
-                ret = xmlTextWriterWriteFormatElement
-                        (writer, (xmlChar *)"name","%s", gf_fop_list[i]);
-                XML_RET_CHECK_AND_GOTO (ret, out);
+        snprintf(key, sizeof(key), "%d-%d-%d-minlatency", brick_index, interval,
+                 i);
+        ret = dict_get_double(dict, key, &min_latency);
+        if (ret)
+            goto cont;
 
-                ret = xmlTextWriterWriteFormatElement
-                        (writer, (xmlChar *)"hits", "%"PRIu64, hits);
-                XML_RET_CHECK_AND_GOTO (ret, out);
+        snprintf(key, sizeof(key), "%d-%d-%d-maxlatency", brick_index, interval,
+                 i);
+        ret = dict_get_double(dict, key, &max_latency);
+        if (ret)
+            goto cont;
 
-                ret = xmlTextWriterWriteFormatElement
-                        (writer, (xmlChar *)"avgLatency", "%f", avg_latency);
-                XML_RET_CHECK_AND_GOTO (ret, out);
+        /* <fop> */
+        ret = xmlTextWriterStartElement(writer, (xmlChar *)"fop");
+        XML_RET_CHECK_AND_GOTO(ret, out);
 
-                ret = xmlTextWriterWriteFormatElement
-                        (writer, (xmlChar *)"minLatency", "%f", min_latency);
-                XML_RET_CHECK_AND_GOTO (ret, out);
+        ret = xmlTextWriterWriteFormatElement(writer, (xmlChar *)"name", "%s",
+                                              gf_fop_list[i]);
+        XML_RET_CHECK_AND_GOTO(ret, out);
 
-                ret = xmlTextWriterWriteFormatElement
-                        (writer, (xmlChar *)"maxLatency", "%f", max_latency);
-                XML_RET_CHECK_AND_GOTO (ret, out);
+        ret = xmlTextWriterWriteFormatElement(writer, (xmlChar *)"hits",
+                                              "%" PRIu64, hits);
+        XML_RET_CHECK_AND_GOTO(ret, out);
 
-                /* </fop> */
-                ret = xmlTextWriterEndElement (writer);
-                XML_RET_CHECK_AND_GOTO (ret, out);
+        ret = xmlTextWriterWriteFormatElement(writer, (xmlChar *)"avgLatency",
+                                              "%f", avg_latency);
+        XML_RET_CHECK_AND_GOTO(ret, out);
 
-cont:
-                hits = 0;
-                avg_latency = 0.0;
-                min_latency = 0.0;
-                max_latency = 0.0;
-        }
+        ret = xmlTextWriterWriteFormatElement(writer, (xmlChar *)"minLatency",
+                                              "%f", min_latency);
+        XML_RET_CHECK_AND_GOTO(ret, out);
 
-        /* </fopStats> */
-        ret = xmlTextWriterEndElement (writer);
-        XML_RET_CHECK_AND_GOTO (ret, out);
+        ret = xmlTextWriterWriteFormatElement(writer, (xmlChar *)"maxLatency",
+                                              "%f", max_latency);
+        XML_RET_CHECK_AND_GOTO(ret, out);
 
-        memset (key, 0, sizeof (key));
-        snprintf (key, sizeof (key), "%d-%d-duration", brick_index, interval);
-        ret = dict_get_uint64 (dict, key, &duration);
-        if (ret)
-                goto out;
-        ret = xmlTextWriterWriteFormatElement (writer, (xmlChar *)"duration",
-                                               "%"PRIu64, duration);
-        XML_RET_CHECK_AND_GOTO (ret, out);
+        /* </fop> */
+        ret = xmlTextWriterEndElement(writer);
+        XML_RET_CHECK_AND_GOTO(ret, out);
 
-        memset (key, 0, sizeof (key));
-        snprintf (key, sizeof (key), "%d-%d-total-read", brick_index, interval);
-        ret = dict_get_uint64 (dict, key, &total_read);
+    cont:
+        hits = 0;
+        avg_latency = 0.0;
+        min_latency = 0.0;
+        max_latency = 0.0;
+    }
+
+    for (i = 0; i < GF_UPCALL_FLAGS_MAXVALUE; i++) {
+        hits = 0;
+        avg_latency = 0.0;
+        min_latency = 0.0;
+        max_latency = 0.0;
+
+        snprintf(key, sizeof(key), "%d-%d-%d-upcall-hits", brick_index,
+                 interval, i);
+        ret = dict_get_uint64(dict, key, &hits);
         if (ret)
-                goto out;
-        ret = xmlTextWriterWriteFormatElement (writer, (xmlChar *)"totalRead",
-                                               "%"PRIu64, total_read);
-        XML_RET_CHECK_AND_GOTO (ret, out);
+            continue;
 
-        memset (key, 0, sizeof (key));
-        snprintf (key, sizeof (key), "%d-%d-total-write", brick_index, interval);
-        ret = dict_get_uint64 (dict, key, &total_write);
-        if (ret)
-                goto out;
-        ret = xmlTextWriterWriteFormatElement (writer, (xmlChar *)"totalWrite",
-                                               "%"PRIu64, total_write);
-        XML_RET_CHECK_AND_GOTO (ret, out);
+        /* <fop> */
+        ret = xmlTextWriterStartElement(writer, (xmlChar *)"fop");
+        XML_RET_CHECK_AND_GOTO(ret, out);
 
-        /* </cumulativeStats> || </intervalStats> */
-        ret = xmlTextWriterEndElement (writer);
-        XML_RET_CHECK_AND_GOTO (ret, out);
+        ret = xmlTextWriterWriteFormatElement(writer, (xmlChar *)"name", "%s",
+                                              gf_fop_list[i]);
+        XML_RET_CHECK_AND_GOTO(ret, out);
+
+        ret = xmlTextWriterWriteFormatElement(writer, (xmlChar *)"hits",
+                                              "%" PRIu64, hits);
+        XML_RET_CHECK_AND_GOTO(ret, out);
+
+        ret = xmlTextWriterWriteFormatElement(writer, (xmlChar *)"avgLatency",
+                                              "%f", avg_latency);
+        XML_RET_CHECK_AND_GOTO(ret, out);
+
+        ret = xmlTextWriterWriteFormatElement(writer, (xmlChar *)"minLatency",
+                                              "%f", min_latency);
+        XML_RET_CHECK_AND_GOTO(ret, out);
+
+        ret = xmlTextWriterWriteFormatElement(writer, (xmlChar *)"maxLatency",
+                                              "%f", max_latency);
+        XML_RET_CHECK_AND_GOTO(ret, out);
+
+        /* </fop> */
+        ret = xmlTextWriterEndElement(writer);
+        XML_RET_CHECK_AND_GOTO(ret, out);
+    }
+
+    /* </fopStats> */
+    ret = xmlTextWriterEndElement(writer);
+    XML_RET_CHECK_AND_GOTO(ret, out);
+
+    snprintf(key, sizeof(key), "%d-%d-duration", brick_index, interval);
+    ret = dict_get_uint64(dict, key, &duration);
+    if (ret)
+        goto out;
+    ret = xmlTextWriterWriteFormatElement(writer, (xmlChar *)"duration",
+                                          "%" PRIu64, duration);
+    XML_RET_CHECK_AND_GOTO(ret, out);
+
+    snprintf(key, sizeof(key), "%d-%d-total-read", brick_index, interval);
+    ret = dict_get_uint64(dict, key, &total_read);
+    if (ret)
+        goto out;
+    ret = xmlTextWriterWriteFormatElement(writer, (xmlChar *)"totalRead",
+                                          "%" PRIu64, total_read);
+    XML_RET_CHECK_AND_GOTO(ret, out);
+
+    snprintf(key, sizeof(key), "%d-%d-total-write", brick_index, interval);
+    ret = dict_get_uint64(dict, key, &total_write);
+    if (ret)
+        goto out;
+    ret = xmlTextWriterWriteFormatElement(writer, (xmlChar *)"totalWrite",
+                                          "%" PRIu64, total_write);
+    XML_RET_CHECK_AND_GOTO(ret, out);
+
+    /* </cumulativeStats> || </intervalStats> */
+    ret = xmlTextWriterEndElement(writer);
+    XML_RET_CHECK_AND_GOTO(ret, out);
 
 out:
-        gf_log ("cli", GF_LOG_DEBUG, "Returning %d", ret);
-        return ret;
+    gf_log("cli", GF_LOG_DEBUG, "Returning %d", ret);
+    return ret;
 }
 #endif
 
 int
-cli_xml_output_vol_profile (dict_t *dict, int op_ret, int op_errno,
-                            char *op_errstr)
+cli_xml_output_vol_profile(dict_t *dict, int op_ret, int op_errno,
+                           char *op_errstr)
 {
 #if (HAVE_LIB_XML)
-        int                     ret = -1;
-        xmlTextWriterPtr        writer = NULL;
-        xmlDocPtr               doc = NULL;
-        char                    *volname = NULL;
-        int                     op = GF_CLI_STATS_NONE;
-        int                     info_op = GF_CLI_INFO_NONE;
-        int                     brick_count = 0;
-        char                    *brick_name = NULL;
-        int                     interval = 0;
-        char                    key[1024] = {0,};
-        int                     i = 0;
-        int                     stats_cleared = 0;
-
-        ret = cli_begin_xml_output (&writer, &doc);
-        if (ret)
-                goto out;
-
-        ret = cli_xml_output_common (writer, op_ret, op_errno, op_errstr);
-        if (ret)
-                goto out;
-
-        /* <volProfile> */
-        ret = xmlTextWriterStartElement (writer, (xmlChar *)"volProfile");
-        XML_RET_CHECK_AND_GOTO (ret, out);
-
-        ret = dict_get_str (dict, "volname", &volname);
-        if (ret)
-                goto out;
-        ret = xmlTextWriterWriteFormatElement (writer, (xmlChar *)"volname",
-                                               "%s", volname);
-        XML_RET_CHECK_AND_GOTO (ret, out);
-
-        ret = dict_get_int32 (dict, "op", &op);
-        if (ret)
-                goto out;
-        ret = xmlTextWriterWriteFormatElement (writer, (xmlChar *)"profileOp",
-                                               "%d", op);
-        XML_RET_CHECK_AND_GOTO (ret, out);
-
-        if (GF_CLI_STATS_INFO != op)
-                goto cont;
+    int ret = -1;
+    xmlTextWriterPtr writer = NULL;
+    xmlDocPtr doc = NULL;
+    char *volname = NULL;
+    int op = GF_CLI_STATS_NONE;
+    int info_op = GF_CLI_INFO_NONE;
+    int brick_count = 0;
+    char *brick_name = NULL;
+    int interval = 0;
+    char key[1024] = {
+        0,
+    };
+    int i = 0;
+    int stats_cleared = 0;
+
+    ret = cli_begin_xml_output(&writer, &doc);
+    if (ret)
+        goto out;
+
+    ret = cli_xml_output_common(writer, op_ret, op_errno, op_errstr);
+    if (ret)
+        goto out;
+
+    /* <volProfile> */
+    ret = xmlTextWriterStartElement(writer, (xmlChar *)"volProfile");
+    XML_RET_CHECK_AND_GOTO(ret, out);
+
+    ret = dict_get_str(dict, "volname", &volname);
+    if (ret)
+        goto out;
+    ret = xmlTextWriterWriteFormatElement(writer, (xmlChar *)"volname", "%s",
+                                          volname);
+    XML_RET_CHECK_AND_GOTO(ret, out);
+
+    ret = dict_get_int32(dict, "op", &op);
+    if (ret)
+        goto out;
+    ret = xmlTextWriterWriteFormatElement(writer, (xmlChar *)"profileOp", "%d",
+                                          op);
+    XML_RET_CHECK_AND_GOTO(ret, out);
+
+    if (GF_CLI_STATS_INFO != op)
+        goto cont;
+
+    ret = dict_get_int32(dict, "count", &brick_count);
+    if (ret)
+        goto out;
+    ret = xmlTextWriterWriteFormatElement(writer, (xmlChar *)"brickCount", "%d",
+                                          brick_count);
+    XML_RET_CHECK_AND_GOTO(ret, out);
+
+    ret = dict_get_int32(dict, "info-op", &info_op);
+    if (ret)
+        goto out;
+
+    while (i < brick_count) {
+        i++;
+
+        /* <brick> */
+        ret = xmlTextWriterStartElement(writer, (xmlChar *)"brick");
+        XML_RET_CHECK_AND_GOTO(ret, out);
 
-        ret = dict_get_int32 (dict, "count", &brick_count);
+        snprintf(key, sizeof(key), "%d-brick", i);
+        ret = dict_get_str(dict, key, &brick_name);
         if (ret)
-                goto out;
-        ret = xmlTextWriterWriteFormatElement (writer, (xmlChar *)"brickCount",
-                                               "%d", brick_count);
-        XML_RET_CHECK_AND_GOTO (ret, out);
+            goto out;
+        ret = xmlTextWriterWriteFormatElement(writer, (xmlChar *)"brickName",
+                                              "%s", brick_name);
+        XML_RET_CHECK_AND_GOTO(ret, out);
 
-        ret = dict_get_int32 (dict, "info-op", &info_op);
-        if (ret)
+        if (GF_CLI_INFO_CLEAR == info_op) {
+            snprintf(key, sizeof(key), "%d-stats-cleared", i);
+            ret = dict_get_int32(dict, key, &stats_cleared);
+            if (ret)
                 goto out;
 
-        while (i < brick_count) {
-                i++;
-
-                /* <brick> */
-                ret = xmlTextWriterStartElement (writer, (xmlChar *)"brick");
-                XML_RET_CHECK_AND_GOTO (ret, out);
-
-                snprintf (key, sizeof (key), "%d-brick", i);
-                ret = dict_get_str (dict, key, &brick_name);
+            ret = xmlTextWriterWriteFormatElement(
+                writer, (xmlChar *)"clearStats", "%s",
+                stats_cleared ? "Cleared stats." : "Failed to clear stats.");
+            XML_RET_CHECK_AND_GOTO(ret, out);
+        } else {
+            snprintf(key, sizeof(key), "%d-cumulative", i);
+            ret = dict_get_int32(dict, key, &interval);
+            if (ret == 0) {
+                ret = cli_xml_output_vol_profile_stats(writer, dict, i,
+                                                       interval);
                 if (ret)
-                        goto out;
-                ret = xmlTextWriterWriteFormatElement
-                        (writer, (xmlChar *)"brickName", "%s", brick_name);
-                XML_RET_CHECK_AND_GOTO (ret, out);
-
-                if (GF_CLI_INFO_CLEAR == info_op) {
-                        snprintf (key, sizeof (key), "%d-stats-cleared", i);
-                        ret = dict_get_int32 (dict, key, &stats_cleared);
-                        if (ret)
-                                goto out;
-
-                        ret = xmlTextWriterWriteFormatElement
-                                (writer, (xmlChar *)"clearStats", "%s",
-                                stats_cleared ? "Cleared stats." :
-                                                 "Failed to clear stats.");
-                        if (ret)
-                                goto out;
-                } else {
-                        snprintf (key, sizeof (key), "%d-cumulative", i);
-                        ret = dict_get_int32 (dict, key, &interval);
-                        if (ret == 0) {
-                                ret = cli_xml_output_vol_profile_stats
-                                        (writer, dict, i, interval);
-                                if (ret)
-                                        goto out;
-                        }
-
-                        memset (key, 0, sizeof (key));
-                        snprintf (key, sizeof (key), "%d-interval", i);
-                        ret = dict_get_int32 (dict, key, &interval);
-                        if (ret == 0) {
-                                ret = cli_xml_output_vol_profile_stats
-                                        (writer, dict, i, interval);
-                                if (ret)
-                                        goto out;
-                        }
-                }
-
-                /* </brick> */
-                ret = xmlTextWriterEndElement (writer);
-                XML_RET_CHECK_AND_GOTO (ret, out);
+                    goto out;
+            }
+
+            snprintf(key, sizeof(key), "%d-interval", i);
+            ret = dict_get_int32(dict, key, &interval);
+            if (ret == 0) {
+                ret = cli_xml_output_vol_profile_stats(writer, dict, i,
+                                                       interval);
+                if (ret)
+                    goto out;
+            }
         }
 
+        /* </brick> */
+        ret = xmlTextWriterEndElement(writer);
+        XML_RET_CHECK_AND_GOTO(ret, out);
+    }
+
 cont:
-        /* </volProfile> */
-        ret = xmlTextWriterEndElement (writer);
-        XML_RET_CHECK_AND_GOTO (ret, out);
+    /* </volProfile> */
+    ret = xmlTextWriterEndElement(writer);
+    XML_RET_CHECK_AND_GOTO(ret, out);
 
-        ret = cli_end_xml_output (writer, doc);
+    ret = cli_end_xml_output(writer, doc);
 out:
-        gf_log ("cli", GF_LOG_DEBUG, "Returning %d", ret);
-        return ret;
+    gf_log("cli", GF_LOG_DEBUG, "Returning %d", ret);
+    return ret;
 #else
-        return 0;
+    return 0;
 #endif
 }
 
 int
-cli_xml_output_vol_list (dict_t *dict, int op_ret, int op_errno,
-                         char *op_errstr)
+cli_xml_output_vol_list(dict_t *dict, int op_ret, int op_errno, char *op_errstr)
 {
 #if (HAVE_LIB_XML)
-        int                     ret = -1;
-        xmlTextWriterPtr        writer = NULL;
-        xmlDocPtr               doc = NULL;
-        int                     count = 0;
-        char                    *volname = NULL;
-        char                    key[1024] = {0,};
-        int                     i = 0;
-
-        ret = cli_begin_xml_output (&writer, &doc);
+    int ret = -1;
+    xmlTextWriterPtr writer = NULL;
+    xmlDocPtr doc = NULL;
+    int count = 0;
+    char *volname = NULL;
+    char key[1024] = {
+        0,
+    };
+    int i = 0;
+
+    ret = cli_begin_xml_output(&writer, &doc);
+    if (ret)
+        goto out;
+
+    ret = cli_xml_output_common(writer, op_ret, op_errno, op_errstr);
+    if (ret)
+        goto out;
+
+    /* <volList> */
+    ret = xmlTextWriterStartElement(writer, (xmlChar *)"volList");
+    XML_RET_CHECK_AND_GOTO(ret, out);
+
+    ret = dict_get_int32(dict, "count", &count);
+    if (ret)
+        goto out;
+    ret = xmlTextWriterWriteFormatElement(writer, (xmlChar *)"count", "%d",
+                                          count);
+    XML_RET_CHECK_AND_GOTO(ret, out);
+
+    for (i = 0; i < count; i++) {
+        snprintf(key, sizeof(key), "volume%d", i);
+        ret = dict_get_str(dict, key, &volname);
         if (ret)
-                goto out;
-
-        ret = cli_xml_output_common (writer, op_ret, op_errno, op_errstr);
-        if (ret)
-                goto out;
-
-        /* <volList> */
-        ret = xmlTextWriterStartElement (writer, (xmlChar *)"volList");
-        XML_RET_CHECK_AND_GOTO (ret, out);
-
-        ret = dict_get_int32 (dict, "count", &count);
-        if (ret)
-                goto out;
-        ret = xmlTextWriterWriteFormatElement (writer, (xmlChar *)"count",
-                                               "%d", count);
-        XML_RET_CHECK_AND_GOTO (ret, out);
-
-        for (i = 0; i < count; i++) {
-                memset (key, 0, sizeof (key));
-                snprintf (key, sizeof (key), "volume%d", i);
-                ret = dict_get_str (dict, key, &volname);
-                if (ret)
-                        goto out;
-                ret = xmlTextWriterWriteFormatElement (writer,
-                                                       (xmlChar *)"volume",
-                                                       "%s", volname);
-                XML_RET_CHECK_AND_GOTO (ret, out);
-        }
+            goto out;
+        ret = xmlTextWriterWriteFormatElement(writer, (xmlChar *)"volume", "%s",
+                                              volname);
+        XML_RET_CHECK_AND_GOTO(ret, out);
+    }
 
-        /* </volList> */
-        ret = xmlTextWriterEndElement (writer);
-        XML_RET_CHECK_AND_GOTO (ret, out);
+    /* </volList> */
+    ret = xmlTextWriterEndElement(writer);
+    XML_RET_CHECK_AND_GOTO(ret, out);
 
-        ret = cli_end_xml_output (writer, doc);
+    ret = cli_end_xml_output(writer, doc);
 out:
-        gf_log ("cli", GF_LOG_DEBUG, "Returning %d", ret);
-        return ret;
+    gf_log("cli", GF_LOG_DEBUG, "Returning %d", ret);
+    return ret;
 #else
-        return 0;
+    return 0;
 #endif
 }
 
 #if (HAVE_LIB_XML)
 int
-cli_xml_output_vol_info_option (xmlTextWriterPtr writer, char *substr,
-                                char *optstr, char *valstr)
+cli_xml_output_vol_info_option(xmlTextWriterPtr writer, char *substr,
+                               char *optstr, char *valstr)
 {
-        int             ret = -1;
-        char            *ptr1 = NULL;
-        char            *ptr2 = NULL;
-
-        ptr1 = substr;
-        ptr2 = optstr;
-
-        while (ptr1) {
-                if (*ptr1 != *ptr2)
-                        break;
-                ptr1++;
-                ptr2++;
-                if (!ptr1)
-                        goto out;
-                if (!ptr2)
-                        goto out;
-        }
-        if (*ptr2 == '\0')
-                goto out;
-
-        /* <option> */
-        ret = xmlTextWriterStartElement (writer, (xmlChar *)"option");
-        XML_RET_CHECK_AND_GOTO (ret, out);
-
-        ret = xmlTextWriterWriteFormatElement (writer, (xmlChar *)"name",
-                                               "%s", ptr2);
-        XML_RET_CHECK_AND_GOTO (ret, out);
-
-        ret = xmlTextWriterWriteFormatElement (writer, (xmlChar *)"value",
-                                               "%s", valstr);
-        XML_RET_CHECK_AND_GOTO (ret, out);
-
-        /* </option> */
-        ret = xmlTextWriterEndElement (writer);
-        XML_RET_CHECK_AND_GOTO (ret, out);
+    int ret = -1;
+    char *ptr1 = NULL;
+    char *ptr2 = NULL;
+
+    ptr1 = substr;
+    ptr2 = optstr;
+
+    while (ptr1) {
+        if (*ptr1 != *ptr2)
+            break;
+        ptr1++;
+        ptr2++;
+        if (!*ptr1)
+            break;
+        if (!*ptr2)
+            break;
+    }
+    if (*ptr2 == '\0')
+        goto out;
+
+    /* <option> */
+    ret = xmlTextWriterStartElement(writer, (xmlChar *)"option");
+    XML_RET_CHECK_AND_GOTO(ret, out);
+
+    ret = xmlTextWriterWriteFormatElement(writer, (xmlChar *)"name", "%s",
+                                          ptr2);
+    XML_RET_CHECK_AND_GOTO(ret, out);
+
+    ret = xmlTextWriterWriteFormatElement(writer, (xmlChar *)"value", "%s",
+                                          valstr);
+    XML_RET_CHECK_AND_GOTO(ret, out);
+
+    /* </option> */
+    ret = xmlTextWriterEndElement(writer);
+    XML_RET_CHECK_AND_GOTO(ret, out);
 out:
-        gf_log ("cli", GF_LOG_DEBUG, "Returning %d", ret);
-        return ret;
+    gf_log("cli", GF_LOG_DEBUG, "Returning %d", ret);
+    return ret;
 }
 
 struct tmp_xml_option_logger {
-        char *key;
-        xmlTextWriterPtr writer;
+    char *key;
+    xmlTextWriterPtr writer;
 };
 
 static int
-_output_vol_info_option (dict_t *d, char *k, data_t *v,
-                         void *data)
+_output_vol_info_option(dict_t *d, char *k, data_t *v, void *data)
 {
-        int                           ret   = 0;
-        char                         *ptr   = NULL;
-        struct tmp_xml_option_logger *tmp   = NULL;
+    int ret = 0;
+    char *ptr = NULL;
+    struct tmp_xml_option_logger *tmp = NULL;
 
-        tmp = data;
+    tmp = data;
 
-        ptr = strstr (k, "option.");
-        if (!ptr)
-                goto out;
+    ptr = strstr(k, "option.");
+    if (!ptr)
+        goto out;
 
-        if (!v) {
-                ret = -1;
-                goto out;
-        }
-        ret = cli_xml_output_vol_info_option (tmp->writer, tmp->key, k,
-                                              v->data);
+    if (!v) {
+        ret = -1;
+        goto out;
+    }
+    ret = cli_xml_output_vol_info_option(tmp->writer, tmp->key, k, v->data);
 
 out:
-        return ret;
+    return ret;
 }
 
 int
-cli_xml_output_vol_info_options (xmlTextWriterPtr writer, dict_t *dict,
-                                 char *prefix)
+cli_xml_output_vol_info_options(xmlTextWriterPtr writer, dict_t *dict,
+                                char *prefix)
 {
-        int             ret = -1;
-        int             opt_count = 0;
-        char            key[1024] = {0,};
-        struct tmp_xml_option_logger tmp = {0,};
-
-        snprintf (key, sizeof (key), "%s.opt_count", prefix);
-        ret = dict_get_int32 (dict, key, &opt_count);
-        if (ret)
-                goto out;
-        ret = xmlTextWriterWriteFormatElement (writer, (xmlChar *)"optCount",
-                                               "%d", opt_count);
-        XML_RET_CHECK_AND_GOTO (ret, out);
-
-        /* <options> */
-        ret = xmlTextWriterStartElement (writer, (xmlChar *)"options");
-        XML_RET_CHECK_AND_GOTO (ret, out);
-        snprintf (key, sizeof (key), "%s.option.", prefix);
-
-        tmp.key = key;
-        tmp.writer = writer;
-        ret = dict_foreach (dict, _output_vol_info_option, &tmp);
-        if (ret)
-                goto out;
-
-        /* </options> */
-        ret = xmlTextWriterEndElement (writer);
-        XML_RET_CHECK_AND_GOTO (ret, out);
+    int ret = -1;
+    int opt_count = 0;
+    char key[1024] = {
+        0,
+    };
+    struct tmp_xml_option_logger tmp = {
+        0,
+    };
+
+    snprintf(key, sizeof(key), "%s.opt_count", prefix);
+    ret = dict_get_int32(dict, key, &opt_count);
+    if (ret)
+        goto out;
+    ret = xmlTextWriterWriteFormatElement(writer, (xmlChar *)"optCount", "%d",
+                                          opt_count);
+    XML_RET_CHECK_AND_GOTO(ret, out);
+
+    /* <options> */
+    ret = xmlTextWriterStartElement(writer, (xmlChar *)"options");
+    XML_RET_CHECK_AND_GOTO(ret, out);
+    snprintf(key, sizeof(key), "%s.option.", prefix);
+
+    tmp.key = key;
+    tmp.writer = writer;
+    ret = dict_foreach(dict, _output_vol_info_option, &tmp);
+    if (ret)
+        goto out;
+
+    /* </options> */
+    ret = xmlTextWriterEndElement(writer);
+    XML_RET_CHECK_AND_GOTO(ret, out);
 out:
-        gf_log ("cli", GF_LOG_DEBUG, "Returning %d", ret);
-        return ret;
+    gf_log("cli", GF_LOG_DEBUG, "Returning %d", ret);
+    return ret;
 }
 #endif
 
 int
-cli_xml_output_vol_info (cli_local_t *local, dict_t *dict)
+cli_xml_output_vol_info(cli_local_t *local, dict_t *dict)
 {
 #if (HAVE_LIB_XML)
-        int                     ret = 0;
-        int                     count = 0;
-        char                    *volname = NULL;
-        char                    *volume_id = NULL;
-        char                    *uuid = NULL;
-        int                     type = 0;
-        int                     status = 0;
-        int                     brick_count = 0;
-        int                     dist_count = 0;
-        int                     stripe_count = 0;
-        int                     replica_count = 0;
-        int                     disperse_count = 0;
-        int                     redundancy_count = 0;
-        int                     transport = 0;
-        char                    *brick = NULL;
-        char                    key[1024] = {0,};
-        int                     i = 0;
-        int                     j = 1;
-        char                    *caps = NULL;
-        int                     k __attribute__((unused)) = 0;
-        int                     index = 1;
-        int                     vol_type               = -1;
-        int                     tier_vol_type          = 0;
-        /*hot disperse count, redundancy count and dist count are always
-         * zero so need for them to be included in the array.*/
-        int                     hot_disperse_count     = 0;
-        int                     hot_dist_count         = 0;
-        int                     hot_redundancy_count   = 0;
-        values                  c                      = 0;
-        char                    *keys[30]              = {
-                [COLD_BRICK_COUNT]      = "volume%d.cold_brick_count",
-                [COLD_TYPE]             = "volume%d.cold_type",
-                [COLD_DIST_COUNT]       = "volume%d.cold_dist_count",
-                [COLD_REPLICA_COUNT]    = "volume%d.cold_replica_count",
-                [COLD_DISPERSE_COUNT]   = "volume%d.cold_disperse_count",
-                [COLD_REDUNDANCY_COUNT] = "volume%d.cold_redundancy_count",
-                [HOT_BRICK_COUNT]       = "volume%d.hot_brick_count",
-                [HOT_TYPE]              = "volume%d.hot_type",
-                [HOT_REPLICA_COUNT]     = "volume%d.hot_replica_count"};
-        int                     value[9]             = {};
-
-
-        ret = dict_get_int32 (dict, "count", &count);
+    int ret = 0;
+    int count = 0;
+    char *volname = NULL;
+    char *volume_id = NULL;
+    char *uuid = NULL;
+    int type = 0;
+    int status = 0;
+    int brick_count = 0;
+    int dist_count = 0;
+    int stripe_count = 0;
+    int replica_count = 0;
+    int arbiter_count = 0;
+    int snap_count = 0;
+    int isArbiter = 0;
+    int disperse_count = 0;
+    int redundancy_count = 0;
+    int transport = 0;
+    char *brick = NULL;
+    char key[1024] = {
+        0,
+    };
+    int i = 0;
+    int j = 1;
+    char *caps __attribute__((unused)) = NULL;
+    int k __attribute__((unused)) = 0;
+
+    ret = dict_get_int32(dict, "count", &count);
+    if (ret)
+        goto out;
+
+    for (i = 0; i < count; i++) {
+        /* <volume> */
+        ret = xmlTextWriterStartElement(local->writer, (xmlChar *)"volume");
+        XML_RET_CHECK_AND_GOTO(ret, out);
+
+        snprintf(key, sizeof(key), "volume%d.name", i);
+        ret = dict_get_str(dict, key, &volname);
+        if (ret)
+            goto out;
+        ret = xmlTextWriterWriteFormatElement(local->writer, (xmlChar *)"name",
+                                              "%s", volname);
+        XML_RET_CHECK_AND_GOTO(ret, out);
+
+        snprintf(key, sizeof(key), "volume%d.volume_id", i);
+        ret = dict_get_str(dict, key, &volume_id);
+        if (ret)
+            goto out;
+        ret = xmlTextWriterWriteFormatElement(local->writer, (xmlChar *)"id",
+                                              "%s", volume_id);
+        XML_RET_CHECK_AND_GOTO(ret, out);
+
+        snprintf(key, sizeof(key), "volume%d.status", i);
+        ret = dict_get_int32(dict, key, &status);
+        if (ret)
+            goto out;
+        ret = xmlTextWriterWriteFormatElement(
+            local->writer, (xmlChar *)"status", "%d", status);
+        XML_RET_CHECK_AND_GOTO(ret, out);
+
+        ret = xmlTextWriterWriteFormatElement(local->writer,
+                                              (xmlChar *)"statusStr", "%s",
+                                              cli_vol_status_str[status]);
+        XML_RET_CHECK_AND_GOTO(ret, out);
+
+        snprintf(key, sizeof(key), "volume%d.snap_count", i);
+        ret = dict_get_int32(dict, key, &snap_count);
+        if (ret)
+            goto out;
+        ret = xmlTextWriterWriteFormatElement(
+            local->writer, (xmlChar *)"snapshotCount", "%d", snap_count);
+        XML_RET_CHECK_AND_GOTO(ret, out);
+
+        snprintf(key, sizeof(key), "volume%d.brick_count", i);
+        ret = dict_get_int32(dict, key, &brick_count);
+        if (ret)
+            goto out;
+        ret = xmlTextWriterWriteFormatElement(
+            local->writer, (xmlChar *)"brickCount", "%d", brick_count);
+        XML_RET_CHECK_AND_GOTO(ret, out);
+
+        snprintf(key, sizeof(key), "volume%d.dist_count", i);
+        ret = dict_get_int32(dict, key, &dist_count);
+        if (ret)
+            goto out;
+        ret = xmlTextWriterWriteFormatElement(local->writer,
+                                              (xmlChar *)"distCount", "%d",
+                                              (brick_count / dist_count));
+        XML_RET_CHECK_AND_GOTO(ret, out);
+
+        snprintf(key, sizeof(key), "volume%d.stripe_count", i);
+        ret = dict_get_int32(dict, key, &stripe_count);
         if (ret)
+            goto out;
+        ret = xmlTextWriterWriteFormatElement(
+            local->writer, (xmlChar *)"stripeCount", "%d", stripe_count);
+        XML_RET_CHECK_AND_GOTO(ret, out);
+
+        snprintf(key, sizeof(key), "volume%d.replica_count", i);
+        ret = dict_get_int32(dict, key, &replica_count);
+        if (ret)
+            goto out;
+        ret = xmlTextWriterWriteFormatElement(
+            local->writer, (xmlChar *)"replicaCount", "%d", replica_count);
+        XML_RET_CHECK_AND_GOTO(ret, out);
+
+        snprintf(key, sizeof(key), "volume%d.arbiter_count", i);
+        ret = dict_get_int32(dict, key, &arbiter_count);
+        if (ret)
+            goto out;
+        ret = xmlTextWriterWriteFormatElement(
+            local->writer, (xmlChar *)"arbiterCount", "%d", arbiter_count);
+        XML_RET_CHECK_AND_GOTO(ret, out);
+
+        snprintf(key, sizeof(key), "volume%d.disperse_count", i);
+        ret = dict_get_int32(dict, key, &disperse_count);
+        if (ret)
+            goto out;
+        ret = xmlTextWriterWriteFormatElement(
+            local->writer, (xmlChar *)"disperseCount", "%d", disperse_count);
+        XML_RET_CHECK_AND_GOTO(ret, out);
+
+        snprintf(key, sizeof(key), "volume%d.redundancy_count", i);
+        ret = dict_get_int32(dict, key, &redundancy_count);
+        if (ret)
+            goto out;
+        ret = xmlTextWriterWriteFormatElement(local->writer,
+                                              (xmlChar *)"redundancyCount",
+                                              "%d", redundancy_count);
+        XML_RET_CHECK_AND_GOTO(ret, out);
+
+        snprintf(key, sizeof(key), "volume%d.type", i);
+        ret = dict_get_int32(dict, key, &type);
+        if (ret)
+            goto out;
+        /* For Distributed-(stripe,replicate,stipe-replicate,disperse)
+           types
+         */
+        type = get_vol_type(type, dist_count, brick_count);
+
+        ret = xmlTextWriterWriteFormatElement(local->writer, (xmlChar *)"type",
+                                              "%d", type);
+        XML_RET_CHECK_AND_GOTO(ret, out);
+
+        ret = xmlTextWriterWriteFormatElement(
+            local->writer, (xmlChar *)"typeStr", "%s", vol_type_str[type]);
+        XML_RET_CHECK_AND_GOTO(ret, out);
+
+        snprintf(key, sizeof(key), "volume%d.transport", i);
+        ret = dict_get_int32(dict, key, &transport);
+        if (ret)
+            goto out;
+        ret = xmlTextWriterWriteFormatElement(
+            local->writer, (xmlChar *)"transport", "%d", transport);
+        XML_RET_CHECK_AND_GOTO(ret, out);
+
+        j = 1;
+
+        /* <bricks> */
+        ret = xmlTextWriterStartElement(local->writer, (xmlChar *)"bricks");
+        XML_RET_CHECK_AND_GOTO(ret, out);
+
+        while (j <= brick_count) {
+            ret = xmlTextWriterStartElement(local->writer, (xmlChar *)"brick");
+            XML_RET_CHECK_AND_GOTO(ret, out);
+
+            snprintf(key, sizeof(key), "volume%d.brick%d.uuid", i, j);
+            ret = dict_get_str(dict, key, &uuid);
+            if (ret)
                 goto out;
+            ret = xmlTextWriterWriteFormatAttribute(
+                local->writer, (xmlChar *)"uuid", "%s", uuid);
+            XML_RET_CHECK_AND_GOTO(ret, out);
 
-        for (i = 0; i < count; i++) {
-                /* <volume> */
-                ret = xmlTextWriterStartElement (local->writer,
-                                                 (xmlChar *)"volume");
-                XML_RET_CHECK_AND_GOTO (ret, out);
+            snprintf(key, sizeof(key), "volume%d.brick%d", i, j);
+            ret = dict_get_str(dict, key, &brick);
+            if (ret)
+                goto out;
+            ret = xmlTextWriterWriteFormatString(local->writer, "%s", brick);
+            XML_RET_CHECK_AND_GOTO(ret, out);
 
-                memset (key, 0, sizeof (key));
-                snprintf (key, sizeof (key), "volume%d.name", i);
-                ret = dict_get_str (dict, key, &volname);
-                if (ret)
-                        goto out;
-                ret = xmlTextWriterWriteFormatElement (local->writer,
-                                                       (xmlChar *)"name",
-                                                       "%s", volname);
-                XML_RET_CHECK_AND_GOTO (ret, out);
-
-                memset (key, 0, sizeof (key));
-                snprintf (key, sizeof (key), "volume%d.volume_id", i);
-                ret = dict_get_str (dict, key, &volume_id);
-                if (ret)
-                        goto out;
-                ret = xmlTextWriterWriteFormatElement (local->writer,
-                                                       (xmlChar *)"id",
-                                                       "%s", volume_id);
-                XML_RET_CHECK_AND_GOTO (ret, out);
-
-                memset (key, 0, sizeof (key));
-                snprintf (key, sizeof (key), "volume%d.status", i);
-                ret = dict_get_int32 (dict, key, &status);
-                if (ret)
-                        goto out;
-                ret = xmlTextWriterWriteFormatElement (local->writer,
-                                                       (xmlChar *)"status",
-                                                       "%d", status);
-                XML_RET_CHECK_AND_GOTO (ret, out);
-
-                ret =xmlTextWriterWriteFormatElement
-                        (local->writer, (xmlChar *)"statusStr", "%s",
-                         cli_vol_status_str[status]);
-                XML_RET_CHECK_AND_GOTO (ret, out);
-
-                memset (key, 0, sizeof (key));
-                snprintf (key, sizeof (key), "volume%d.brick_count", i);
-                ret = dict_get_int32 (dict, key, &brick_count);
-                if (ret)
-                        goto out;
-                ret = xmlTextWriterWriteFormatElement (local->writer,
-                                                       (xmlChar *)"brickCount",
-                                                       "%d", brick_count);
-                XML_RET_CHECK_AND_GOTO (ret, out);
-
-                memset (key, 0, sizeof (key));
-                snprintf (key, sizeof (key), "volume%d.dist_count", i);
-                ret = dict_get_int32 (dict, key, &dist_count);
-                if (ret)
-                        goto out;
-                ret = xmlTextWriterWriteFormatElement (local->writer,
-                                                       (xmlChar *)"distCount",
-                                                       "%d", dist_count);
-                XML_RET_CHECK_AND_GOTO (ret, out);
-
-                memset (key, 0, sizeof (key));
-                snprintf (key, sizeof (key), "volume%d.stripe_count", i);
-                ret = dict_get_int32 (dict, key, &stripe_count);
-                if (ret)
-                        goto out;
-                ret = xmlTextWriterWriteFormatElement (local->writer,
-                                                       (xmlChar *)"stripeCount",
-                                                       "%d", stripe_count);
-                XML_RET_CHECK_AND_GOTO (ret, out);
-
-                memset (key, 0, sizeof (key));
-                snprintf (key, sizeof (key), "volume%d.replica_count", i);
-                ret = dict_get_int32 (dict, key, &replica_count);
-                if (ret)
-                        goto out;
-                ret = xmlTextWriterWriteFormatElement (local->writer,
-                                                       (xmlChar *)"replicaCount",
-                                                       "%d", replica_count);
-                XML_RET_CHECK_AND_GOTO (ret, out);
-
-                memset (key, 0, sizeof (key));
-                snprintf (key, sizeof (key), "volume%d.disperse_count", i);
-                ret = dict_get_int32 (dict, key, &disperse_count);
-                if (ret)
-                        goto out;
-                ret = xmlTextWriterWriteFormatElement (local->writer,
-                                                       (xmlChar *)"disperseCount",
-                                                       "%d", disperse_count);
-                XML_RET_CHECK_AND_GOTO (ret, out);
-
-                memset (key, 0, sizeof (key));
-                snprintf (key, sizeof (key), "volume%d.redundancy_count", i);
-                ret = dict_get_int32 (dict, key, &redundancy_count);
-                if (ret)
-                        goto out;
-                ret = xmlTextWriterWriteFormatElement (local->writer,
-                                                       (xmlChar *)"redundancyCount",
-                                                       "%d", redundancy_count);
-                XML_RET_CHECK_AND_GOTO (ret, out);
-
-                memset (key, 0, sizeof (key));
-                snprintf (key, sizeof (key), "volume%d.type", i);
-                ret = dict_get_int32 (dict, key, &type);
-                if (ret)
-                        goto out;
-                /* For Distributed-(stripe,replicate,stipe-replicate,disperse)
-                   types
-                 */
-                if ((type != GF_CLUSTER_TYPE_TIER) && (type > 0) &&
-                    (dist_count < brick_count))
-                       type = type + GF_CLUSTER_TYPE_MAX - 1;
-
-                ret = xmlTextWriterWriteFormatElement (local->writer,
-                                                       (xmlChar *)"type",
-                                                       "%d", type);
-                XML_RET_CHECK_AND_GOTO (ret, out);
-
-                ret = xmlTextWriterWriteFormatElement (local->writer,
-                                                       (xmlChar *)"typeStr",
-                                                       "%s",
-                                                       cli_vol_type_str[type]);
-                XML_RET_CHECK_AND_GOTO (ret, out);
-
-                memset (key, 0, sizeof (key));
-                snprintf (key, sizeof (key), "volume%d.transport", i);
-                ret = dict_get_int32 (dict, key, &transport);
-                if (ret)
-                        goto out;
-                ret = xmlTextWriterWriteFormatElement (local->writer,
-                                                       (xmlChar *)"transport",
-                                                       "%d", transport);
-                XML_RET_CHECK_AND_GOTO (ret, out);
-
-#ifdef HAVE_BD_XLATOR
-                /* <xlators> */
-                ret = xmlTextWriterStartElement (local->writer,
-                                                 (xmlChar *)"xlators");
-                XML_RET_CHECK_AND_GOTO (ret, out);
-
-                for (k = 0; ; k++) {
-                        memset (key, 0, sizeof (key));
-                        snprintf (key, sizeof (key),"volume%d.xlator%d", i, k);
-                        ret = dict_get_str (dict, key, &caps);
-                        if (ret)
-                                break;
-
-                        /* <xlator> */
-                        ret = xmlTextWriterStartElement (local->writer,
-                                                         (xmlChar *)"xlator");
-                        XML_RET_CHECK_AND_GOTO (ret, out);
-
-                        ret = xmlTextWriterWriteFormatElement
-                                (local->writer, (xmlChar *)"name", "%s", caps);
-                        XML_RET_CHECK_AND_GOTO (ret, out);
-
-                        /* <capabilities> */
-                        ret = xmlTextWriterStartElement (local->writer,
-                                                         (xmlChar *)
-                                                         "capabilities");
-                        XML_RET_CHECK_AND_GOTO (ret, out);
-
-                        j = 0;
-                        for (j = 0; ;j++) {
-                                memset (key, 0, sizeof (key));
-                                snprintf (key, sizeof (key),
-                                          "volume%d.xlator%d.caps%d", i, k, j);
-                                ret = dict_get_str (dict, key, &caps);
-                                if (ret)
-                                        break;
-                                ret = xmlTextWriterWriteFormatElement
-                                        (local->writer, (xmlChar *)"capability",
-                                         "%s", caps);
-                                XML_RET_CHECK_AND_GOTO (ret, out);
-                        }
-                        /* </capabilities> */
-                        ret = xmlTextWriterEndElement (local->writer);
-                        XML_RET_CHECK_AND_GOTO (ret, out);
-                        /* </xlator> */
-                        ret = xmlTextWriterEndElement (local->writer);
-                        XML_RET_CHECK_AND_GOTO (ret, out);
-                }
-                ret = xmlTextWriterFullEndElement (local->writer);
-                XML_RET_CHECK_AND_GOTO (ret, out);
-                /* </xlators> */
-#else
-                caps = 0; /* Avoid compiler warnings when BD not enabled */
-#endif
-                j = 1;
-
-                /* <bricks> */
-                ret = xmlTextWriterStartElement (local->writer,
-                                                 (xmlChar *)"bricks");
-                XML_RET_CHECK_AND_GOTO (ret, out);
-
-                if (type == GF_CLUSTER_TYPE_TIER) {
-                        /*the values for hot stripe, disperse and redundancy
-                         * should not be looped in here as they are zero
-                         * always */
-                        for (c = COLD_BRICK_COUNT; c < MAX; c++) {
-
-                                memset (key, 0, sizeof (key));
-                                snprintf (key, 256, keys[c], i);
-                                ret = dict_get_int32 (dict, key, &value[c]);
-                                if (ret)
-                                        goto out;
-                        }
-
-                        tier_vol_type = value[HOT_TYPE];
-                        hot_dist_count = (value[HOT_REPLICA_COUNT] ?
-                                          value[HOT_REPLICA_COUNT] : 1);
-                        if ((value[HOT_TYPE] != GF_CLUSTER_TYPE_TIER) &&
-                            (value[HOT_TYPE] > 0) &&
-                            (hot_dist_count < value[HOT_BRICK_COUNT]))
-                                tier_vol_type = value[HOT_TYPE] +
-                                        GF_CLUSTER_TYPE_MAX - 1;
-
-                        ret = xmlTextWriterStartElement (local->writer,
-                                                         (xmlChar *)
-                                                         "hotBricks");
-                        XML_RET_CHECK_AND_GOTO (ret, out);
-
-                        ret = xmlTextWriterWriteFormatElement
-                                (local->writer, (xmlChar *)"hotBrickType",
-                                 "%s", cli_vol_type_str[tier_vol_type]);
-
-                        ret = xmlTextWriterWriteFormatElement (local->writer,
-                                                   (xmlChar *)"hotreplicaCount",
-                                                    "%d",
-                                                    value[HOT_REPLICA_COUNT]);
-                        XML_RET_CHECK_AND_GOTO (ret, out);
-
-                        ret = xmlTextWriterWriteFormatElement (local->writer,
-                                                   (xmlChar *)"hotbrickCount",
-                                                    "%d",
-                                                    value[HOT_BRICK_COUNT]);
-                        XML_RET_CHECK_AND_GOTO (ret, out);
-
-                        if (value[HOT_TYPE] == GF_CLUSTER_TYPE_NONE ||
-                                        value[HOT_TYPE] ==
-                                        GF_CLUSTER_TYPE_TIER) {
-                                ret = xmlTextWriterWriteFormatElement
-                                        (local->writer,
-                                         (xmlChar *)"numberOfBricks",
-                                         "%d", value[HOT_BRICK_COUNT]);
-                                XML_RET_CHECK_AND_GOTO (ret, out);
-                        } else {
-                                ret = xmlTextWriterWriteFormatElement
-                                        (local->writer,
-                                         (xmlChar *)"numberOfBricks",
-                                         "%d x %d = %d",
-                                         (value[HOT_BRICK_COUNT] /
-                                          hot_dist_count),
-                                         hot_dist_count,
-                                         value[HOT_BRICK_COUNT]);
-                        }
-
-                        while (index <= value[HOT_BRICK_COUNT]) {
-                                snprintf (key, 1024, "volume%d.brick%d", i,
-                                          index);
-                                ret = dict_get_str (dict, key, &brick);
-                                if (ret)
-                                        goto out;
-
-                                ret = xmlTextWriterStartElement
-                                        (local->writer, (xmlChar *)"brick");
-                                XML_RET_CHECK_AND_GOTO (ret, out);
-
-                                memset (key, 0, sizeof (key));
-                                snprintf (key, sizeof (key),
-                                          "volume%d.brick%d.uuid", i, j);
-                                ret = dict_get_str (dict, key, &uuid);
-                                if (ret)
-                                        goto out;
-                                ret = xmlTextWriterWriteFormatAttribute
-                                        (local->writer, (xmlChar *)"uuid", "%s",
-                                         uuid);
-                                XML_RET_CHECK_AND_GOTO (ret, out);
-
-                                ret = xmlTextWriterWriteFormatString
-                                        (local->writer, "%s", brick);
-                                XML_RET_CHECK_AND_GOTO (ret, out);
-
-                                ret = xmlTextWriterWriteFormatElement
-                                        (local->writer, (xmlChar *)"name", "%s",
-                                         brick);
-                                XML_RET_CHECK_AND_GOTO (ret, out);
-
-                                ret = xmlTextWriterWriteFormatElement
-                                        (local->writer, (xmlChar *)"hostUuid",
-                                         "%s", uuid);
-                                XML_RET_CHECK_AND_GOTO (ret, out);
-
-                                ret = xmlTextWriterEndElement (local->writer);
-                                XML_RET_CHECK_AND_GOTO (ret, out);
-
-                                index++;
-                        }
-                        ret = xmlTextWriterEndElement (local->writer);
-                        XML_RET_CHECK_AND_GOTO (ret, out);
-
-                        tier_vol_type = value[COLD_TYPE];
-                        if ((value[COLD_TYPE] != GF_CLUSTER_TYPE_TIER) &&
-                            (value[COLD_TYPE] > 0) &&
-                            (value[COLD_DIST_COUNT] < value[COLD_BRICK_COUNT]))
-                                tier_vol_type = value[COLD_TYPE] +
-                                        GF_CLUSTER_TYPE_MAX - 1;
-
-
-                        ret = xmlTextWriterStartElement (local->writer,
-                                                         (xmlChar *)
-                                                         "coldBricks");
-                        XML_RET_CHECK_AND_GOTO (ret, out);
-
-                        ret = xmlTextWriterWriteFormatElement
-                                (local->writer, (xmlChar *)"coldBrickType",
-                                 "%s", cli_vol_type_str[tier_vol_type]);
-
-                        ret = xmlTextWriterWriteFormatElement (local->writer,
-                                        (xmlChar *)"coldreplicaCount",
-                                        "%d", value[COLD_REPLICA_COUNT]);
-                        XML_RET_CHECK_AND_GOTO (ret, out);
-
-                        ret = xmlTextWriterWriteFormatElement (local->writer,
-                                                   (xmlChar *)"coldbrickCount",
-                                                    "%d",
-                                                    value[COLD_BRICK_COUNT]);
-                        XML_RET_CHECK_AND_GOTO (ret, out);
-
-                        ret = xmlTextWriterWriteFormatElement (local->writer,
-                                              (xmlChar *)"colddisperseCount",
-                                              "%d", value[COLD_DISPERSE_COUNT]);
-                        XML_RET_CHECK_AND_GOTO (ret, out);
-
-                        if (value[COLD_TYPE] == GF_CLUSTER_TYPE_NONE ||
-                                        value[COLD_TYPE] ==
-                                        GF_CLUSTER_TYPE_TIER) {
-                                ret = xmlTextWriterWriteFormatElement
-                                        (local->writer,
-                                         (xmlChar *)"numberOfBricks",
-                                         "%d", value[COLD_BRICK_COUNT]);
-                                XML_RET_CHECK_AND_GOTO (ret, out);
-
-                        } else if (value[COLD_TYPE] ==
-                                        GF_CLUSTER_TYPE_DISPERSE) {
-                                ret = xmlTextWriterWriteFormatElement
-                                        (local->writer,
-                                         (xmlChar *)"numberOfBricks",
-                                         " %d x (%d + %d) = %d",
-                                         (value[COLD_BRICK_COUNT] /
-                                          value[COLD_DIST_COUNT]),
-                                         value[COLD_DISPERSE_COUNT] -
-                                         value[COLD_REDUNDANCY_COUNT],
-                                         value[COLD_REDUNDANCY_COUNT],
-                                         value[COLD_BRICK_COUNT]);
-                        } else {
-                                ret = xmlTextWriterWriteFormatElement
-                                        (local->writer,
-                                         (xmlChar *)"numberOfBricks",
-                                         "%d x %d = %d",
-                                         (value[COLD_BRICK_COUNT] /
-                                          value[COLD_DIST_COUNT]),
-                                         value[COLD_DIST_COUNT],
-                                         value[COLD_BRICK_COUNT]);
-                        }
-
-                        index = value[HOT_BRICK_COUNT] + 1;
-
-                        while (index <= brick_count) {
-                                snprintf (key, 1024, "volume%d.brick%d", i,
-                                          index);
-                                ret = dict_get_str (dict, key, &brick);
-                                if (ret)
-                                        goto out;
-
-                                ret = xmlTextWriterStartElement
-                                        (local->writer, (xmlChar *)"brick");
-                                XML_RET_CHECK_AND_GOTO (ret, out);
-
-                                memset (key, 0, sizeof (key));
-                                snprintf (key, sizeof (key),
-                                          "volume%d.brick%d.uuid", i, j);
-                                ret = dict_get_str (dict, key, &uuid);
-                                if (ret)
-                                        goto out;
-                                ret = xmlTextWriterWriteFormatAttribute
-                                        (local->writer, (xmlChar *)"uuid", "%s",
-                                         uuid);
-                                XML_RET_CHECK_AND_GOTO (ret, out);
-
-                                ret = xmlTextWriterWriteFormatString
-                                        (local->writer, "%s", brick);
-                                XML_RET_CHECK_AND_GOTO (ret, out);
-
-                                ret = xmlTextWriterWriteFormatElement
-                                        (local->writer, (xmlChar *)"name", "%s",
-                                         brick);
-                                XML_RET_CHECK_AND_GOTO (ret, out);
-
-                                ret = xmlTextWriterWriteFormatElement
-                                        (local->writer, (xmlChar *)"hostUuid",
-                                         "%s", uuid);
-                                XML_RET_CHECK_AND_GOTO (ret, out);
-
-                                ret = xmlTextWriterEndElement (local->writer);
-                                XML_RET_CHECK_AND_GOTO (ret, out);
-
-                                index++;
-                        }
-                        ret = xmlTextWriterEndElement (local->writer);
-                        XML_RET_CHECK_AND_GOTO (ret, out);
-
-                } else {
-                        while (j <= brick_count) {
-                                ret = xmlTextWriterStartElement
-                                        (local->writer, (xmlChar *)"brick");
-                                XML_RET_CHECK_AND_GOTO (ret, out);
-
-                                memset (key, 0, sizeof (key));
-                                snprintf (key, sizeof (key),
-                                          "volume%d.brick%d.uuid", i, j);
-                                ret = dict_get_str (dict, key, &uuid);
-                                if (ret)
-                                        goto out;
-                                ret = xmlTextWriterWriteFormatAttribute
-                                        (local->writer, (xmlChar *)"uuid", "%s",
-                                         uuid);
-                                XML_RET_CHECK_AND_GOTO (ret, out);
-
-                                memset (key, 0, sizeof (key));
-                                snprintf (key, sizeof (key),
-                                          "volume%d.brick%d", i, j);
-                                ret = dict_get_str (dict, key, &brick);
-                                if (ret)
-                                        goto out;
-                                ret = xmlTextWriterWriteFormatString
-                                        (local->writer, "%s", brick);
-                                XML_RET_CHECK_AND_GOTO (ret, out);
-
-                                ret = xmlTextWriterWriteFormatElement
-                                        (local->writer, (xmlChar *)"name", "%s",
-                                         brick);
-                                XML_RET_CHECK_AND_GOTO (ret, out);
-
-                                ret = xmlTextWriterWriteFormatElement
-                                        (local->writer, (xmlChar *)"hostUuid",
-                                         "%s", uuid);
-                                XML_RET_CHECK_AND_GOTO (ret, out);
-
-                                /* </brick> */
-                                ret = xmlTextWriterEndElement (local->writer);
-                                XML_RET_CHECK_AND_GOTO (ret, out);
-
-                                j++;
-                        }
-                }
-                /* </bricks> */
-                ret = xmlTextWriterEndElement (local->writer);
-                XML_RET_CHECK_AND_GOTO (ret, out);
-
-                memset (key, 0, sizeof (key));
-                snprintf (key, sizeof (key), "volume%d", i);
-                ret = cli_xml_output_vol_info_options (local->writer, dict,
-                                                       key);
-                if (ret)
-                        goto out;
+            ret = xmlTextWriterWriteFormatElement(
+                local->writer, (xmlChar *)"name", "%s", brick);
+            XML_RET_CHECK_AND_GOTO(ret, out);
 
-                /* </volume> */
-                ret = xmlTextWriterEndElement (local->writer);
-                XML_RET_CHECK_AND_GOTO (ret, out);
-        }
+            ret = xmlTextWriterWriteFormatElement(
+                local->writer, (xmlChar *)"hostUuid", "%s", uuid);
+            XML_RET_CHECK_AND_GOTO(ret, out);
+
+            snprintf(key, sizeof(key), "volume%d.brick%d.isArbiter", i, j);
+            if (dict_get(dict, key))
+                isArbiter = 1;
+            else
+                isArbiter = 0;
+            ret = xmlTextWriterWriteFormatElement(
+                local->writer, (xmlChar *)"isArbiter", "%d", isArbiter);
+            XML_RET_CHECK_AND_GOTO(ret, out);
+
+            /* </brick> */
+            ret = xmlTextWriterEndElement(local->writer);
+            XML_RET_CHECK_AND_GOTO(ret, out);
 
-        if (volname) {
-                GF_FREE (local->get_vol.volname);
-                local->get_vol.volname = gf_strdup (volname);
-                local->vol_count += count;
+            j++;
         }
+        /* </bricks> */
+        ret = xmlTextWriterEndElement(local->writer);
+        XML_RET_CHECK_AND_GOTO(ret, out);
+
+        snprintf(key, sizeof(key), "volume%d", i);
+        ret = cli_xml_output_vol_info_options(local->writer, dict, key);
+        if (ret)
+            goto out;
+
+        /* </volume> */
+        ret = xmlTextWriterEndElement(local->writer);
+        XML_RET_CHECK_AND_GOTO(ret, out);
+    }
+
+    if (volname) {
+        GF_FREE(local->get_vol.volname);
+        local->get_vol.volname = gf_strdup(volname);
+        local->vol_count += count;
+    }
 out:
-        gf_log ("cli", GF_LOG_DEBUG, "Returning %d", ret);
-        return ret;
+    gf_log("cli", GF_LOG_DEBUG, "Returning %d", ret);
+    return ret;
 #else
-        return 0;
+    return 0;
 #endif
 }
 
 int
-cli_xml_output_vol_info_begin (cli_local_t *local, int op_ret, int op_errno,
-                               char *op_errstr)
+cli_xml_output_vol_info_begin(cli_local_t *local, int op_ret, int op_errno,
+                              char *op_errstr)
 {
 #if (HAVE_LIB_XML)
-        int                     ret = -1;
+    int ret = -1;
 
-        GF_ASSERT (local);
+    GF_ASSERT(local);
 
-        ret = cli_begin_xml_output (&(local->writer), &(local->doc));
-        if (ret)
-                goto out;
+    ret = cli_begin_xml_output(&(local->writer), &(local->doc));
+    if (ret)
+        goto out;
 
-        ret = cli_xml_output_common (local->writer, op_ret, op_errno,
-                                     op_errstr);
-        if (ret)
-                goto out;
+    ret = cli_xml_output_common(local->writer, op_ret, op_errno, op_errstr);
+    if (ret)
+        goto out;
 
-        /* <volInfo> */
-        ret = xmlTextWriterStartElement (local->writer, (xmlChar *)"volInfo");
-        XML_RET_CHECK_AND_GOTO (ret, out);
+    /* <volInfo> */
+    ret = xmlTextWriterStartElement(local->writer, (xmlChar *)"volInfo");
+    XML_RET_CHECK_AND_GOTO(ret, out);
 
-        /* <volumes> */
-        ret = xmlTextWriterStartElement (local->writer, (xmlChar *)"volumes");
-        XML_RET_CHECK_AND_GOTO (ret, out);
+    /* <volumes> */
+    ret = xmlTextWriterStartElement(local->writer, (xmlChar *)"volumes");
+    XML_RET_CHECK_AND_GOTO(ret, out);
 
-        /* Init vol count */
-        local->vol_count = 0;
+    /* Init vol count */
+    local->vol_count = 0;
 
 out:
-        gf_log ("cli", GF_LOG_DEBUG, "Returning %d", ret);
-        return ret;
+    gf_log("cli", GF_LOG_DEBUG, "Returning %d", ret);
+    return ret;
 #else
-        return 0;
+    return 0;
 #endif
 }
 
 int
-cli_xml_output_vol_info_end (cli_local_t *local)
+cli_xml_output_vol_info_end(cli_local_t *local)
 {
 #if (HAVE_LIB_XML)
-        int             ret = -1;
+    int ret = -1;
 
-        GF_ASSERT (local);
+    GF_ASSERT(local);
 
-        ret = xmlTextWriterWriteFormatElement (local->writer,
-                                               (xmlChar *)"count",
-                                               "%d", local->vol_count);
+    ret = xmlTextWriterWriteFormatElement(local->writer, (xmlChar *)"count",
+                                          "%d", local->vol_count);
+    XML_RET_CHECK_AND_GOTO(ret, out);
+    /* </volumes> */
+    ret = xmlTextWriterEndElement(local->writer);
+    XML_RET_CHECK_AND_GOTO(ret, out);
 
-        /* </volumes> */
-        ret = xmlTextWriterEndElement (local->writer);
-        XML_RET_CHECK_AND_GOTO (ret, out);
+    /* </volInfo> */
+    ret = xmlTextWriterEndElement(local->writer);
+    XML_RET_CHECK_AND_GOTO(ret, out);
 
-        /* </volInfo> */
-        ret = xmlTextWriterEndElement (local->writer);
-        XML_RET_CHECK_AND_GOTO (ret, out);
-
-        ret = cli_end_xml_output (local->writer, local->doc);
+    ret = cli_end_xml_output(local->writer, local->doc);
 
 out:
-        gf_log ("cli", GF_LOG_DEBUG, "Returning %d", ret);
-        return ret;
+    gf_log("cli", GF_LOG_DEBUG, "Returning %d", ret);
+    return ret;
 #else
-        return 0;
+    return 0;
 #endif
 }
 
 int
-cli_xml_output_vol_quota_limit_list_end (cli_local_t *local)
+cli_xml_output_vol_quota_limit_list_end(cli_local_t *local)
 {
 #if (HAVE_LIB_XML)
-        int     ret = -1;
+    int ret = -1;
 
-        ret = xmlTextWriterEndElement (local->writer);
-        if (ret) {
-                goto out;
-        }
+    ret = xmlTextWriterEndElement(local->writer);
+    XML_RET_CHECK_AND_GOTO(ret, out);
 
-        ret = cli_end_xml_output (local->writer, local->doc);
+    ret = cli_end_xml_output(local->writer, local->doc);
 
 out:
-        return ret;
+    return ret;
 #else
-        return 0;
+    return 0;
 #endif
 }
 
 int
-cli_xml_output_vol_quota_limit_list_begin (cli_local_t *local, int op_ret,
-                                           int op_errno, char *op_errstr)
+cli_xml_output_vol_quota_limit_list_begin(cli_local_t *local, int op_ret,
+                                          int op_errno, char *op_errstr)
 {
 #if (HAVE_LIB_XML)
-        int                     ret = -1;
-
-        ret = cli_begin_xml_output (&(local->writer), &(local->doc));
-        if (ret)
-                goto out;
+    int ret = -1;
 
-        ret = cli_xml_output_common (local->writer, op_ret, op_errno,
-                                     op_errstr);
-        if (ret)
-                goto out;
+    ret = cli_begin_xml_output(&(local->writer), &(local->doc));
+    if (ret)
+        goto out;
 
-        /* <volQuota> */
-        ret = xmlTextWriterStartElement (local->writer, (xmlChar *)"volQuota");
-        XML_RET_CHECK_AND_GOTO (ret, out);
+    ret = cli_xml_output_common(local->writer, op_ret, op_errno, op_errstr);
+    if (ret)
+        goto out;
 
+    /* <volQuota> */
+    ret = xmlTextWriterStartElement(local->writer, (xmlChar *)"volQuota");
+    XML_RET_CHECK_AND_GOTO(ret, out);
 
 out:
-        gf_log ("cli", GF_LOG_DEBUG, "Returning %d", ret);
-        return ret;
+    gf_log("cli", GF_LOG_DEBUG, "Returning %d", ret);
+    return ret;
 #else
-        return 0;
+    return 0;
 #endif
 }
 
 #if (HAVE_LIB_XML)
 static int
-cli_xml_output_peer_hostnames (xmlTextWriterPtr writer, dict_t *dict,
-                               const char *prefix, int count)
+cli_xml_output_peer_hostnames(xmlTextWriterPtr writer, dict_t *dict,
+                              const char *prefix, int count)
 {
-        int   ret       = -1;
-        int   i         = 0;
-        char *hostname  = NULL;
-        char  key[1024] = {0,};
-
-        /* <hostnames> */
-        ret = xmlTextWriterStartElement (writer, (xmlChar *)"hostnames");
-        XML_RET_CHECK_AND_GOTO (ret, out);
-
-        for (i = 0; i < count; i++) {
-                memset (key, 0, sizeof (key));
-                snprintf (key, sizeof (key), "%s.hostname%d", prefix, i);
-                ret = dict_get_str (dict, key, &hostname);
-                if (ret)
-                        goto out;
-                ret = xmlTextWriterWriteFormatElement
-                        (writer, (xmlChar *)"hostname", "%s", hostname);
-                XML_RET_CHECK_AND_GOTO (ret, out);
-                hostname = NULL;
-        }
-
-        /* </hostnames> */
-        ret = xmlTextWriterEndElement (writer);
+    int ret = -1;
+    int i = 0;
+    char *hostname = NULL;
+    char key[1024] = {
+        0,
+    };
+
+    /* <hostnames> */
+    ret = xmlTextWriterStartElement(writer, (xmlChar *)"hostnames");
+    XML_RET_CHECK_AND_GOTO(ret, out);
+
+    for (i = 0; i < count; i++) {
+        ret = snprintf(key, sizeof(key), "%s.hostname%d", prefix, i);
+        if (ret < 0)
+            goto out;
+        ret = dict_get_str(dict, key, &hostname);
+        if (ret)
+            goto out;
+        ret = xmlTextWriterWriteFormatElement(writer, (xmlChar *)"hostname",
+                                              "%s", hostname);
+        XML_RET_CHECK_AND_GOTO(ret, out);
+        hostname = NULL;
+    }
 
+    /* </hostnames> */
+    ret = xmlTextWriterEndElement(writer);
+    XML_RET_CHECK_AND_GOTO(ret, out);
 out:
-        gf_log ("cli", GF_LOG_DEBUG, "Returning %d", ret);
-        return ret;
+    gf_log("cli", GF_LOG_DEBUG, "Returning %d", ret);
+    return ret;
 }
 #endif
 
 int
-cli_xml_output_peer_status (dict_t *dict, int op_ret, int op_errno,
-                            char *op_errstr)
+cli_xml_output_peer_status(dict_t *dict, int op_ret, int op_errno,
+                           char *op_errstr)
 {
 #if (HAVE_LIB_XML)
-        int                     ret = -1;
-        xmlTextWriterPtr        writer = NULL;
-        xmlDocPtr               doc = NULL;
-        int                     count = 0;
-        char                    *uuid = NULL;
-        char                    *hostname = NULL;
-        int                     connected = 0;
-        int                     state_id = 0;
-        char                    *state_str = NULL;
-        int                     hostname_count = 0;
-        int                     i = 1;
-        char                    key[1024] = {0,};
-
-        ret = cli_begin_xml_output (&writer, &doc);
-        if (ret)
-                goto out;
+    int ret = -1;
+    xmlTextWriterPtr writer = NULL;
+    xmlDocPtr doc = NULL;
+    int count = 0;
+    char *uuid = NULL;
+    char *hostname = NULL;
+    int connected = 0;
+    int state_id = 0;
+    char *state_str = NULL;
+    int hostname_count = 0;
+    int i = 1;
+    char key[1024] = {
+        0,
+    };
+
+    ret = cli_begin_xml_output(&writer, &doc);
+    if (ret)
+        goto out;
+
+    ret = cli_xml_output_common(writer, op_ret, op_errno, op_errstr);
+    if (ret)
+        goto out;
+
+    /* <peerStatus> */
+    ret = xmlTextWriterStartElement(writer, (xmlChar *)"peerStatus");
+    XML_RET_CHECK_AND_GOTO(ret, out);
+
+    if (!dict)
+        goto cont;
+
+    ret = dict_get_int32(dict, "count", &count);
+    if (ret)
+        goto out;
+
+    while (i <= count) {
+        /* <peer> */
+        ret = xmlTextWriterStartElement(writer, (xmlChar *)"peer");
+        XML_RET_CHECK_AND_GOTO(ret, out);
 
-        ret = cli_xml_output_common (writer, op_ret, op_errno, op_errstr);
+        snprintf(key, sizeof(key), "friend%d.uuid", i);
+        ret = dict_get_str(dict, key, &uuid);
         if (ret)
-                goto out;
+            goto out;
 
-        /* <peerStatus> */
-        ret = xmlTextWriterStartElement (writer, (xmlChar *)"peerStatus");
-        XML_RET_CHECK_AND_GOTO (ret, out);
-
-        if (!dict)
-                goto cont;
+        ret = xmlTextWriterWriteFormatElement(writer, (xmlChar *)"uuid", "%s",
+                                              uuid);
+        XML_RET_CHECK_AND_GOTO(ret, out);
 
-        ret = dict_get_int32 (dict, "count", &count);
+        snprintf(key, sizeof(key), "friend%d.hostname", i);
+        ret = dict_get_str(dict, key, &hostname);
         if (ret)
-                goto out;
-
-        while (i <= count) {
-                /* <peer> */
-                ret = xmlTextWriterStartElement (writer, (xmlChar *)"peer");
-                XML_RET_CHECK_AND_GOTO (ret, out);
-
-                memset (key, 0, sizeof (key));
-                snprintf (key, sizeof (key), "friend%d.uuid", i);
-                ret = dict_get_str (dict, key, &uuid);
-                if (ret)
-                        goto out;
+            goto out;
 
-                ret = xmlTextWriterWriteFormatElement (writer,
-                                                       (xmlChar *)"uuid",
-                                                       "%s", uuid);
-                XML_RET_CHECK_AND_GOTO (ret, out);
-
-                memset (key, 0, sizeof (key));
-                snprintf (key, sizeof (key), "friend%d.hostname", i);
-                ret = dict_get_str (dict, key, &hostname);
-                if (ret)
-                        goto out;
+        ret = xmlTextWriterWriteFormatElement(writer, (xmlChar *)"hostname",
+                                              "%s", hostname);
+        XML_RET_CHECK_AND_GOTO(ret, out);
 
-                ret = xmlTextWriterWriteFormatElement (writer,
-                                                       (xmlChar *)"hostname",
-                                                       "%s", hostname);
-                XML_RET_CHECK_AND_GOTO (ret, out);
-
-                memset (key, 0, sizeof (key));
-                snprintf (key, sizeof (key), "friend%d.hostname_count", i);
-                ret = dict_get_int32 (dict, key, &hostname_count);
-                if ((ret == 0) && (hostname_count > 0)) {
-                        memset (key, 0, sizeof (key));
-                        snprintf (key, sizeof (key), "friend%d", i);
-                        ret = cli_xml_output_peer_hostnames (writer, dict, key,
-                                                             hostname_count);
-                        XML_RET_CHECK_AND_GOTO (ret, out);
-                }
+        snprintf(key, sizeof(key), "friend%d.hostname_count", i);
+        ret = dict_get_int32(dict, key, &hostname_count);
+        if ((ret == 0) && (hostname_count > 0)) {
+            snprintf(key, sizeof(key), "friend%d", i);
+            ret = cli_xml_output_peer_hostnames(writer, dict, key,
+                                                hostname_count);
+            XML_RET_CHECK_AND_GOTO(ret, out);
+        }
 
-                memset (key, 0, sizeof (key));
-                snprintf (key, sizeof (key), "friend%d.connected", i);
-                ret = dict_get_int32 (dict, key, &connected);
-                if (ret)
-                        goto out;
+        snprintf(key, sizeof(key), "friend%d.connected", i);
+        ret = dict_get_int32(dict, key, &connected);
+        if (ret)
+            goto out;
 
-                ret = xmlTextWriterWriteFormatElement (writer,
-                                                       (xmlChar *)"connected",
-                                                       "%d", connected);
-                XML_RET_CHECK_AND_GOTO (ret, out);
+        ret = xmlTextWriterWriteFormatElement(writer, (xmlChar *)"connected",
+                                              "%d", connected);
+        XML_RET_CHECK_AND_GOTO(ret, out);
 
-                memset (key, 0, sizeof (key));
-                snprintf (key, sizeof (key), "friend%d.stateId", i);
-                ret = dict_get_int32 (dict, key, &state_id);
-                if (!ret) {
-                        /* ignore */
+        snprintf(key, sizeof(key), "friend%d.stateId", i);
+        ret = dict_get_int32(dict, key, &state_id);
+        if (!ret) {
+            /* ignore */
 
-                        ret = xmlTextWriterWriteFormatElement (writer,
-                                           (xmlChar *)"state", "%d", state_id);
-                        XML_RET_CHECK_AND_GOTO (ret, out);
-                }
+            ret = xmlTextWriterWriteFormatElement(writer, (xmlChar *)"state",
+                                                  "%d", state_id);
+            XML_RET_CHECK_AND_GOTO(ret, out);
+        }
 
-                memset (key, 0, sizeof (key));
-                snprintf (key, sizeof (key), "friend%d.state", i);
-                ret = dict_get_str (dict, key, &state_str);
-                if (!ret) {
-                        /* ignore */
+        snprintf(key, sizeof(key), "friend%d.state", i);
+        ret = dict_get_str(dict, key, &state_str);
+        if (!ret) {
+            /* ignore */
 
-                        ret = xmlTextWriterWriteFormatElement (writer,
-                                       (xmlChar *)"stateStr", "%s", state_str);
-                        XML_RET_CHECK_AND_GOTO (ret, out);
-                }
+            ret = xmlTextWriterWriteFormatElement(writer, (xmlChar *)"stateStr",
+                                                  "%s", state_str);
+            XML_RET_CHECK_AND_GOTO(ret, out);
+        }
 
-                /* </peer> */
-                ret = xmlTextWriterEndElement (writer);
-                XML_RET_CHECK_AND_GOTO (ret, out);
+        /* </peer> */
+        ret = xmlTextWriterEndElement(writer);
+        XML_RET_CHECK_AND_GOTO(ret, out);
 
-                i++;
-        }
+        i++;
+    }
 
 cont:
-        /* </peerStatus> */
-        ret = xmlTextWriterEndElement (writer);
-        XML_RET_CHECK_AND_GOTO (ret, out);
+    /* </peerStatus> */
+    ret = xmlTextWriterEndElement(writer);
+    XML_RET_CHECK_AND_GOTO(ret, out);
 
-        ret = cli_end_xml_output (writer, doc);
+    ret = cli_end_xml_output(writer, doc);
 
 out:
-        gf_log ("cli", GF_LOG_DEBUG, "Returning %d", ret);
-        return ret;
+    gf_log("cli", GF_LOG_DEBUG, "Returning %d", ret);
+    return ret;
 #else
-        return 0;
+    return 0;
 #endif
 }
 
 #if (HAVE_LIB_XML)
 /* Used for rebalance stop/status, remove-brick status */
 int
-cli_xml_output_vol_rebalance_status (xmlTextWriterPtr writer, dict_t *dict,
-                                     enum gf_task_types task_type)
+cli_xml_output_vol_rebalance_status(xmlTextWriterPtr writer, dict_t *dict,
+                                    enum gf_task_types task_type)
 {
-        int                     ret = -1;
-        int                     count = 0;
-        char                    *node_name = NULL;
-        char                    *node_uuid = NULL;
-        uint64_t                files = 0;
-        uint64_t                size = 0;
-        uint64_t                lookups = 0;
-        int                     status_rcd = 0;
-        uint64_t                failures = 0;
-        uint64_t                skipped = 0;
-        uint64_t                total_files = 0;
-        uint64_t                total_size = 0;
-        uint64_t                total_lookups = 0;
-        uint64_t                total_failures = 0;
-        uint64_t                total_skipped = 0;
-        char                    key[1024] = {0,};
-        int                     i = 0;
-        int                     overall_status = -1;
-        double                  elapsed = 0;
-        double                  overall_elapsed = 0;
-
-        if (!dict) {
-                ret = 0;
-                goto out;
-        }
+    int ret = -1;
+    int count = 0;
+    char *node_name = NULL;
+    char *node_uuid = NULL;
+    uint64_t files = 0;
+    uint64_t size = 0;
+    uint64_t lookups = 0;
+    int status_rcd = 0;
+    uint64_t failures = 0;
+    uint64_t skipped = 0;
+    uint64_t total_files = 0;
+    uint64_t total_size = 0;
+    uint64_t total_lookups = 0;
+    uint64_t total_failures = 0;
+    uint64_t total_skipped = 0;
+    char key[1024] = {
+        0,
+    };
+    int i = 0;
+    int overall_status = -1;
+    double elapsed = 0;
+    double overall_elapsed = 0;
+
+    if (!dict) {
+        ret = 0;
+        goto out;
+    }
+
+    ret = dict_get_int32(dict, "count", &count);
+    if (ret)
+        goto out;
+    ret = xmlTextWriterWriteFormatElement(writer, (xmlChar *)"nodeCount", "%d",
+                                          count);
+    XML_RET_CHECK_AND_GOTO(ret, out);
+
+    while (i < count) {
+        i++;
+        /* Getting status early, to skip nodes that don't have the
+         * rebalance process started
+         */
+        snprintf(key, sizeof(key), "status-%d", i);
+        ret = dict_get_int32(dict, key, &status_rcd);
 
-        ret = dict_get_int32 (dict, "count", &count);
+        /* If glusterd is down it fails to get the status, try
+         getting status from other nodes */
         if (ret)
-                goto out;
-        ret = xmlTextWriterWriteFormatElement (writer, (xmlChar *)"nodeCount",
-                                               "%d", count);
-        XML_RET_CHECK_AND_GOTO (ret, out);
-
-        while (i < count) {
-                i++;
-                /* Getting status early, to skip nodes that don't have the
-                 * rebalance process started
-                 */
-                memset (key, 0, sizeof (key));
-                snprintf (key, sizeof (key), "status-%d", i);
-                ret = dict_get_int32 (dict, key, &status_rcd);
-
-                /* If glusterd is down it fails to get the status, try
-                 getting status from other nodes */
-                if (ret)
-                    continue;
-                if (GF_DEFRAG_STATUS_NOT_STARTED == status_rcd)
-                        continue;
+            continue;
+        if (GF_DEFRAG_STATUS_NOT_STARTED == status_rcd)
+            continue;
 
-                /* <node> */
-                ret = xmlTextWriterStartElement (writer, (xmlChar *)"node");
-                XML_RET_CHECK_AND_GOTO (ret, out);
-
-                memset (key, 0, sizeof (key));
-                snprintf (key, sizeof (key), "node-name-%d", i);
-                ret = dict_get_str (dict, key, &node_name);
-                if (ret)
-                        goto out;
-                ret = xmlTextWriterWriteFormatElement (writer,
-                                                       (xmlChar *)"nodeName",
-                                                       "%s", node_name);
-
-                memset (key, 0, sizeof (key));
-                snprintf (key, sizeof (key), "node-uuid-%d", i);
-                ret = dict_get_str (dict, key, &node_uuid);
-                if (ret)
-                        goto out;
-                ret = xmlTextWriterWriteFormatElement (writer,
-                                                       (xmlChar *)"id",
-                                                       "%s", node_uuid);
-                XML_RET_CHECK_AND_GOTO (ret, out);
-
-                memset (key, 0, sizeof (key));
-                snprintf (key, sizeof (key), "files-%d", i);
-                ret = dict_get_uint64 (dict, key, &files);
-                if (ret)
-                        goto out;
-                total_files += files;
-                ret = xmlTextWriterWriteFormatElement (writer,
-                                                       (xmlChar *)"files",
-                                                       "%"PRIu64, files);
-                XML_RET_CHECK_AND_GOTO (ret, out);
-
-                memset (key, 0, sizeof (key));
-                snprintf (key, sizeof (key), "size-%d", i);
-                ret = dict_get_uint64 (dict, key, &size);
-                if (ret)
-                        goto out;
-                total_size += size;
-                ret = xmlTextWriterWriteFormatElement (writer,
-                                                       (xmlChar *)"size",
-                                                       "%"PRIu64,size);
-                XML_RET_CHECK_AND_GOTO (ret, out);
-
-                memset (key, 0, sizeof (key));
-                snprintf (key, sizeof (key), "lookups-%d", i);
-                ret = dict_get_uint64 (dict, key, &lookups);
-                if (ret)
-                        goto out;
-                total_lookups += lookups;
-                ret = xmlTextWriterWriteFormatElement (writer,
-                                                       (xmlChar *)"lookups",
-                                                       "%"PRIu64, lookups);
-                XML_RET_CHECK_AND_GOTO (ret, out);
-
-                memset (key, 0, sizeof (key));
-                snprintf (key, sizeof (key), "failures-%d", i);
-                ret = dict_get_uint64 (dict, key, &failures);
-                if (ret)
-                        goto out;
-
-                memset (key, 0, sizeof (key));
-                snprintf (key, sizeof (key), "skipped-%d", i);
-
-                ret = dict_get_uint64 (dict, key, &skipped);
-                if (ret)
-                        goto out;
+        /* <node> */
+        ret = xmlTextWriterStartElement(writer, (xmlChar *)"node");
+        XML_RET_CHECK_AND_GOTO(ret, out);
 
-                if (task_type == GF_TASK_TYPE_REMOVE_BRICK) {
-                        failures += skipped;
-                        skipped = 0;
-                }
+        snprintf(key, sizeof(key), "node-name-%d", i);
+        ret = dict_get_str(dict, key, &node_name);
+        if (ret)
+            goto out;
+        ret = xmlTextWriterWriteFormatElement(writer, (xmlChar *)"nodeName",
+                                              "%s", node_name);
+        XML_RET_CHECK_AND_GOTO(ret, out);
 
-                total_failures += failures;
-                ret = xmlTextWriterWriteFormatElement (writer,
-                                               (xmlChar *)"failures",
-                                               "%"PRIu64, failures);
-                XML_RET_CHECK_AND_GOTO (ret, out);
+        snprintf(key, sizeof(key), "node-uuid-%d", i);
+        ret = dict_get_str(dict, key, &node_uuid);
+        if (ret)
+            goto out;
+        ret = xmlTextWriterWriteFormatElement(writer, (xmlChar *)"id", "%s",
+                                              node_uuid);
+        XML_RET_CHECK_AND_GOTO(ret, out);
 
-                total_skipped += skipped;
+        snprintf(key, sizeof(key), "files-%d", i);
+        ret = dict_get_uint64(dict, key, &files);
+        if (ret)
+            goto out;
+        total_files += files;
+        ret = xmlTextWriterWriteFormatElement(writer, (xmlChar *)"files",
+                                              "%" PRIu64, files);
+        XML_RET_CHECK_AND_GOTO(ret, out);
 
-                ret = xmlTextWriterWriteFormatElement (writer,
-                                               (xmlChar *)"skipped",
-                                               "%"PRIu64, skipped);
-                XML_RET_CHECK_AND_GOTO (ret, out);
+        snprintf(key, sizeof(key), "size-%d", i);
+        ret = dict_get_uint64(dict, key, &size);
+        if (ret)
+            goto out;
+        total_size += size;
+        ret = xmlTextWriterWriteFormatElement(writer, (xmlChar *)"size",
+                                              "%" PRIu64, size);
+        XML_RET_CHECK_AND_GOTO(ret, out);
 
-                ret = xmlTextWriterWriteFormatElement (writer,
-                                                       (xmlChar *)"status",
-                                                       "%d", status_rcd);
-                XML_RET_CHECK_AND_GOTO (ret, out);
+        snprintf(key, sizeof(key), "lookups-%d", i);
+        ret = dict_get_uint64(dict, key, &lookups);
+        if (ret)
+            goto out;
+        total_lookups += lookups;
+        ret = xmlTextWriterWriteFormatElement(writer, (xmlChar *)"lookups",
+                                              "%" PRIu64, lookups);
+        XML_RET_CHECK_AND_GOTO(ret, out);
 
-                ret = xmlTextWriterWriteFormatElement (writer,
-                                                       (xmlChar *)"statusStr",
-                                                       "%s",
-                                         cli_vol_task_status_str[status_rcd]);
+        snprintf(key, sizeof(key), "failures-%d", i);
+        ret = dict_get_uint64(dict, key, &failures);
+        if (ret)
+            goto out;
 
-                memset (key, 0, 256);
-                snprintf (key, 256, "run-time-%d", i);
-                ret = dict_get_double (dict, key, &elapsed);
-                if (ret)
-                        goto out;
-                ret = xmlTextWriterWriteFormatElement (writer,
-                                                       (xmlChar *)"runtime",
-                                                       "%.2f", elapsed);
-                XML_RET_CHECK_AND_GOTO (ret, out);
+        snprintf(key, sizeof(key), "skipped-%d", i);
 
-                if (elapsed > overall_elapsed) {
-                    overall_elapsed = elapsed;
-                }
+        ret = dict_get_uint64(dict, key, &skipped);
+        if (ret)
+            goto out;
 
-                /* Rebalance has 5 states,
-                 * NOT_STARTED, STARTED, STOPPED, COMPLETE, FAILED
-                 * The precedence used to determine the aggregate status is as
-                 * below,
-                 * STARTED > FAILED > STOPPED > COMPLETE > NOT_STARTED
-                 */
-                /* TODO: Move this to a common place utilities that both CLI and
-                 * glusterd need.
-                 * Till then if the below algorithm is changed, change it in
-                 * glusterd_volume_status_aggregate_tasks_status in
-                 * glusterd-utils.c
-                 */
-
-                if (-1 == overall_status)
-                        overall_status = status_rcd;
-                int rank[] = {
-                        [GF_DEFRAG_STATUS_STARTED] = 1,
-                        [GF_DEFRAG_STATUS_FAILED] = 2,
-                        [GF_DEFRAG_STATUS_STOPPED] = 3,
-                        [GF_DEFRAG_STATUS_COMPLETE] = 4,
-                        [GF_DEFRAG_STATUS_NOT_STARTED] = 5
-                };
-                if (rank[status_rcd] <= rank[overall_status])
-                        overall_status = status_rcd;
-
-                /* </node> */
-                ret = xmlTextWriterEndElement (writer);
-                XML_RET_CHECK_AND_GOTO (ret, out);
+        if (task_type == GF_TASK_TYPE_REMOVE_BRICK) {
+            failures += skipped;
+            skipped = 0;
         }
 
-        /* Aggregate status */
-        /* <aggregate> */
-        ret = xmlTextWriterStartElement (writer, (xmlChar *)"aggregate");
-        XML_RET_CHECK_AND_GOTO (ret, out);
-
-        ret = xmlTextWriterWriteFormatElement (writer,(xmlChar *)"files",
-                                               "%"PRIu64, total_files);
-        XML_RET_CHECK_AND_GOTO (ret, out);
-
-        ret = xmlTextWriterWriteFormatElement (writer,(xmlChar *)"size",
-                                               "%"PRIu64, total_size);
-        XML_RET_CHECK_AND_GOTO (ret, out);
+        total_failures += failures;
+        ret = xmlTextWriterWriteFormatElement(writer, (xmlChar *)"failures",
+                                              "%" PRIu64, failures);
+        XML_RET_CHECK_AND_GOTO(ret, out);
 
-        ret = xmlTextWriterWriteFormatElement (writer,(xmlChar *)"lookups",
-                                               "%"PRIu64, total_lookups);
-        XML_RET_CHECK_AND_GOTO (ret, out);
+        total_skipped += skipped;
 
-        ret = xmlTextWriterWriteFormatElement (writer,(xmlChar *)"failures",
-                                               "%"PRIu64, total_failures);
-        XML_RET_CHECK_AND_GOTO (ret, out);
+        ret = xmlTextWriterWriteFormatElement(writer, (xmlChar *)"skipped",
+                                              "%" PRIu64, skipped);
+        XML_RET_CHECK_AND_GOTO(ret, out);
 
-        ret = xmlTextWriterWriteFormatElement (writer,(xmlChar *)"skipped",
-                                               "%"PRIu64, total_skipped);
-        XML_RET_CHECK_AND_GOTO (ret, out);
+        ret = xmlTextWriterWriteFormatElement(writer, (xmlChar *)"status", "%d",
+                                              status_rcd);
+        XML_RET_CHECK_AND_GOTO(ret, out);
 
-        ret = xmlTextWriterWriteFormatElement (writer,(xmlChar *)"status",
-                                               "%d", overall_status);
-        XML_RET_CHECK_AND_GOTO (ret, out);
+        ret = xmlTextWriterWriteFormatElement(
+            writer, (xmlChar *)"statusStr", "%s",
+            cli_vol_task_status_str[status_rcd]);
+        XML_RET_CHECK_AND_GOTO(ret, out);
 
-        ret = xmlTextWriterWriteFormatElement (writer,(xmlChar *)"statusStr",
-                                               "%s",
-                                      cli_vol_task_status_str[overall_status]);
-        XML_RET_CHECK_AND_GOTO (ret, out);
+        snprintf(key, sizeof(key), "run-time-%d", i);
+        ret = dict_get_double(dict, key, &elapsed);
+        if (ret)
+            goto out;
+        ret = xmlTextWriterWriteFormatElement(writer, (xmlChar *)"runtime",
+                                              "%.2f", elapsed);
+        XML_RET_CHECK_AND_GOTO(ret, out);
 
-        ret = xmlTextWriterWriteFormatElement (writer,(xmlChar *)"runtime",
-                                               "%.2f", overall_elapsed);
-        XML_RET_CHECK_AND_GOTO (ret, out);
+        if (elapsed > overall_elapsed) {
+            overall_elapsed = elapsed;
+        }
 
-        /* </aggregate> */
-        ret = xmlTextWriterEndElement (writer);
-        XML_RET_CHECK_AND_GOTO (ret, out);
+        /* Rebalance has 5 states,
+         * NOT_STARTED, STARTED, STOPPED, COMPLETE, FAILED
+         * The precedence used to determine the aggregate status is as
+         * below,
+         * STARTED > FAILED > STOPPED > COMPLETE > NOT_STARTED
+         */
+        /* TODO: Move this to a common place utilities that both CLI and
+         * glusterd need.
+         * Till then if the below algorithm is changed, change it in
+         * glusterd_volume_status_aggregate_tasks_status in
+         * glusterd-utils.c
+         */
 
-out:
-        gf_log ("cli", GF_LOG_DEBUG, "Returning %d", ret);
-        return ret;
-}
-#endif
+        if (-1 == overall_status)
+            overall_status = status_rcd;
+        int rank[] = {[GF_DEFRAG_STATUS_STARTED] = 1,
+                      [GF_DEFRAG_STATUS_FAILED] = 2,
+                      [GF_DEFRAG_STATUS_STOPPED] = 3,
+                      [GF_DEFRAG_STATUS_COMPLETE] = 4,
+                      [GF_DEFRAG_STATUS_NOT_STARTED] = 5};
+        if (rank[status_rcd] <= rank[overall_status])
+            overall_status = status_rcd;
+
+        /* </node> */
+        ret = xmlTextWriterEndElement(writer);
+        XML_RET_CHECK_AND_GOTO(ret, out);
+    }
 
-int
-cli_xml_output_vol_tier_status (xmlTextWriterPtr writer, dict_t *dict,
-                enum gf_task_types task_type)
-{
-#if (HAVE_LIB_XML)
+    /* Aggregate status */
+    /* <aggregate> */
+    ret = xmlTextWriterStartElement(writer, (xmlChar *)"aggregate");
+    XML_RET_CHECK_AND_GOTO(ret, out);
 
-        int                     ret = -1;
-        int                     count = 0;
-        char                    *node_name = NULL;
-        char                    *status_str  = NULL;
-        uint64_t                promoted     = 0;
-        uint64_t                demoted      = 0;
-        int                     i = 1;
-        char                    key[1024] = {0,};
-        gf_defrag_status_t      status_rcd   = GF_DEFRAG_STATUS_NOT_STARTED;
+    ret = xmlTextWriterWriteFormatElement(writer, (xmlChar *)"files",
+                                          "%" PRIu64, total_files);
+    XML_RET_CHECK_AND_GOTO(ret, out);
 
+    ret = xmlTextWriterWriteFormatElement(writer, (xmlChar *)"size", "%" PRIu64,
+                                          total_size);
+    XML_RET_CHECK_AND_GOTO(ret, out);
 
-        GF_VALIDATE_OR_GOTO ("cli", dict, out);
+    ret = xmlTextWriterWriteFormatElement(writer, (xmlChar *)"lookups",
+                                          "%" PRIu64, total_lookups);
+    XML_RET_CHECK_AND_GOTO(ret, out);
 
-        ret = dict_get_int32 (dict, "count", &count);
-        if (ret) {
-                gf_log ("cli", GF_LOG_ERROR, "count not set");
-                goto out;
-        }
+    ret = xmlTextWriterWriteFormatElement(writer, (xmlChar *)"failures",
+                                          "%" PRIu64, total_failures);
+    XML_RET_CHECK_AND_GOTO(ret, out);
 
-        ret = xmlTextWriterWriteFormatElement (writer, (xmlChar *)"nodeCount",
-                                               "%d", count);
-        XML_RET_CHECK_AND_GOTO (ret, out);
+    ret = xmlTextWriterWriteFormatElement(writer, (xmlChar *)"skipped",
+                                          "%" PRIu64, total_skipped);
+    XML_RET_CHECK_AND_GOTO(ret, out);
 
-        while (i <= count) {
-                promoted = 0;
-                node_name = NULL;
-                demoted = 0;
+    if (overall_status == -1) {
+        overall_status = status_rcd;
+    }
 
-                ret = xmlTextWriterStartElement (writer, (xmlChar *)"node");
-                XML_RET_CHECK_AND_GOTO (ret, out);
+    ret = xmlTextWriterWriteFormatElement(writer, (xmlChar *)"status", "%d",
+                                          overall_status);
+    XML_RET_CHECK_AND_GOTO(ret, out);
 
-                memset (key, 0, sizeof (key));
-                snprintf (key, sizeof (key), "node-name-%d", i);
-                ret = dict_get_str (dict, key, &node_name);
-                if (ret)
-                        goto out;
-                ret = xmlTextWriterWriteFormatElement (writer,
-                                                       (xmlChar *)"nodeName",
-                                                       "%s", node_name);
-                XML_RET_CHECK_AND_GOTO (ret, out);
-
-                memset (key, 0, sizeof (key));
-                snprintf (key, sizeof (key), "promoted-%d", i);
-                ret = dict_get_uint64 (dict, key, &promoted);
-                if (ret)
-                        goto out;
-                ret = xmlTextWriterWriteFormatElement (writer,
-                                                       (xmlChar *)"promoted"
-                                                       "Files", "%"PRIu64,
-                                                       promoted);
-                XML_RET_CHECK_AND_GOTO (ret, out);
-
-                memset (key, 0, sizeof (key));
-                snprintf (key, sizeof (key), "demoted-%d", i);
-                ret = dict_get_uint64 (dict, key, &demoted);
-                if (ret)
-                        goto out;
-                ret = xmlTextWriterWriteFormatElement (writer,
-                                                       (xmlChar *)"demoted"
-                                                       "Files", "%"PRIu64,
-                                                       demoted);
-                XML_RET_CHECK_AND_GOTO (ret, out);
+    ret = xmlTextWriterWriteFormatElement(
+        writer, (xmlChar *)"statusStr", "%s",
+        cli_vol_task_status_str[overall_status]);
+    XML_RET_CHECK_AND_GOTO(ret, out);
 
-                memset (key, 0, 256);
-                snprintf (key, 256, "status-%d", i);
-                ret = dict_get_int32 (dict, key, (int32_t *)&status_rcd);
+    ret = xmlTextWriterWriteFormatElement(writer, (xmlChar *)"runtime", "%.2f",
+                                          overall_elapsed);
+    XML_RET_CHECK_AND_GOTO(ret, out);
 
-                status_str = cli_vol_task_status_str[status_rcd];
-
-                ret = xmlTextWriterWriteFormatElement (writer,
-                                                       (xmlChar *)"statusStr",
-                                                       "%s", status_str);
-                XML_RET_CHECK_AND_GOTO (ret, out);
-
-
-                ret = xmlTextWriterEndElement (writer);
-                XML_RET_CHECK_AND_GOTO (ret, out);
-
-                i++;
-        }
+    /* </aggregate> */
+    ret = xmlTextWriterEndElement(writer);
+    XML_RET_CHECK_AND_GOTO(ret, out);
 
 out:
-        gf_log ("cli", GF_LOG_DEBUG, "Returning %d", ret);
-        return ret;
-
-#else
-        return 0;
-
-#endif
+    gf_log("cli", GF_LOG_DEBUG, "Returning %d", ret);
+    return ret;
 }
-
-
-
+#endif
 
 int
-cli_xml_output_vol_rebalance (gf_cli_defrag_type op, dict_t *dict, int op_ret,
-                              int op_errno, char *op_errstr)
+cli_xml_output_vol_rebalance(gf_cli_defrag_type op, dict_t *dict, int op_ret,
+                             int op_errno, char *op_errstr)
 {
 #if (HAVE_LIB_XML)
-        int                     ret = -1;
-        xmlTextWriterPtr        writer = NULL;
-        xmlDocPtr               doc = NULL;
-        char                    *task_id_str = NULL;
+    int ret = -1;
+    xmlTextWriterPtr writer = NULL;
+    xmlDocPtr doc = NULL;
+    char *task_id_str = NULL;
+
+    ret = cli_begin_xml_output(&writer, &doc);
+    if (ret)
+        goto out;
+
+    ret = cli_xml_output_common(writer, op_ret, op_errno, op_errstr);
+    if (ret)
+        goto out;
+
+    /* <volRebalance> */
+    ret = xmlTextWriterStartElement(writer, (xmlChar *)"volRebalance");
+    XML_RET_CHECK_AND_GOTO(ret, out);
+
+    ret = dict_get_str(dict, GF_REBALANCE_TID_KEY, &task_id_str);
+    if (ret == 0) {
+        ret = xmlTextWriterWriteFormatElement(writer, (xmlChar *)"task-id",
+                                              "%s", task_id_str);
+        XML_RET_CHECK_AND_GOTO(ret, out);
+    }
 
-        ret = cli_begin_xml_output (&writer, &doc);
-        if (ret)
-                goto out;
+    ret = xmlTextWriterWriteFormatElement(writer, (xmlChar *)"op", "%d", op);
+    XML_RET_CHECK_AND_GOTO(ret, out);
 
-        ret = cli_xml_output_common (writer, op_ret, op_errno, op_errstr);
+    if ((GF_DEFRAG_CMD_STOP == op) || (GF_DEFRAG_CMD_STATUS == op)) {
+        ret = cli_xml_output_vol_rebalance_status(writer, dict,
+                                                  GF_TASK_TYPE_REBALANCE);
         if (ret)
-                goto out;
-
-        /* <volRebalance> */
-        ret = xmlTextWriterStartElement (writer, (xmlChar *)"volRebalance");
-        XML_RET_CHECK_AND_GOTO (ret, out);
+            goto out;
+    }
 
-        ret = dict_get_str (dict, GF_REBALANCE_TID_KEY, &task_id_str);
-        if (ret == 0) {
-                ret = xmlTextWriterWriteFormatElement (writer,
-                                                       (xmlChar *)"task-id",
-                                                       "%s", task_id_str);
-                XML_RET_CHECK_AND_GOTO (ret, out);
-        }
+    /* </volRebalance> */
+    ret = xmlTextWriterEndElement(writer);
+    XML_RET_CHECK_AND_GOTO(ret, out);
 
-        ret = xmlTextWriterWriteFormatElement (writer, (xmlChar *)"op",
-                                               "%d", op);
-        XML_RET_CHECK_AND_GOTO (ret, out);
-
-        if (GF_DEFRAG_CMD_STATUS_TIER == op) {
-                ret = cli_xml_output_vol_tier_status (writer,
-                                dict, GF_TASK_TYPE_REBALANCE);
-                if (ret)
-                        goto out;
-                }
-        if ((GF_DEFRAG_CMD_STOP == op) || (GF_DEFRAG_CMD_STATUS == op)) {
-
-                ret = cli_xml_output_vol_rebalance_status (writer, dict,
-                                                      GF_TASK_TYPE_REBALANCE);
-                if (ret)
-                        goto out;
-        }
-
-        /* </volRebalance> */
-        ret = xmlTextWriterEndElement (writer);
-        XML_RET_CHECK_AND_GOTO (ret, out);
-
-
-        ret = cli_end_xml_output (writer, doc);
+    ret = cli_end_xml_output(writer, doc);
 
 out:
-        gf_log ("cli", GF_LOG_DEBUG, "Returning %d", ret);
-        return ret;
+    gf_log("cli", GF_LOG_DEBUG, "Returning %d", ret);
+    return ret;
 #else
-        return 0;
+    return 0;
 #endif
 }
 
 int
-cli_xml_output_vol_remove_brick_detach_tier (gf_boolean_t status_op,
-                                             dict_t *dict, int op_ret,
-                                             int op_errno, char *op_errstr,
-                                             const char *op)
+cli_xml_output_vol_remove_brick(gf_boolean_t status_op, dict_t *dict,
+                                int op_ret, int op_errno, char *op_errstr,
+                                const char *op)
 {
 #if (HAVE_LIB_XML)
-        int                     ret = -1;
-        xmlTextWriterPtr        writer = NULL;
-        xmlDocPtr               doc = NULL;
-        char                    *task_id_str = NULL;
-
-        ret = cli_begin_xml_output (&writer, &doc);
-        if (ret)
-                goto out;
+    int ret = -1;
+    xmlTextWriterPtr writer = NULL;
+    xmlDocPtr doc = NULL;
+    char *task_id_str = NULL;
+
+    ret = cli_begin_xml_output(&writer, &doc);
+    if (ret)
+        goto out;
+
+    ret = cli_xml_output_common(writer, op_ret, op_errno, op_errstr);
+    if (ret)
+        goto out;
+
+    ret = xmlTextWriterStartElement(writer, (xmlChar *)op);
+    XML_RET_CHECK_AND_GOTO(ret, out);
+
+    ret = dict_get_str(dict, GF_REMOVE_BRICK_TID_KEY, &task_id_str);
+    if (ret == 0) {
+        ret = xmlTextWriterWriteFormatElement(writer, (xmlChar *)"task-id",
+                                              "%s", task_id_str);
+        XML_RET_CHECK_AND_GOTO(ret, out);
+    }
 
-        ret = cli_xml_output_common (writer, op_ret, op_errno, op_errstr);
+    if (status_op) {
+        ret = cli_xml_output_vol_rebalance_status(writer, dict,
+                                                  GF_TASK_TYPE_REMOVE_BRICK);
         if (ret)
-                goto out;
-
-        ret = xmlTextWriterStartElement (writer, (xmlChar *) op);
-        XML_RET_CHECK_AND_GOTO (ret, out);
-
-        ret = dict_get_str (dict, GF_REMOVE_BRICK_TID_KEY, &task_id_str);
-        if (ret == 0) {
-                ret = xmlTextWriterWriteFormatElement (writer,
-                                                       (xmlChar *)"task-id",
-                                                       "%s", task_id_str);
-                XML_RET_CHECK_AND_GOTO (ret, out);
-        }
-
-        if (status_op) {
-                ret = cli_xml_output_vol_rebalance_status (writer, dict,
-                                                   GF_TASK_TYPE_REMOVE_BRICK);
-                if (ret)
-                        goto out;
-        }
-
-        ret = xmlTextWriterEndElement (writer);
-        XML_RET_CHECK_AND_GOTO (ret, out);
+            goto out;
+    }
 
+    ret = xmlTextWriterEndElement(writer);
+    XML_RET_CHECK_AND_GOTO(ret, out);
 
-        ret = cli_end_xml_output (writer, doc);
+    ret = cli_end_xml_output(writer, doc);
 
 out:
-        gf_log ("cli", GF_LOG_DEBUG, "Returning %d", ret);
-        return ret;
+    gf_log("cli", GF_LOG_DEBUG, "Returning %d", ret);
+    return ret;
 #else
-        return 0;
+    return 0;
 #endif
 }
 
 int
-cli_xml_output_vol_replace_brick (char *op, dict_t *dict,
-                                  int op_ret, int op_errno, char *op_errstr)
+cli_xml_output_vol_replace_brick(dict_t *dict, int op_ret, int op_errno,
+                                 char *op_errstr)
 {
 #if (HAVE_LIB_XML)
-        int                     ret = -1;
-        int                     status = 0;
-        uint64_t                files = 0;
-        char                    *current_file = 0;
-        char                    *task_id_str = NULL;
-        xmlTextWriterPtr        writer = NULL;
-        xmlDocPtr               doc = NULL;
-
-        ret = cli_begin_xml_output (&writer, &doc);
-        if (ret)
-                goto out;
+    int ret = -1;
+    xmlTextWriterPtr writer = NULL;
+    xmlDocPtr doc = NULL;
 
-        ret = cli_xml_output_common (writer, op_ret, op_errno, op_errstr);
-        if (ret)
-                goto out;
+    ret = cli_begin_xml_output(&writer, &doc);
+    if (ret)
+        goto out;
+
+    ret = cli_xml_output_common(writer, op_ret, op_errno, op_errstr);
+    if (ret)
+        goto out;
 
-        ret = cli_end_xml_output (writer, doc);
+    ret = cli_end_xml_output(writer, doc);
 
 out:
-        gf_log ("cli", GF_LOG_DEBUG, "Returning %d", ret);
-        return ret;
+    gf_log("cli", GF_LOG_DEBUG, "Returning %d", ret);
+    return ret;
 #else
-        return 0;
+    return 0;
 #endif
 }
 
 int
-cli_xml_output_vol_create (dict_t *dict, int op_ret, int op_errno,
-                           char *op_errstr)
+cli_xml_output_vol_create(dict_t *dict, int op_ret, int op_errno,
+                          char *op_errstr)
 {
 #if (HAVE_LIB_XML)
-        int                     ret = -1;
-        xmlTextWriterPtr        writer = NULL;
-        xmlDocPtr               doc = NULL;
-        char                    *volname = NULL;
-        char                    *volid = NULL;
+    int ret = -1;
+    xmlTextWriterPtr writer = NULL;
+    xmlDocPtr doc = NULL;
+    char *volname = NULL;
+    char *volid = NULL;
+
+    ret = cli_begin_xml_output(&writer, &doc);
+    if (ret)
+        goto out;
+
+    ret = cli_xml_output_common(writer, op_ret, op_errno, op_errstr);
+    if (ret)
+        goto out;
+
+    if (dict) {
+        /* <volCreate> */
+        ret = xmlTextWriterStartElement(writer, (xmlChar *)"volCreate");
+        XML_RET_CHECK_AND_GOTO(ret, out);
 
-        ret = cli_begin_xml_output (&writer, &doc);
-        if (ret)
-                goto out;
+        /* <volume> */
+        ret = xmlTextWriterStartElement(writer, (xmlChar *)"volume");
+        XML_RET_CHECK_AND_GOTO(ret, out);
 
-        ret = cli_xml_output_common (writer, op_ret, op_errno, op_errstr);
+        ret = dict_get_str(dict, "volname", &volname);
         if (ret)
-                goto out;
-
-        if (dict) {
-                /* <volCreate> */
-                ret = xmlTextWriterStartElement (writer,
-                                                 (xmlChar *)"volCreate");
-                XML_RET_CHECK_AND_GOTO (ret, out);
-
-                /* <volume> */
-                ret = xmlTextWriterStartElement (writer, (xmlChar *)"volume");
-                XML_RET_CHECK_AND_GOTO (ret, out);
-
-                ret = dict_get_str (dict, "volname", &volname);
-                if (ret)
-                        goto out;
-                ret = xmlTextWriterWriteFormatElement (writer,
-                                                       (xmlChar *) "name",
-                                                       "%s", volname);
-                XML_RET_CHECK_AND_GOTO (ret, out);
+            goto out;
+        ret = xmlTextWriterWriteFormatElement(writer, (xmlChar *)"name", "%s",
+                                              volname);
+        XML_RET_CHECK_AND_GOTO(ret, out);
 
-                ret = dict_get_str (dict, "volume-id", &volid);
-                if (ret)
-                        goto out;
-                ret = xmlTextWriterWriteFormatElement (writer, (xmlChar *)"id",
-                                                       "%s", volid);
-                XML_RET_CHECK_AND_GOTO (ret, out);
+        ret = dict_get_str(dict, "volume-id", &volid);
+        if (ret)
+            goto out;
+        ret = xmlTextWriterWriteFormatElement(writer, (xmlChar *)"id", "%s",
+                                              volid);
+        XML_RET_CHECK_AND_GOTO(ret, out);
 
-                /* </volume> */
-                ret = xmlTextWriterEndElement (writer);
-                XML_RET_CHECK_AND_GOTO (ret, out);
+        /* </volume> */
+        ret = xmlTextWriterEndElement(writer);
+        XML_RET_CHECK_AND_GOTO(ret, out);
 
-                /* </volCreate> */
-                ret = xmlTextWriterEndElement (writer);
-                XML_RET_CHECK_AND_GOTO (ret, out);
-        }
+        /* </volCreate> */
+        ret = xmlTextWriterEndElement(writer);
+        XML_RET_CHECK_AND_GOTO(ret, out);
+    }
 
-        ret = cli_end_xml_output (writer, doc);
+    ret = cli_end_xml_output(writer, doc);
 
 out:
-        gf_log ("cli", GF_LOG_DEBUG, "Returning %d", ret);
-        return ret;
+    gf_log("cli", GF_LOG_DEBUG, "Returning %d", ret);
+    return ret;
 #else
-        return 0;
+    return 0;
 #endif
 }
 
 int
-cli_xml_output_generic_volume (char *op, dict_t *dict, int op_ret, int op_errno,
-                               char *op_errstr)
+cli_xml_output_generic_volume(char *op, dict_t *dict, int op_ret, int op_errno,
+                              char *op_errstr)
 {
 #if (HAVE_LIB_XML)
-        int                     ret = -1;
-        xmlTextWriterPtr        writer = NULL;
-        xmlDocPtr               doc = NULL;
-        char                    *volname = NULL;
-        char                    *volid = NULL;
+    int ret = -1;
+    xmlTextWriterPtr writer = NULL;
+    xmlDocPtr doc = NULL;
+    char *volname = NULL;
+    char *volid = NULL;
 
-        GF_ASSERT (op);
+    GF_ASSERT(op);
 
-        ret = cli_begin_xml_output (&writer, &doc);
-        if (ret)
-                goto out;
+    ret = cli_begin_xml_output(&writer, &doc);
+    if (ret)
+        goto out;
 
-        ret = cli_xml_output_common (writer, op_ret, op_errno, op_errstr);
-        if (ret)
-                goto out;
+    ret = cli_xml_output_common(writer, op_ret, op_errno, op_errstr);
+    if (ret)
+        goto out;
 
-        if (dict) {
-                /* <"op"> */
-                ret = xmlTextWriterStartElement (writer, (xmlChar *)op);
-                XML_RET_CHECK_AND_GOTO (ret, out);
+    if (dict) {
+        /* <"op"> */
+        ret = xmlTextWriterStartElement(writer, (xmlChar *)op);
+        XML_RET_CHECK_AND_GOTO(ret, out);
 
-                /* <volume> */
-                ret = xmlTextWriterStartElement (writer, (xmlChar *)"volume");
-                XML_RET_CHECK_AND_GOTO (ret, out);
+        /* <volume> */
+        ret = xmlTextWriterStartElement(writer, (xmlChar *)"volume");
+        XML_RET_CHECK_AND_GOTO(ret, out);
 
-                ret = dict_get_str (dict, "volname", &volname);
-                if (ret)
-                        goto out;
-                ret = xmlTextWriterWriteFormatElement (writer,
-                                                       (xmlChar *) "name",
-                                                       "%s", volname);
-                XML_RET_CHECK_AND_GOTO (ret, out);
+        ret = dict_get_str(dict, "volname", &volname);
+        if (ret)
+            goto out;
+        ret = xmlTextWriterWriteFormatElement(writer, (xmlChar *)"name", "%s",
+                                              volname);
+        XML_RET_CHECK_AND_GOTO(ret, out);
 
-                ret = dict_get_str (dict, "vol-id", &volid);
-                if (ret)
-                        goto out;
-                ret = xmlTextWriterWriteFormatElement (writer, (xmlChar *)"id",
-                                                       "%s", volid);
-                XML_RET_CHECK_AND_GOTO (ret, out);
+        ret = dict_get_str(dict, "vol-id", &volid);
+        if (ret)
+            goto out;
+        ret = xmlTextWriterWriteFormatElement(writer, (xmlChar *)"id", "%s",
+                                              volid);
+        XML_RET_CHECK_AND_GOTO(ret, out);
 
-                /* </volume> */
-                ret = xmlTextWriterEndElement (writer);
-                XML_RET_CHECK_AND_GOTO (ret, out);
+        /* </volume> */
+        ret = xmlTextWriterEndElement(writer);
+        XML_RET_CHECK_AND_GOTO(ret, out);
 
-                /* </"op"> */
-                ret = xmlTextWriterEndElement (writer);
-                XML_RET_CHECK_AND_GOTO (ret, out);
-        }
+        /* </"op"> */
+        ret = xmlTextWriterEndElement(writer);
+        XML_RET_CHECK_AND_GOTO(ret, out);
+    }
 
-        ret = cli_end_xml_output (writer, doc);
+    ret = cli_end_xml_output(writer, doc);
 
 out:
-        gf_log ("cli", GF_LOG_DEBUG, "Returning %d", ret);
-        return ret;
+    gf_log("cli", GF_LOG_DEBUG, "Returning %d", ret);
+    return ret;
 #else
-        return 0;
+    return 0;
 #endif
 }
 
 #if (HAVE_LIB_XML)
 int
-_output_gsync_config (FILE *fp, xmlTextWriterPtr writer, char *op_name)
+_output_gsync_config(FILE *fp, xmlTextWriterPtr writer, char *op_name)
 {
-        char  resbuf[256 + PATH_MAX] = {0,};
-        char *ptr                    = NULL;
-        char *v                      = NULL;
-        int   blen                   = sizeof(resbuf);
-        int   ret                    = 0;
-
-        for (;;) {
-                ptr = fgets (resbuf, blen, fp);
-                if (!ptr)
-                        break;
-
-                v = resbuf + strlen (resbuf) - 1;
-                while (isspace (*v)) {
-                        /* strip trailing space */
-                        *v-- = '\0';
-                }
-                if (v == resbuf) {
-                        /* skip empty line */
-                        continue;
-                }
+    char resbuf[256 + PATH_MAX] = {
+        0,
+    };
+    char *ptr = NULL;
+    char *v = NULL;
+    int blen = sizeof(resbuf);
+    int ret = 0;
+
+    for (;;) {
+        ptr = fgets(resbuf, blen, fp);
+        if (!ptr)
+            break;
 
-                if (op_name!= NULL){
-                        ret = xmlTextWriterWriteFormatElement (writer,
-                                                        (xmlChar *)op_name,
-                                                        "%s", resbuf);
-                        XML_RET_CHECK_AND_GOTO (ret, out);
-                        goto out;
-                }
+        v = resbuf + strlen(resbuf) - 1;
+        while (isspace(*v)) {
+            /* strip trailing space */
+            *v-- = '\0';
+        }
+        if (v == resbuf) {
+            /* skip empty line */
+            continue;
+        }
 
-                v = strchr (resbuf, ':');
-                if (!v) {
-                        ret = -1;
-                        goto out;
-                }
-                *v++ = '\0';
-                while (isspace (*v))
-                        v++;
-                v = gf_strdup (v);
-                if (!v) {
-                        ret = -1;
-                        goto out;
-                }
+        if (op_name != NULL) {
+            ret = xmlTextWriterWriteFormatElement(writer, (xmlChar *)op_name,
+                                                  "%s", resbuf);
+            XML_RET_CHECK_AND_GOTO(ret, out);
+            goto out;
+        }
 
-                ret = xmlTextWriterWriteFormatElement (writer,
-                                                       (xmlChar *)resbuf,
-                                                       "%s", v);
-                XML_RET_CHECK_AND_GOTO (ret, out);
+        v = strchr(resbuf, ':');
+        if (!v) {
+            ret = -1;
+            goto out;
+        }
+        *v++ = '\0';
+        while (isspace(*v))
+            v++;
+        v = gf_strdup(v);
+        if (!v) {
+            ret = -1;
+            goto out;
         }
+
+        ret = xmlTextWriterWriteFormatElement(writer, (xmlChar *)resbuf, "%s",
+                                              v);
+        XML_RET_CHECK_AND_GOTO(ret, out);
+    }
 out:
-        gf_log ("cli", GF_LOG_DEBUG, "Returning %d", ret);
-        return ret;
+    gf_log("cli", GF_LOG_DEBUG, "Returning %d", ret);
+    return ret;
 }
 #endif
 
 #if (HAVE_LIB_XML)
 int
-get_gsync_config (runner_t *runner,
-                  int (*op_conf)(FILE *fp,
-                              xmlTextWriterPtr writer,
-                              char *op_name),
-                  xmlTextWriterPtr writer, char *op_name)
+get_gsync_config(runner_t *runner,
+                 int (*op_conf)(FILE *fp, xmlTextWriterPtr writer,
+                                char *op_name),
+                 xmlTextWriterPtr writer, char *op_name)
 {
-        int ret = 0;
+    int ret = 0;
 
-        runner_redir (runner, STDOUT_FILENO, RUN_PIPE);
-        if (runner_start (runner) != 0) {
-                gf_log ("cli", GF_LOG_ERROR, "spawning child failed");
-                return -1;
-        }
+    runner_redir(runner, STDOUT_FILENO, RUN_PIPE);
+    if (runner_start(runner) != 0) {
+        gf_log("cli", GF_LOG_ERROR, "spawning child failed");
+        return -1;
+    }
 
-        ret = op_conf (runner_chio (runner, STDOUT_FILENO), writer, op_name);
+    ret = op_conf(runner_chio(runner, STDOUT_FILENO), writer, op_name);
 
-        ret |= runner_end (runner);
-        if (ret)
-                gf_log ("cli", GF_LOG_ERROR, "reading data from child failed");
+    ret |= runner_end(runner);
+    if (ret)
+        gf_log("cli", GF_LOG_ERROR, "reading data from child failed");
 
-        return ret ? -1 : 0;
+    return ret ? -1 : 0;
 }
 #endif
 
 #if (HAVE_LIB_XML)
 int
-cli_xml_generate_gsync_config (dict_t *dict, xmlTextWriterPtr writer)
+cli_xml_generate_gsync_config(dict_t *dict, xmlTextWriterPtr writer)
 {
-        runner_t runner           = {0,};
-        char *subop               = NULL;
-        char *gwd                 = NULL;
-        char *slave               = NULL;
-        char *confpath            = NULL;
-        char *master              = NULL;
-        char *op_name             = NULL;
-        int   ret                 = -1;
-        char  conf_path[PATH_MAX] = "";
-
-        if (dict_get_str (dict, "subop", &subop) != 0) {
-                ret = -1;
-                goto out;
-        }
-
-        if (strcmp (subop, "get") != 0 && strcmp (subop, "get-all") != 0) {
-                ret = xmlTextWriterWriteFormatElement (writer,
-                                 (xmlChar *)"message",
-                                 "%s",GEOREP" config updated successfully" );
-                XML_RET_CHECK_AND_GOTO (ret, out);
-                ret = 0;
-                goto out;
-        }
+    runner_t runner = {
+        0,
+    };
+    char *subop = NULL;
+    char *gwd = NULL;
+    char *slave = NULL;
+    char *confpath = NULL;
+    char *master = NULL;
+    char *op_name = NULL;
+    int ret = -1;
+    char conf_path[PATH_MAX] = "";
+
+    if (dict_get_str(dict, "subop", &subop) != 0) {
+        ret = -1;
+        goto out;
+    }
+
+    if (strcmp(subop, "get") != 0 && strcmp(subop, "get-all") != 0) {
+        ret = xmlTextWriterWriteFormatElement(
+            writer, (xmlChar *)"message", "%s",
+            GEOREP " config updated successfully");
+        XML_RET_CHECK_AND_GOTO(ret, out);
+        ret = 0;
+        goto out;
+    }
 
-        if (dict_get_str (dict, "glusterd_workdir", &gwd) != 0 ||
-            dict_get_str (dict, "slave", &slave) != 0) {
-                ret = -1;
-                goto out;
-        }
+    if (dict_get_str(dict, "glusterd_workdir", &gwd) != 0 ||
+        dict_get_str(dict, "slave", &slave) != 0) {
+        ret = -1;
+        goto out;
+    }
 
-        if (dict_get_str (dict, "master", &master) != 0)
-                master = NULL;
+    if (dict_get_str(dict, "master", &master) != 0)
+        master = NULL;
 
-        if (dict_get_str (dict, "op_name", &op_name) != 0)
-                op_name = NULL;
+    if (dict_get_str(dict, "op_name", &op_name) != 0)
+        op_name = NULL;
 
-        ret = dict_get_str (dict, "conf_path", &confpath);
-        if (!confpath) {
-                ret = snprintf (conf_path, sizeof (conf_path) - 1,
-                                "%s/"GEOREP"/gsyncd_template.conf", gwd);
-                conf_path[ret] = '\0';
-                confpath = conf_path;
-        }
+    ret = dict_get_str(dict, "conf_path", &confpath);
+    if (!confpath) {
+        ret = snprintf(conf_path, sizeof(conf_path) - 1,
+                       "%s/" GEOREP "/gsyncd_template.conf", gwd);
+        conf_path[ret] = '\0';
+        confpath = conf_path;
+    }
 
-        runinit (&runner);
-        runner_add_args (&runner, GSYNCD_PREFIX"/gsyncd", "-c", NULL);
-        runner_argprintf (&runner, "%s", confpath);
-        runner_argprintf (&runner, "--iprefix=%s", DATADIR);
+    runinit(&runner);
+    runner_add_args(&runner, GSYNCD_PREFIX "/gsyncd", "-c", NULL);
+    runner_argprintf(&runner, "%s", confpath);
+    runner_argprintf(&runner, "--iprefix=%s", DATADIR);
 
-        if (master)
-                runner_argprintf (&runner, ":%s", master);
+    if (master)
+        runner_argprintf(&runner, ":%s", master);
 
-        runner_add_arg (&runner, slave);
-        runner_argprintf (&runner, "--config-%s", subop);
+    runner_add_arg(&runner, slave);
+    runner_argprintf(&runner, "--config-%s", subop);
 
-        if (op_name)
-                runner_add_arg (&runner, op_name);
+    if (op_name)
+        runner_add_arg(&runner, op_name);
 
-        ret =  get_gsync_config (&runner, _output_gsync_config,
-                                 writer, op_name);
+    ret = get_gsync_config(&runner, _output_gsync_config, writer, op_name);
 
 out:
-        gf_log ("cli", GF_LOG_DEBUG, "Returning %d", ret);
-        return ret;
+    gf_log("cli", GF_LOG_DEBUG, "Returning %d", ret);
+    return ret;
 }
 #endif
 
 #if (HAVE_LIB_XML)
 int
-cli_xml_output_vol_gsync_status (dict_t *dict,
-                                 xmlTextWriterPtr writer)
+cli_xml_output_vol_gsync_status(dict_t *dict, xmlTextWriterPtr writer)
 {
-        int                  ret                         = -1;
-        int                  i                           = 1;
-        int                  j                           = 0;
-        int                  count                       = 0;
-        const int            number_of_fields            = 20;
-        int                  closed                      = 1;
-        int                  session_closed              = 1;
-        gf_gsync_status_t  **status_values               = NULL;
-        gf_boolean_t         status_detail               = _gf_false;
-        char                 status_value_name[PATH_MAX] = "";
-        char                *tmp                         = NULL;
-        char                *volume                      = NULL;
-        char                *volume_next                 = NULL;
-        char                *slave                       = NULL;
-        char                *slave_next                  = NULL;
-        char                *title_values[]              = {"master_node",
-                                                            "",
-                                                            "master_brick",
-                                                            "slave_user",
-                                                            "slave",
-                                                            "slave_node",
-                                                            "status",
-                                                            "crawl_status",
-                                                            /* last_synced */
-                                                            "",
-                                                            "entry",
-                                                            "data",
-                                                            "meta",
-                                                            "failures",
-                                                           /* checkpoint_time */
-                                                            "",
-                                                         "checkpoint_completed",
-                                               /* checkpoint_completion_time */
-                                                            "",
-                                                            "master_node_uuid",
-                                                           /* last_synced_utc */
-                                                            "last_synced",
-                                                       /* checkpoint_time_utc */
-                                                            "checkpoint_time",
-                                            /* checkpoint_completion_time_utc */
-                                                 "checkpoint_completion_time"};
-
-        GF_ASSERT (dict);
-
-        ret = dict_get_int32 (dict, "gsync-count", &count);
-        if (ret)
-                goto out;
+    int ret = -1;
+    int i = 1;
+    int j = 0;
+    int count = 0;
+    const int number_of_fields = 20;
+    int closed = 1;
+    int session_closed = 1;
+    gf_gsync_status_t **status_values = NULL;
+    char status_value_name[PATH_MAX] = "";
+    char *tmp = NULL;
+    char *volume = NULL;
+    char *volume_next = NULL;
+    char *slave = NULL;
+    char *slave_next = NULL;
+    static const char *title_values[] = {
+        "master_node", "", "master_brick", "slave_user", "slave", "slave_node",
+        "status", "crawl_status",
+        /* last_synced */
+        "", "entry", "data", "meta", "failures",
+        /* checkpoint_time */
+        "", "checkpoint_completed",
+        /* checkpoint_completion_time */
+        "", "master_node_uuid",
+        /* last_synced_utc */
+        "last_synced",
+        /* checkpoint_time_utc */
+        "checkpoint_time",
+        /* checkpoint_completion_time_utc */
+        "checkpoint_completion_time"};
+
+    GF_ASSERT(dict);
+
+    ret = dict_get_int32(dict, "gsync-count", &count);
+    if (ret)
+        goto out;
+
+    status_values = GF_MALLOC(count * sizeof(gf_gsync_status_t *),
+                              gf_common_mt_char);
+    if (!status_values) {
+        ret = -1;
+        goto out;
+    }
 
-        status_detail = dict_get_str_boolean (dict, "status-detail",
-                                              _gf_false);
+    for (i = 0; i < count; i++) {
+        status_values[i] = GF_CALLOC(1, sizeof(gf_gsync_status_t),
+                                     gf_common_mt_char);
+        if (!status_values[i]) {
+            ret = -1;
+            goto out;
+        }
 
-        status_values = GF_CALLOC (count, sizeof (gf_gsync_status_t *),
-                              gf_common_mt_char);
-        if (!status_values) {
-                ret = -1;
-                goto out;
+        snprintf(status_value_name, sizeof(status_value_name), "status_value%d",
+                 i);
+
+        ret = dict_get_bin(dict, status_value_name,
+                           (void **)&(status_values[i]));
+        if (ret) {
+            gf_log("cli", GF_LOG_ERROR, "struct member empty.");
+            goto out;
         }
+    }
 
-        for (i = 0; i < count; i++) {
-                status_values[i] = GF_CALLOC (1, sizeof (gf_gsync_status_t),
-                                         gf_common_mt_char);
-                if (!status_values[i]) {
-                        ret = -1;
-                        goto out;
-                }
+    qsort(status_values, count, sizeof(gf_gsync_status_t *),
+          gf_gsync_status_t_comparator);
 
-                snprintf (status_value_name, sizeof (status_value_name),
-                          "status_value%d", i);
+    for (i = 0; i < count; i++) {
+        if (closed) {
+            ret = xmlTextWriterStartElement(writer, (xmlChar *)"volume");
+            XML_RET_CHECK_AND_GOTO(ret, out);
 
-                ret = dict_get_bin (dict, status_value_name,
-                                    (void **)&(status_values[i]));
-                if (ret) {
-                        gf_log ("cli", GF_LOG_ERROR,
-                                "struct member empty.");
-                        goto out;
-                }
-        }
+            tmp = get_struct_variable(1, status_values[i]);
+            if (!tmp) {
+                gf_log("cli", GF_LOG_ERROR, "struct member empty.");
+                ret = -1;
+                goto out;
+            }
 
-        qsort(status_values, count, sizeof (gf_gsync_status_t *),
-              gf_gsync_status_t_comparator);
-
-        for (i = 0; i < count; i++) {
-                if (closed) {
-                        ret = xmlTextWriterStartElement (writer,
-                                                         (xmlChar *)"volume");
-                        XML_RET_CHECK_AND_GOTO (ret, out);
-
-                        tmp = get_struct_variable (1, status_values[i]);
-                        if (!tmp) {
-                                gf_log ("cli", GF_LOG_ERROR,
-                                        "struct member empty.");
-                                ret = -1;
-                                goto out;
-                        }
-
-                        ret = xmlTextWriterWriteFormatElement (writer,
-                                                        (xmlChar *)"name",
-                                                        "%s",tmp);
-                        XML_RET_CHECK_AND_GOTO (ret, out);
-
-                        ret = xmlTextWriterStartElement (writer,
-                                                    (xmlChar *)"sessions");
-                        XML_RET_CHECK_AND_GOTO (ret, out);
-
-                        closed = 0;
-                }
+            ret = xmlTextWriterWriteFormatElement(writer, (xmlChar *)"name",
+                                                  "%s", tmp);
+            XML_RET_CHECK_AND_GOTO(ret, out);
 
-                if (session_closed) {
-                        ret = xmlTextWriterStartElement (writer,
-                                                         (xmlChar *)"session");
-                        XML_RET_CHECK_AND_GOTO (ret, out);
+            ret = xmlTextWriterStartElement(writer, (xmlChar *)"sessions");
+            XML_RET_CHECK_AND_GOTO(ret, out);
 
-                        session_closed = 0;
+            closed = 0;
+        }
 
-                        tmp = get_struct_variable (21, status_values[i]);
-                        if (!tmp) {
-                                gf_log ("cli", GF_LOG_ERROR,
-                                        "struct member empty.");
-                                ret = -1;
-                                goto out;
-                        }
+        if (session_closed) {
+            ret = xmlTextWriterStartElement(writer, (xmlChar *)"session");
+            XML_RET_CHECK_AND_GOTO(ret, out);
 
-                        ret = xmlTextWriterWriteFormatElement
-                                (writer, (xmlChar *)"session_slave", "%s", tmp);
-                        XML_RET_CHECK_AND_GOTO (ret, out);
-                }
+            session_closed = 0;
 
-                ret = xmlTextWriterStartElement (writer, (xmlChar *)"pair");
-                XML_RET_CHECK_AND_GOTO (ret, out);
+            tmp = get_struct_variable(21, status_values[i]);
+            if (!tmp) {
+                gf_log("cli", GF_LOG_ERROR, "struct member empty.");
+                ret = -1;
+                goto out;
+            }
 
-                for (j = 0; j < number_of_fields; j++) {
-                        /* XML ignore fields */
-                        if (strcmp(title_values[j], "") == 0)
-                                continue;
+            ret = xmlTextWriterWriteFormatElement(
+                writer, (xmlChar *)"session_slave", "%s", tmp);
+            XML_RET_CHECK_AND_GOTO(ret, out);
+        }
 
-                        tmp = get_struct_variable (j, status_values[i]);
-                        if (!tmp) {
-                                gf_log ("cli", GF_LOG_ERROR,
-                                        "struct member empty.");
-                                ret = -1;
-                                goto out;
-                        }
+        ret = xmlTextWriterStartElement(writer, (xmlChar *)"pair");
+        XML_RET_CHECK_AND_GOTO(ret, out);
 
-                        ret = xmlTextWriterWriteFormatElement (writer,
-                                                  (xmlChar *)title_values[j],
-                                                  "%s", tmp);
-                        XML_RET_CHECK_AND_GOTO (ret, out);
-                }
+        for (j = 0; j < number_of_fields; j++) {
+            /* XML ignore fields */
+            if (strcmp(title_values[j], "") == 0)
+                continue;
 
-                ret = xmlTextWriterEndElement (writer);
-                XML_RET_CHECK_AND_GOTO (ret, out);
+            tmp = get_struct_variable(j, status_values[i]);
+            if (!tmp) {
+                gf_log("cli", GF_LOG_ERROR, "struct member empty.");
+                ret = -1;
+                goto out;
+            }
 
-                if (i+1 < count) {
-                        slave = get_struct_variable (20, status_values[i]);
-                        slave_next = get_struct_variable (20,
-                                                          status_values[i+1]);
-                        volume = get_struct_variable (1, status_values[i]);
-                        volume_next = get_struct_variable (1,
-                                                           status_values[i+1]);
-                        if (!slave || !slave_next || !volume || !volume_next) {
-                                gf_log ("cli", GF_LOG_ERROR,
-                                        "struct member empty.");
-                                ret = -1;
-                                goto out;
-                        }
+            ret = xmlTextWriterWriteFormatElement(
+                writer, (xmlChar *)title_values[j], "%s", tmp);
+            XML_RET_CHECK_AND_GOTO(ret, out);
+        }
 
-                        if (strcmp (volume, volume_next)!=0) {
-                                closed = 1;
-                                session_closed = 1;
+        ret = xmlTextWriterEndElement(writer);
+        XML_RET_CHECK_AND_GOTO(ret, out);
 
-                                ret = xmlTextWriterEndElement (writer);
-                                XML_RET_CHECK_AND_GOTO (ret, out);
+        if (i + 1 < count) {
+            slave = get_struct_variable(20, status_values[i]);
+            slave_next = get_struct_variable(20, status_values[i + 1]);
+            volume = get_struct_variable(1, status_values[i]);
+            volume_next = get_struct_variable(1, status_values[i + 1]);
+            if (!slave || !slave_next || !volume || !volume_next) {
+                gf_log("cli", GF_LOG_ERROR, "struct member empty.");
+                ret = -1;
+                goto out;
+            }
 
-                                ret = xmlTextWriterEndElement (writer);
-                                XML_RET_CHECK_AND_GOTO (ret, out);
+            if (strcmp(volume, volume_next) != 0) {
+                closed = 1;
+                session_closed = 1;
 
-                                ret = xmlTextWriterEndElement (writer);
-                                XML_RET_CHECK_AND_GOTO (ret, out);
-                        } else if (strcmp (slave, slave_next)!=0) {
+                ret = xmlTextWriterEndElement(writer);
+                XML_RET_CHECK_AND_GOTO(ret, out);
 
-                                session_closed = 1;
+                ret = xmlTextWriterEndElement(writer);
+                XML_RET_CHECK_AND_GOTO(ret, out);
 
-                                ret = xmlTextWriterEndElement (writer);
-                                XML_RET_CHECK_AND_GOTO (ret, out);
-                        }
-                } else {
+                ret = xmlTextWriterEndElement(writer);
+                XML_RET_CHECK_AND_GOTO(ret, out);
+            } else if (strcmp(slave, slave_next) != 0) {
+                session_closed = 1;
 
-                        ret = xmlTextWriterEndElement (writer);
-                        XML_RET_CHECK_AND_GOTO (ret, out);
+                ret = xmlTextWriterEndElement(writer);
+                XML_RET_CHECK_AND_GOTO(ret, out);
+            }
+        } else {
+            ret = xmlTextWriterEndElement(writer);
+            XML_RET_CHECK_AND_GOTO(ret, out);
 
-                        ret = xmlTextWriterEndElement (writer);
-                        XML_RET_CHECK_AND_GOTO (ret, out);
+            ret = xmlTextWriterEndElement(writer);
+            XML_RET_CHECK_AND_GOTO(ret, out);
 
-                        ret = xmlTextWriterEndElement (writer);
-                        XML_RET_CHECK_AND_GOTO (ret, out);
-                }
+            ret = xmlTextWriterEndElement(writer);
+            XML_RET_CHECK_AND_GOTO(ret, out);
         }
+    }
 out:
-        gf_log ("cli",GF_LOG_DEBUG, "Returning %d", ret);
-        return ret;
+    if (status_values)
+        GF_FREE(status_values);
+    gf_log("cli", GF_LOG_DEBUG, "Returning %d", ret);
+    return ret;
 }
 #endif
 
 int
-cli_xml_output_vol_gsync (dict_t *dict, int op_ret, int op_errno,
-                          char *op_errstr)
+cli_xml_output_vol_gsync(dict_t *dict, int op_ret, int op_errno,
+                         char *op_errstr)
 {
 #if (HAVE_LIB_XML)
-        int                     ret = -1;
-        xmlTextWriterPtr        writer = NULL;
-        xmlDocPtr               doc = NULL;
-        char                    *master = NULL;
-        char                    *slave = NULL;
-        int                     type = 0;
-
-        GF_ASSERT (dict);
-
-        ret = cli_begin_xml_output (&writer, &doc);
-        if (ret)
-                goto out;
-
-        ret = cli_xml_output_common (writer, op_ret, op_errno, op_errstr);
-        if (ret)
-                goto out;
-
-        /* <geoRep> */
-        ret = xmlTextWriterStartElement (writer, (xmlChar *)"geoRep");
-        XML_RET_CHECK_AND_GOTO (ret, out);
-
-        ret = dict_get_int32 (dict, "type", &type);
-        if (ret) {
-                gf_log ("cli", GF_LOG_ERROR, "Failed to get type");
-                goto out;
-        }
-
-        switch (type) {
+    int ret = -1;
+    xmlTextWriterPtr writer = NULL;
+    xmlDocPtr doc = NULL;
+    char *master = NULL;
+    char *slave = NULL;
+    int type = 0;
+
+    GF_ASSERT(dict);
+
+    ret = cli_begin_xml_output(&writer, &doc);
+    if (ret)
+        goto out;
+
+    ret = cli_xml_output_common(writer, op_ret, op_errno, op_errstr);
+    if (ret)
+        goto out;
+
+    /* <geoRep> */
+    ret = xmlTextWriterStartElement(writer, (xmlChar *)"geoRep");
+    XML_RET_CHECK_AND_GOTO(ret, out);
+
+    ret = dict_get_int32(dict, "type", &type);
+    if (ret) {
+        gf_log("cli", GF_LOG_ERROR, "Failed to get type");
+        goto out;
+    }
+
+    switch (type) {
         case GF_GSYNC_OPTION_TYPE_START:
         case GF_GSYNC_OPTION_TYPE_STOP:
         case GF_GSYNC_OPTION_TYPE_PAUSE:
         case GF_GSYNC_OPTION_TYPE_RESUME:
         case GF_GSYNC_OPTION_TYPE_CREATE:
         case GF_GSYNC_OPTION_TYPE_DELETE:
-                if (dict_get_str (dict, "master", &master) != 0)
-                        master = "???";
-                if (dict_get_str (dict, "slave", &slave) != 0)
-                        slave = "???";
+            if (dict_get_str(dict, "master", &master) != 0)
+                master = "???";
+            if (dict_get_str(dict, "slave", &slave) != 0)
+                slave = "???";
 
-                ret = xmlTextWriterWriteFormatElement (writer,
-                                                       (xmlChar *)"master",
-                                                       "%s", master);
-                XML_RET_CHECK_AND_GOTO (ret, out);
+            ret = xmlTextWriterWriteFormatElement(writer, (xmlChar *)"master",
+                                                  "%s", master);
+            XML_RET_CHECK_AND_GOTO(ret, out);
 
-                ret = xmlTextWriterWriteFormatElement (writer,
-                                                       (xmlChar *)"slave",
-                                                       "%s", slave);
-                XML_RET_CHECK_AND_GOTO (ret, out);
+            ret = xmlTextWriterWriteFormatElement(writer, (xmlChar *)"slave",
+                                                  "%s", slave);
+            XML_RET_CHECK_AND_GOTO(ret, out);
 
-                break;
+            break;
 
         case GF_GSYNC_OPTION_TYPE_CONFIG:
-                if (op_ret == 0) {
-                        ret = xmlTextWriterStartElement (writer, (xmlChar *)"config");
-                        XML_RET_CHECK_AND_GOTO (ret, out);
+            if (op_ret == 0) {
+                ret = xmlTextWriterStartElement(writer, (xmlChar *)"config");
+                XML_RET_CHECK_AND_GOTO(ret, out);
 
-                        ret = cli_xml_generate_gsync_config (dict, writer);
-                        if (ret)
-                                goto out;
+                ret = cli_xml_generate_gsync_config(dict, writer);
+                if (ret)
+                    goto out;
 
-                        ret = xmlTextWriterEndElement (writer);
-                        XML_RET_CHECK_AND_GOTO (ret, out);
-                }
+                ret = xmlTextWriterEndElement(writer);
+                XML_RET_CHECK_AND_GOTO(ret, out);
+            }
 
-                break;
+            break;
         case GF_GSYNC_OPTION_TYPE_STATUS:
-                ret = cli_xml_output_vol_gsync_status (dict, writer);
-                break;
+            ret = cli_xml_output_vol_gsync_status(dict, writer);
+            if (ret) {
+                gf_log("cli", GF_LOG_DEBUG, "Failed to get gsync status");
+                goto out;
+            }
+            break;
         default:
-                ret = 0;
-                break;
-        }
+            ret = 0;
+            break;
+    }
 
-        /* </geoRep> */
-        ret = xmlTextWriterEndElement (writer);
-        XML_RET_CHECK_AND_GOTO (ret, out);
+    /* </geoRep> */
+    ret = xmlTextWriterEndElement(writer);
+    XML_RET_CHECK_AND_GOTO(ret, out);
 
-        ret = cli_end_xml_output (writer, doc);
+    ret = cli_end_xml_output(writer, doc);
 out:
-        gf_log ("cli",GF_LOG_DEBUG, "Returning %d", ret);
-        return ret;
+    gf_log("cli", GF_LOG_DEBUG, "Returning %d", ret);
+    return ret;
 #else
-        return 0;
+    return 0;
 #endif
 }
 
@@ -4433,56 +3842,113 @@ out:
  * @return 0 on success and -1 on failure
  */
 static int
-cli_xml_snapshot_create (xmlTextWriterPtr writer, xmlDocPtr doc, dict_t *dict)
+cli_xml_snapshot_create(xmlTextWriterPtr writer, xmlDocPtr doc, dict_t *dict)
 {
-        int     ret             = -1;
-        char   *str_value       = NULL;
-
-        GF_ASSERT (writer);
-        GF_ASSERT (doc);
-        GF_ASSERT (dict);
-
-        /* <snapCreate> */
-        ret = xmlTextWriterStartElement (writer, (xmlChar *)"snapCreate");
-        XML_RET_CHECK_AND_GOTO (ret, out);
-
-        /* <snapshot> */
-        ret = xmlTextWriterStartElement (writer, (xmlChar *)"snapshot");
-        XML_RET_CHECK_AND_GOTO (ret, out);
-
-        ret = dict_get_str (dict, "snapname", &str_value);
-        if (ret) {
-                gf_log ("cli", GF_LOG_ERROR, "Failed to get snap name");
-                goto out;
-        }
-
-        ret = xmlTextWriterWriteFormatElement (writer, (xmlChar *) "name",
-                                               "%s", str_value);
-        XML_RET_CHECK_AND_GOTO (ret, out);
-
-        ret = dict_get_str (dict, "snapuuid", &str_value);
-        if (ret) {
-                gf_log ("cli", GF_LOG_ERROR, "Failed to get snap uuid");
-                goto out;
-        }
-
-        ret = xmlTextWriterWriteFormatElement (writer, (xmlChar *) "uuid",
-                                               "%s", str_value);
-        XML_RET_CHECK_AND_GOTO (ret, out);
-
-        /* </snapshot> */
-        ret = xmlTextWriterEndElement (writer);
-        XML_RET_CHECK_AND_GOTO (ret, out);
-
-        /* </snapCreate> */
-        ret = xmlTextWriterEndElement (writer);
-        XML_RET_CHECK_AND_GOTO (ret, out);
-
-        ret = 0;
+    int ret = -1;
+    char *str_value = NULL;
+
+    GF_ASSERT(writer);
+    GF_ASSERT(doc);
+    GF_ASSERT(dict);
+
+    /* <snapCreate> */
+    ret = xmlTextWriterStartElement(writer, (xmlChar *)"snapCreate");
+    XML_RET_CHECK_AND_GOTO(ret, out);
+
+    /* <snapshot> */
+    ret = xmlTextWriterStartElement(writer, (xmlChar *)"snapshot");
+    XML_RET_CHECK_AND_GOTO(ret, out);
+
+    ret = dict_get_str(dict, "snapname", &str_value);
+    if (ret) {
+        gf_log("cli", GF_LOG_ERROR, "Failed to get snap name");
+        goto out;
+    }
+
+    ret = xmlTextWriterWriteFormatElement(writer, (xmlChar *)"name", "%s",
+                                          str_value);
+    XML_RET_CHECK_AND_GOTO(ret, out);
+
+    ret = dict_get_str(dict, "snapuuid", &str_value);
+    if (ret) {
+        gf_log("cli", GF_LOG_ERROR, "Failed to get snap uuid");
+        goto out;
+    }
+
+    ret = xmlTextWriterWriteFormatElement(writer, (xmlChar *)"uuid", "%s",
+                                          str_value);
+    XML_RET_CHECK_AND_GOTO(ret, out);
+
+    /* </snapshot> */
+    ret = xmlTextWriterEndElement(writer);
+    XML_RET_CHECK_AND_GOTO(ret, out);
+
+    /* </snapCreate> */
+    ret = xmlTextWriterEndElement(writer);
+    XML_RET_CHECK_AND_GOTO(ret, out);
+
+    ret = 0;
 out:
-        return ret;
+    return ret;
 }
 
+/* This function will generate snapshot clone output in xml format.
+ *
+ * @param writer        xmlTextWriterPtr
+ * @param doc           xmlDocPtr
+ * @param dict          dict containing create output
+ *
+ * @return 0 on success and -1 on failure
+ */
+static int
+cli_xml_snapshot_clone(xmlTextWriterPtr writer, xmlDocPtr doc, dict_t *dict)
+{
+    int ret = -1;
+    char *str_value = NULL;
+
+    GF_VALIDATE_OR_GOTO("cli", writer, out);
+    GF_VALIDATE_OR_GOTO("cli", doc, out);
+    GF_VALIDATE_OR_GOTO("cli", dict, out);
+
+    /* <CloneCreate> */
+    ret = xmlTextWriterStartElement(writer, (xmlChar *)"CloneCreate");
+    XML_RET_CHECK_AND_GOTO(ret, out);
+
+    /* <volume> */
+    ret = xmlTextWriterStartElement(writer, (xmlChar *)"volume");
+    XML_RET_CHECK_AND_GOTO(ret, out);
+
+    ret = dict_get_str(dict, "clonename", &str_value);
+    if (ret) {
+        gf_log("cli", GF_LOG_ERROR, "Failed to get clone name");
+        goto out;
+    }
+    ret = xmlTextWriterWriteFormatElement(writer, (xmlChar *)"name", "%s",
+                                          str_value);
+    XML_RET_CHECK_AND_GOTO(ret, out);
+
+    ret = dict_get_str(dict, "snapuuid", &str_value);
+    if (ret) {
+        gf_log("cli", GF_LOG_ERROR, "Failed to get clone uuid");
+        goto out;
+    }
+
+    ret = xmlTextWriterWriteFormatElement(writer, (xmlChar *)"uuid", "%s",
+                                          str_value);
+    XML_RET_CHECK_AND_GOTO(ret, out);
+
+    /* </volume> */
+    ret = xmlTextWriterEndElement(writer);
+    XML_RET_CHECK_AND_GOTO(ret, out);
+
+    /* </CloneCreate> */
+    ret = xmlTextWriterEndElement(writer);
+    XML_RET_CHECK_AND_GOTO(ret, out);
+
+    ret = 0;
+out:
+    return ret;
+}
 
 /* This function will generate snapshot restore output in xml format.
  *
@@ -4493,84 +3959,83 @@ out:
  * @return 0 on success and -1 on failure
  */
 static int
-cli_xml_snapshot_restore (xmlTextWriterPtr writer, xmlDocPtr doc, dict_t *dict)
+cli_xml_snapshot_restore(xmlTextWriterPtr writer, xmlDocPtr doc, dict_t *dict)
 {
-        int     ret             = -1;
-        char   *str_value       = NULL;
-
-        GF_ASSERT (writer);
-        GF_ASSERT (doc);
-        GF_ASSERT (dict);
-
-        /* <snapRestore> */
-        ret = xmlTextWriterStartElement (writer, (xmlChar *)"snapRestore");
-        XML_RET_CHECK_AND_GOTO (ret, out);
-
-        /* <volume> */
-        ret = xmlTextWriterStartElement (writer, (xmlChar *)"volume");
-        XML_RET_CHECK_AND_GOTO (ret, out);
-
-        ret = dict_get_str (dict, "volname", &str_value);
-        if (ret) {
-                gf_log ("cli", GF_LOG_ERROR, "Failed to get vol name");
-                goto out;
-        }
-
-        ret = xmlTextWriterWriteFormatElement (writer, (xmlChar *) "name",
-                                               "%s", str_value);
-        XML_RET_CHECK_AND_GOTO (ret, out);
-
-        ret = dict_get_str (dict, "volid", &str_value);
-        if (ret) {
-                gf_log ("cli", GF_LOG_ERROR, "Failed to get volume id");
-                goto out;
-        }
-
-        ret = xmlTextWriterWriteFormatElement (writer, (xmlChar *) "uuid",
-                                               "%s", str_value);
-        XML_RET_CHECK_AND_GOTO (ret, out);
-
-        /* </volume> */
-        ret = xmlTextWriterEndElement (writer);
-        XML_RET_CHECK_AND_GOTO (ret, out);
-
-
-        /* <snapshot> */
-        ret = xmlTextWriterStartElement (writer, (xmlChar *)"snapshot");
-        XML_RET_CHECK_AND_GOTO (ret, out);
-
-        ret = dict_get_str (dict, "snapname", &str_value);
-        if (ret) {
-                gf_log ("cli", GF_LOG_ERROR, "Failed to get snap name");
-                goto out;
-        }
-
-        ret = xmlTextWriterWriteFormatElement (writer, (xmlChar *) "name",
-                                               "%s", str_value);
-        XML_RET_CHECK_AND_GOTO (ret, out);
-
-        ret = dict_get_str (dict, "snapuuid", &str_value);
-        if (ret) {
-                gf_log ("cli", GF_LOG_ERROR, "Failed to get snap uuid");
-                goto out;
-        }
-
-        ret = xmlTextWriterWriteFormatElement (writer, (xmlChar *) "uuid",
-                                               "%s", str_value);
-        XML_RET_CHECK_AND_GOTO (ret, out);
-
-        /* </snapshot> */
-        ret = xmlTextWriterEndElement (writer);
-        XML_RET_CHECK_AND_GOTO (ret, out);
-
-        /* </snapRestore> */
-        ret = xmlTextWriterEndElement (writer);
-        XML_RET_CHECK_AND_GOTO (ret, out);
-
-        ret = 0;
+    int ret = -1;
+    char *str_value = NULL;
+
+    GF_ASSERT(writer);
+    GF_ASSERT(doc);
+    GF_ASSERT(dict);
+
+    /* <snapRestore> */
+    ret = xmlTextWriterStartElement(writer, (xmlChar *)"snapRestore");
+    XML_RET_CHECK_AND_GOTO(ret, out);
+
+    /* <volume> */
+    ret = xmlTextWriterStartElement(writer, (xmlChar *)"volume");
+    XML_RET_CHECK_AND_GOTO(ret, out);
+
+    ret = dict_get_str(dict, "volname", &str_value);
+    if (ret) {
+        gf_log("cli", GF_LOG_ERROR, "Failed to get vol name");
+        goto out;
+    }
+
+    ret = xmlTextWriterWriteFormatElement(writer, (xmlChar *)"name", "%s",
+                                          str_value);
+    XML_RET_CHECK_AND_GOTO(ret, out);
+
+    ret = dict_get_str(dict, "volid", &str_value);
+    if (ret) {
+        gf_log("cli", GF_LOG_ERROR, "Failed to get volume id");
+        goto out;
+    }
+
+    ret = xmlTextWriterWriteFormatElement(writer, (xmlChar *)"uuid", "%s",
+                                          str_value);
+    XML_RET_CHECK_AND_GOTO(ret, out);
+
+    /* </volume> */
+    ret = xmlTextWriterEndElement(writer);
+    XML_RET_CHECK_AND_GOTO(ret, out);
+
+    /* <snapshot> */
+    ret = xmlTextWriterStartElement(writer, (xmlChar *)"snapshot");
+    XML_RET_CHECK_AND_GOTO(ret, out);
+
+    ret = dict_get_str(dict, "snapname", &str_value);
+    if (ret) {
+        gf_log("cli", GF_LOG_ERROR, "Failed to get snap name");
+        goto out;
+    }
+
+    ret = xmlTextWriterWriteFormatElement(writer, (xmlChar *)"name", "%s",
+                                          str_value);
+    XML_RET_CHECK_AND_GOTO(ret, out);
+
+    ret = dict_get_str(dict, "snapuuid", &str_value);
+    if (ret) {
+        gf_log("cli", GF_LOG_ERROR, "Failed to get snap uuid");
+        goto out;
+    }
+
+    ret = xmlTextWriterWriteFormatElement(writer, (xmlChar *)"uuid", "%s",
+                                          str_value);
+    XML_RET_CHECK_AND_GOTO(ret, out);
+
+    /* </snapshot> */
+    ret = xmlTextWriterEndElement(writer);
+    XML_RET_CHECK_AND_GOTO(ret, out);
+
+    /* </snapRestore> */
+    ret = xmlTextWriterEndElement(writer);
+    XML_RET_CHECK_AND_GOTO(ret, out);
+
+    ret = 0;
 out:
 
-        return ret;
+    return ret;
 }
 
 /* This function will generate snapshot list output in xml format.
@@ -4582,57 +4047,57 @@ out:
  * @return 0 on success and -1 on failure
  */
 static int
-cli_xml_snapshot_list (xmlTextWriterPtr writer, xmlDocPtr doc, dict_t *dict)
+cli_xml_snapshot_list(xmlTextWriterPtr writer, xmlDocPtr doc, dict_t *dict)
 {
-        int     ret             = -1;
-        int     i               = 0;
-        int     snapcount       = 0;
-        char   *str_value       = NULL;
-        char    key[PATH_MAX]   = "";
+    int ret = -1;
+    int i = 0;
+    int snapcount = 0;
+    char *str_value = NULL;
+    char key[PATH_MAX] = "";
 
-        GF_ASSERT (writer);
-        GF_ASSERT (doc);
-        GF_ASSERT (dict);
+    GF_ASSERT(writer);
+    GF_ASSERT(doc);
+    GF_ASSERT(dict);
 
-        /* <snapList> */
-        ret = xmlTextWriterStartElement (writer, (xmlChar *)"snapList");
-        XML_RET_CHECK_AND_GOTO (ret, out);
+    /* <snapList> */
+    ret = xmlTextWriterStartElement(writer, (xmlChar *)"snapList");
+    XML_RET_CHECK_AND_GOTO(ret, out);
 
-        ret = dict_get_int32 (dict, "snapcount", &snapcount);
-        if (ret) {
-                gf_log ("cli", GF_LOG_ERROR, "Failed to get snapcount");
-                goto out;
-        }
+    ret = dict_get_int32(dict, "snapcount", &snapcount);
+    if (ret) {
+        gf_log("cli", GF_LOG_ERROR, "Failed to get snapcount");
+        goto out;
+    }
 
-        ret = xmlTextWriterWriteFormatElement (writer, (xmlChar *) "count",
-                                               "%d", snapcount);
-        XML_RET_CHECK_AND_GOTO (ret, out);
+    ret = xmlTextWriterWriteFormatElement(writer, (xmlChar *)"count", "%d",
+                                          snapcount);
+    XML_RET_CHECK_AND_GOTO(ret, out);
 
-        for (i = 1; i <= snapcount; ++i) {
-                ret = snprintf (key, sizeof (key), "snapname%d", i);
-                if (ret < 0) {
-                        goto out;
-                }
+    for (i = 1; i <= snapcount; ++i) {
+        ret = snprintf(key, sizeof(key), "snapname%d", i);
+        if (ret < 0) {
+            goto out;
+        }
 
-                ret = dict_get_str (dict, key, &str_value);
-                if (ret) {
-                        gf_log ("cli", GF_LOG_ERROR, "Could not get %s ", key);
-                        goto out;
-                } else {
-                        ret = xmlTextWriterWriteFormatElement (writer,
-                                        (xmlChar *)"snapshot", "%s", str_value);
-                        XML_RET_CHECK_AND_GOTO (ret, out);
-                }
+        ret = dict_get_str(dict, key, &str_value);
+        if (ret) {
+            gf_log("cli", GF_LOG_ERROR, "Could not get %s ", key);
+            goto out;
+        } else {
+            ret = xmlTextWriterWriteFormatElement(writer, (xmlChar *)"snapshot",
+                                                  "%s", str_value);
+            XML_RET_CHECK_AND_GOTO(ret, out);
         }
+    }
 
-        /* </snapList> */
-        ret = xmlTextWriterEndElement (writer);
-        XML_RET_CHECK_AND_GOTO (ret, out);
+    /* </snapList> */
+    ret = xmlTextWriterEndElement(writer);
+    XML_RET_CHECK_AND_GOTO(ret, out);
 
-        ret = 0;
+    ret = 0;
 out:
 
-        return ret;
+    return ret;
 }
 
 /* This function will generate xml output for origin volume
@@ -4646,66 +4111,66 @@ out:
  * @return 0 on success and -1 on failure
  */
 static int
-cli_xml_snapshot_info_orig_vol (xmlTextWriterPtr writer, xmlDocPtr doc,
-                                dict_t *dict, char *keyprefix)
+cli_xml_snapshot_info_orig_vol(xmlTextWriterPtr writer, xmlDocPtr doc,
+                               dict_t *dict, char *keyprefix)
 {
-        int     ret             = -1;
-        int     value           = 0;
-        char   *buffer          = NULL;
-        char    key [PATH_MAX]  = "";
+    int ret = -1;
+    int value = 0;
+    char *buffer = NULL;
+    char key[PATH_MAX] = "";
 
-        GF_ASSERT (dict);
-        GF_ASSERT (keyprefix);
-        GF_ASSERT (writer);
-        GF_ASSERT (doc);
+    GF_ASSERT(dict);
+    GF_ASSERT(keyprefix);
+    GF_ASSERT(writer);
+    GF_ASSERT(doc);
 
-        /* <originVolume> */
-        ret = xmlTextWriterStartElement (writer, (xmlChar *)"originVolume");
-        XML_RET_CHECK_AND_GOTO (ret, out);
+    /* <originVolume> */
+    ret = xmlTextWriterStartElement(writer, (xmlChar *)"originVolume");
+    XML_RET_CHECK_AND_GOTO(ret, out);
 
-        snprintf (key, sizeof (key), "%sorigin-volname", keyprefix);
+    snprintf(key, sizeof(key), "%sorigin-volname", keyprefix);
 
-        ret = dict_get_str (dict, key, &buffer);
-        if (ret) {
-                gf_log ("cli", GF_LOG_WARNING, "Failed to get %s", key);
-                goto out;
-        }
+    ret = dict_get_str(dict, key, &buffer);
+    if (ret) {
+        gf_log("cli", GF_LOG_WARNING, "Failed to get %s", key);
+        goto out;
+    }
 
-        ret = xmlTextWriterWriteFormatElement (writer, (xmlChar *) "name",
-                                               "%s", buffer);
-        XML_RET_CHECK_AND_GOTO (ret, out);
+    ret = xmlTextWriterWriteFormatElement(writer, (xmlChar *)"name", "%s",
+                                          buffer);
+    XML_RET_CHECK_AND_GOTO(ret, out);
 
-        snprintf (key, sizeof (key), "%ssnapcount", keyprefix);
+    snprintf(key, sizeof(key), "%ssnapcount", keyprefix);
 
-        ret = dict_get_int32 (dict, key, &value);
-        if (ret) {
-                gf_log ("cli", GF_LOG_ERROR, "Failed to get %s", key);
-                goto out;
-        }
+    ret = dict_get_int32(dict, key, &value);
+    if (ret) {
+        gf_log("cli", GF_LOG_ERROR, "Failed to get %s", key);
+        goto out;
+    }
 
-        ret = xmlTextWriterWriteFormatElement (writer, (xmlChar *) "snapCount",
-                                               "%d", value);
-        XML_RET_CHECK_AND_GOTO (ret, out);
+    ret = xmlTextWriterWriteFormatElement(writer, (xmlChar *)"snapCount", "%d",
+                                          value);
+    XML_RET_CHECK_AND_GOTO(ret, out);
 
-        snprintf (key, sizeof (key), "%ssnaps-available", keyprefix);
+    snprintf(key, sizeof(key), "%ssnaps-available", keyprefix);
 
-        ret = dict_get_int32 (dict, key, &value);
-        if (ret) {
-                gf_log ("cli", GF_LOG_ERROR, "Failed to get %s", key);
-                goto out;
-        }
+    ret = dict_get_int32(dict, key, &value);
+    if (ret) {
+        gf_log("cli", GF_LOG_ERROR, "Failed to get %s", key);
+        goto out;
+    }
 
-        ret = xmlTextWriterWriteFormatElement (writer,
-                                (xmlChar *) "snapRemaining", "%d", value);
-        XML_RET_CHECK_AND_GOTO (ret, out);
+    ret = xmlTextWriterWriteFormatElement(writer, (xmlChar *)"snapRemaining",
+                                          "%d", value);
+    XML_RET_CHECK_AND_GOTO(ret, out);
 
-        /* </originVolume> */
-        ret = xmlTextWriterEndElement (writer);
-        XML_RET_CHECK_AND_GOTO (ret, out);
+    /* </originVolume> */
+    ret = xmlTextWriterEndElement(writer);
+    XML_RET_CHECK_AND_GOTO(ret, out);
 
-        ret = 0;
+    ret = 0;
 out:
-        return ret;
+    return ret;
 }
 
 /* This function will generate xml output of snapshot volume info.
@@ -4720,67 +4185,74 @@ out:
  * @return 0 on success and -1 on failure
  */
 static int
-cli_xml_snapshot_info_snap_vol (xmlTextWriterPtr writer, xmlDocPtr doc,
-                                dict_t *dict, char *keyprefix,
-                                gf_boolean_t snap_driven)
+cli_xml_snapshot_info_snap_vol(xmlTextWriterPtr writer, xmlDocPtr doc,
+                               dict_t *dict, char *keyprefix,
+                               gf_boolean_t snap_driven)
 {
-        char            key [PATH_MAX]          = "";
-        char           *buffer                  = NULL;
-        int             value                   = 0;
-        int             ret                     = -1;
-
-        GF_ASSERT (dict);
-        GF_ASSERT (keyprefix);
-        GF_ASSERT (writer);
-        GF_ASSERT (doc);
-
-        /* <snapVolume> */
-        ret = xmlTextWriterStartElement (writer, (xmlChar *)"snapVolume");
-        XML_RET_CHECK_AND_GOTO (ret, out);
-
-        snprintf (key, sizeof (key), "%s.volname", keyprefix);
-
-        ret = dict_get_str (dict, key, &buffer);
+    char key[PATH_MAX] = "";
+    char *buffer = NULL;
+    int ret = -1;
+
+    GF_ASSERT(dict);
+    GF_ASSERT(keyprefix);
+    GF_ASSERT(writer);
+    GF_ASSERT(doc);
+
+    /* <snapVolume> */
+    ret = xmlTextWriterStartElement(writer, (xmlChar *)"snapVolume");
+    XML_RET_CHECK_AND_GOTO(ret, out);
+
+    ret = snprintf(key, sizeof(key), "%s.volname", keyprefix);
+    if (ret < 0)
+        goto out;
+
+    ret = dict_get_str(dict, key, &buffer);
+    if (ret) {
+        gf_log("cli", GF_LOG_ERROR, "Failed to get %s", key);
+        goto out;
+    }
+
+    ret = xmlTextWriterWriteFormatElement(writer, (xmlChar *)"name", "%s",
+                                          buffer);
+    XML_RET_CHECK_AND_GOTO(ret, out);
+
+    ret = snprintf(key, sizeof(key), "%s.vol-status", keyprefix);
+    if (ret < 0)
+        goto out;
+
+    ret = dict_get_str(dict, key, &buffer);
+    if (ret) {
+        gf_log("cli", GF_LOG_ERROR, "Failed to get %s", key);
+        goto out;
+    }
+
+    ret = xmlTextWriterWriteFormatElement(writer, (xmlChar *)"status", "%s",
+                                          buffer);
+    XML_RET_CHECK_AND_GOTO(ret, out);
+
+    /* If the command is snap_driven then we need to show origin volume
+     * info. Else this is shown in the start of info display.*/
+    if (snap_driven) {
+        ret = snprintf(key, sizeof(key), "%s.", keyprefix);
+        if (ret < 0)
+            goto out;
+
+        ret = cli_xml_snapshot_info_orig_vol(writer, doc, dict, key);
         if (ret) {
-                gf_log ("cli", GF_LOG_ERROR, "Failed to get %s", key);
-                goto out;
+            gf_log("cli", GF_LOG_ERROR,
+                   "Failed to create "
+                   "xml output for snapshot's origin volume");
+            goto out;
         }
+    }
 
-        ret = xmlTextWriterWriteFormatElement (writer, (xmlChar *) "name",
-                                               "%s", buffer);
-        XML_RET_CHECK_AND_GOTO (ret, out);
-
-        snprintf (key, sizeof (key), "%s.vol-status", keyprefix);
+    /* </snapVolume> */
+    ret = xmlTextWriterEndElement(writer);
+    XML_RET_CHECK_AND_GOTO(ret, out);
 
-        ret = dict_get_str (dict, key, &buffer);
-        if (ret) {
-                gf_log ("cli", GF_LOG_ERROR, "Failed to get %s", key);
-                goto out;
-        }
-
-        ret = xmlTextWriterWriteFormatElement (writer, (xmlChar *) "status",
-                                               "%s", buffer);
-        XML_RET_CHECK_AND_GOTO (ret, out);
-
-        /* If the command is snap_driven then we need to show origin volume
-         * info. Else this is shown in the start of info display.*/
-        if (snap_driven) {
-                snprintf (key, sizeof (key), "%s.", keyprefix);
-                ret = cli_xml_snapshot_info_orig_vol (writer, doc, dict, key);
-                if (ret) {
-                        gf_log ("cli", GF_LOG_ERROR, "Failed to create "
-                                "xml output for snapshot's origin volume");
-                        goto out;
-                }
-        }
-
-        /* </snapVolume> */
-        ret = xmlTextWriterEndElement (writer);
-        XML_RET_CHECK_AND_GOTO (ret, out);
-
-        ret = 0;
+    ret = 0;
 out:
-        return ret;
+    return ret;
 }
 
 /* This function will generate snapshot info of individual snapshot
@@ -4796,112 +4268,118 @@ out:
  * @return 0 on success and -1 on failure
  */
 static int
-cli_xml_snapshot_info_per_snap (xmlTextWriterPtr writer, xmlDocPtr doc,
-                                dict_t *dict, char *keyprefix,
-                                gf_boolean_t snap_driven)
+cli_xml_snapshot_info_per_snap(xmlTextWriterPtr writer, xmlDocPtr doc,
+                               dict_t *dict, char *keyprefix,
+                               gf_boolean_t snap_driven)
 {
-        char            key_buffer[PATH_MAX]    = "";
-        char           *buffer                  = NULL;
-        int             volcount                = 0;
-        int             ret                     = -1;
-        int             i                       = 0;
-
-        GF_ASSERT (dict);
-        GF_ASSERT (keyprefix);
-        GF_ASSERT (writer);
-        GF_ASSERT (doc);
-
-        /* <snapshot> */
-        ret = xmlTextWriterStartElement (writer, (xmlChar *)"snapshot");
-        XML_RET_CHECK_AND_GOTO (ret, out);
-
-        snprintf (key_buffer, sizeof (key_buffer), "%s.snapname",
-                  keyprefix);
-
-        ret = dict_get_str (dict, key_buffer, &buffer);
-        if (ret) {
-                gf_log ("cli", GF_LOG_ERROR, "Unable to fetch snapname %s ",
-                        key_buffer);
-                goto out;
-        }
-
-        ret = xmlTextWriterWriteFormatElement (writer, (xmlChar *) "name",
-                                               "%s", buffer);
-        XML_RET_CHECK_AND_GOTO (ret, out);
-
-        snprintf (key_buffer, sizeof (key_buffer), "%s.snap-id", keyprefix);
-
-        ret = dict_get_str (dict, key_buffer, &buffer);
-        if (ret) {
-                gf_log ("cli", GF_LOG_ERROR, "Unable to fetch snap-id %s ",
-                                key_buffer);
-                goto out;
-        }
-
-        ret = xmlTextWriterWriteFormatElement (writer, (xmlChar *) "uuid",
-                                               "%s", buffer);
-        XML_RET_CHECK_AND_GOTO (ret, out);
-
-        snprintf (key_buffer, sizeof (key_buffer), "%s.snap-desc", keyprefix);
-
-        ret = dict_get_str (dict, key_buffer, &buffer);
-        if (!ret) {
-                ret = xmlTextWriterWriteFormatElement (writer,
-                                                (xmlChar *) "description",
-                                                "%s", buffer);
-        } else {
-                ret = xmlTextWriterWriteFormatElement (writer,
-                                                (xmlChar *) "description",
-                                                "%s", "");
-        }
-        XML_RET_CHECK_AND_GOTO (ret, out);
-
-        snprintf (key_buffer, sizeof (key_buffer), "%s.snap-time", keyprefix);
-
-        ret = dict_get_str (dict, key_buffer, &buffer);
+    char key_buffer[PATH_MAX] = "";
+    char *buffer = NULL;
+    int volcount = 0;
+    int ret = -1;
+    int i = 0;
+
+    GF_ASSERT(dict);
+    GF_ASSERT(keyprefix);
+    GF_ASSERT(writer);
+    GF_ASSERT(doc);
+
+    /* <snapshot> */
+    ret = xmlTextWriterStartElement(writer, (xmlChar *)"snapshot");
+    XML_RET_CHECK_AND_GOTO(ret, out);
+
+    ret = snprintf(key_buffer, sizeof(key_buffer), "%s.snapname", keyprefix);
+    if (ret < 0)
+        goto out;
+
+    ret = dict_get_str(dict, key_buffer, &buffer);
+    if (ret) {
+        gf_log("cli", GF_LOG_ERROR, "Unable to fetch snapname %s ", key_buffer);
+        goto out;
+    }
+
+    ret = xmlTextWriterWriteFormatElement(writer, (xmlChar *)"name", "%s",
+                                          buffer);
+    XML_RET_CHECK_AND_GOTO(ret, out);
+
+    ret = snprintf(key_buffer, sizeof(key_buffer), "%s.snap-id", keyprefix);
+    if (ret < 0)
+        goto out;
+
+    ret = dict_get_str(dict, key_buffer, &buffer);
+    if (ret) {
+        gf_log("cli", GF_LOG_ERROR, "Unable to fetch snap-id %s ", key_buffer);
+        goto out;
+    }
+
+    ret = xmlTextWriterWriteFormatElement(writer, (xmlChar *)"uuid", "%s",
+                                          buffer);
+    XML_RET_CHECK_AND_GOTO(ret, out);
+
+    ret = snprintf(key_buffer, sizeof(key_buffer), "%s.snap-desc", keyprefix);
+    if (ret < 0)
+        goto out;
+
+    ret = dict_get_str(dict, key_buffer, &buffer);
+    if (!ret) {
+        ret = xmlTextWriterWriteFormatElement(writer, (xmlChar *)"description",
+                                              "%s", buffer);
+    } else {
+        ret = xmlTextWriterWriteFormatElement(writer, (xmlChar *)"description",
+                                              "%s", "");
+    }
+    XML_RET_CHECK_AND_GOTO(ret, out);
+
+    ret = snprintf(key_buffer, sizeof(key_buffer), "%s.snap-time", keyprefix);
+    if (ret < 0)
+        goto out;
+
+    ret = dict_get_str(dict, key_buffer, &buffer);
+    if (ret) {
+        gf_log("cli", GF_LOG_ERROR, "Unable to fetch snap-time %s ", keyprefix);
+        goto out;
+    }
+
+    ret = xmlTextWriterWriteFormatElement(writer, (xmlChar *)"createTime", "%s",
+                                          buffer);
+    XML_RET_CHECK_AND_GOTO(ret, out);
+
+    ret = snprintf(key_buffer, sizeof(key_buffer), "%s.vol-count", keyprefix);
+    if (ret < 0)
+        goto out;
+
+    ret = dict_get_int32(dict, key_buffer, &volcount);
+    if (ret) {
+        gf_log("cli", GF_LOG_ERROR, "Fail to get snap vol count");
+        goto out;
+    }
+
+    ret = xmlTextWriterWriteFormatElement(writer, (xmlChar *)"volCount", "%d",
+                                          volcount);
+    XML_RET_CHECK_AND_GOTO(ret, out);
+
+    ret = dict_get_int32(dict, key_buffer, &volcount);
+    /* Display info of each snapshot volume */
+    for (i = 1; i <= volcount; i++) {
+        ret = snprintf(key_buffer, sizeof(key_buffer), "%s.vol%d", keyprefix,
+                       i);
+        if (ret < 0)
+            goto out;
+
+        ret = cli_xml_snapshot_info_snap_vol(writer, doc, dict, key_buffer,
+                                             snap_driven);
         if (ret) {
-                gf_log ("cli", GF_LOG_ERROR, "Unable to fetch snap-time %s ",
-                                keyprefix);
-                goto out;
+            gf_log("cli", GF_LOG_ERROR,
+                   "Could not list "
+                   "details of volume in a snap");
+            goto out;
         }
+    }
 
-        ret = xmlTextWriterWriteFormatElement (writer,
-                                               (xmlChar *) "createTime",
-                                               "%s", buffer);
-        XML_RET_CHECK_AND_GOTO (ret, out);
-
-        snprintf (key_buffer, sizeof (key_buffer), "%s.vol-count", keyprefix);
-        ret = dict_get_int32 (dict, key_buffer, &volcount);
-        if (ret) {
-                gf_log ("cli", GF_LOG_ERROR, "Fail to get snap vol count");
-                goto out;
-        }
-
-        ret = xmlTextWriterWriteFormatElement (writer,
-                                               (xmlChar *) "volCount",
-                                               "%d", volcount);
-        XML_RET_CHECK_AND_GOTO (ret, out);
-
-        ret = dict_get_int32 (dict, key_buffer, &volcount);
-        /* Display info of each snapshot volume */
-        for (i = 1 ; i <= volcount ; i++) {
-                snprintf (key_buffer, sizeof (key_buffer), "%s.vol%d",
-                          keyprefix, i);
-
-                ret = cli_xml_snapshot_info_snap_vol (writer, doc, dict,
-                                                      key_buffer, snap_driven);
-                if (ret) {
-                        gf_log ("cli", GF_LOG_ERROR, "Could not list "
-                                        "details of volume in a snap");
-                        goto out;
-                }
-        }
-
-        /* </snapshot> */
-        ret = xmlTextWriterEndElement (writer);
-        XML_RET_CHECK_AND_GOTO (ret, out);
+    /* </snapshot> */
+    ret = xmlTextWriterEndElement(writer);
+    XML_RET_CHECK_AND_GOTO(ret, out);
 out:
-        return ret;
+    return ret;
 }
 
 /* This function will generate snapshot info output in xml format.
@@ -4913,74 +4391,74 @@ out:
  * @return 0 on success and -1 on failure
  */
 static int
-cli_xml_snapshot_info (xmlTextWriterPtr writer, xmlDocPtr doc, dict_t *dict)
+cli_xml_snapshot_info(xmlTextWriterPtr writer, xmlDocPtr doc, dict_t *dict)
 {
-        int             ret             = -1;
-        int             i               = 0;
-        int             snapcount       = 0;
-        char            key [PATH_MAX]  = "";
-        char           *str_value       = NULL;
-        gf_boolean_t    snap_driven     = _gf_false;
-
-        GF_ASSERT (writer);
-        GF_ASSERT (doc);
-        GF_ASSERT (dict);
-
-        /* <snapInfo> */
-        ret = xmlTextWriterStartElement (writer, (xmlChar *)"snapInfo");
-        XML_RET_CHECK_AND_GOTO (ret, out);
-
-        snap_driven = dict_get_str_boolean (dict, "snap-driven", _gf_false);
-
-        /* If the approach is volume based then we should display orgin volume
-         * information first followed by per snap info*/
-        if (!snap_driven) {
-                ret = cli_xml_snapshot_info_orig_vol (writer, doc, dict, "");
-                if (ret) {
-                        gf_log ("cli", GF_LOG_ERROR, "Failed to create "
-                                "xml output for snapshot's origin volume");
-                        goto out;
-                }
-        }
-
-        ret = dict_get_int32 (dict, "snapcount", &snapcount);
+    int ret = -1;
+    int i = 0;
+    int snapcount = 0;
+    char key[PATH_MAX] = "";
+    gf_boolean_t snap_driven = _gf_false;
+
+    GF_ASSERT(writer);
+    GF_ASSERT(doc);
+    GF_ASSERT(dict);
+
+    /* <snapInfo> */
+    ret = xmlTextWriterStartElement(writer, (xmlChar *)"snapInfo");
+    XML_RET_CHECK_AND_GOTO(ret, out);
+
+    snap_driven = dict_get_str_boolean(dict, "snap-driven", _gf_false);
+
+    /* If the approach is volume based then we should display origin volume
+     * information first followed by per snap info*/
+    if (!snap_driven) {
+        ret = cli_xml_snapshot_info_orig_vol(writer, doc, dict, "");
         if (ret) {
-                gf_log ("cli", GF_LOG_ERROR, "Failed to get snapcount");
-                goto out;
+            gf_log("cli", GF_LOG_ERROR,
+                   "Failed to create "
+                   "xml output for snapshot's origin volume");
+            goto out;
         }
+    }
 
-        ret = xmlTextWriterWriteFormatElement (writer, (xmlChar *) "count",
-                                               "%d", snapcount);
-        XML_RET_CHECK_AND_GOTO (ret, out);
+    ret = dict_get_int32(dict, "snapcount", &snapcount);
+    if (ret) {
+        gf_log("cli", GF_LOG_ERROR, "Failed to get snapcount");
+        goto out;
+    }
 
-        /* <snapshots> */
-        ret = xmlTextWriterStartElement (writer, (xmlChar *)"snapshots");
-        XML_RET_CHECK_AND_GOTO (ret, out);
+    ret = xmlTextWriterWriteFormatElement(writer, (xmlChar *)"count", "%d",
+                                          snapcount);
+    XML_RET_CHECK_AND_GOTO(ret, out);
 
-        /* Get snapshot info of individual snapshots */
-        for (i = 1; i <= snapcount; ++i) {
-                snprintf (key, sizeof (key), "snap%d", i);
+    /* <snapshots> */
+    ret = xmlTextWriterStartElement(writer, (xmlChar *)"snapshots");
+    XML_RET_CHECK_AND_GOTO(ret, out);
 
-                ret = cli_xml_snapshot_info_per_snap (writer, doc, dict,
-                                                      key, snap_driven);
-                if (ret) {
-                        gf_log ("cli", GF_LOG_ERROR, "Could not get %s ", key);
-                        goto out;
-                }
+    /* Get snapshot info of individual snapshots */
+    for (i = 1; i <= snapcount; ++i) {
+        snprintf(key, sizeof(key), "snap%d", i);
+
+        ret = cli_xml_snapshot_info_per_snap(writer, doc, dict, key,
+                                             snap_driven);
+        if (ret) {
+            gf_log("cli", GF_LOG_ERROR, "Could not get %s ", key);
+            goto out;
         }
+    }
 
-        /* </snapshots> */
-        ret = xmlTextWriterEndElement (writer);
-        XML_RET_CHECK_AND_GOTO (ret, out);
+    /* </snapshots> */
+    ret = xmlTextWriterEndElement(writer);
+    XML_RET_CHECK_AND_GOTO(ret, out);
 
-        /* </snapInfo> */
-        ret = xmlTextWriterEndElement (writer);
-        XML_RET_CHECK_AND_GOTO (ret, out);
+    /* </snapInfo> */
+    ret = xmlTextWriterEndElement(writer);
+    XML_RET_CHECK_AND_GOTO(ret, out);
 
-        ret = 0;
+    ret = 0;
 out:
 
-        return ret;
+    return ret;
 }
 
 /* This function will generate snapshot status of individual
@@ -4994,125 +4472,148 @@ out:
  * @return 0 on success and -1 on failure
  */
 static int
-cli_xml_snapshot_volume_status (xmlTextWriterPtr writer, xmlDocPtr doc,
-                                dict_t *dict, const char *keyprefix)
+cli_xml_snapshot_volume_status(xmlTextWriterPtr writer, xmlDocPtr doc,
+                               dict_t *dict, const char *keyprefix)
 {
-        int     ret             = -1;
-        int     brickcount      = 0;
-        int     i               = 0;
-        int     value           = 0;
-        int     pid             = 0;
-        char   *buffer          = NULL;
-        char    key[PATH_MAX]   = "";
-
-        GF_ASSERT (writer);
-        GF_ASSERT (doc);
-        GF_ASSERT (dict);
-        GF_ASSERT (keyprefix);
-
-        snprintf (key, sizeof (key), "%s.brickcount", keyprefix);
-
-        ret = dict_get_int32 (dict, key, &brickcount);
-        if (ret) {
-                gf_log ("cli", GF_LOG_ERROR, "Failed to fetch brickcount");
-                goto out;
-        }
-
-        ret = xmlTextWriterWriteFormatElement (writer, (xmlChar *) "brickCount",
-                                               "%d", brickcount);
-        XML_RET_CHECK_AND_GOTO (ret, out);
+    int ret = -1;
+    int brickcount = 0;
+    int i = 0;
+    int pid = 0;
+    char *buffer = NULL;
+    char key[PATH_MAX] = "";
+
+    GF_ASSERT(writer);
+    GF_ASSERT(doc);
+    GF_ASSERT(dict);
+    GF_ASSERT(keyprefix);
+
+    ret = snprintf(key, sizeof(key), "%s.brickcount", keyprefix);
+    if (ret < 0)
+        goto out;
+
+    ret = dict_get_int32(dict, key, &brickcount);
+    if (ret) {
+        gf_log("cli", GF_LOG_ERROR, "Failed to fetch brickcount");
+        goto out;
+    }
+
+    ret = xmlTextWriterWriteFormatElement(writer, (xmlChar *)"brickCount", "%d",
+                                          brickcount);
+    XML_RET_CHECK_AND_GOTO(ret, out);
+
+    /* Get status of every brick belonging to the snapshot volume */
+    for (i = 0; i < brickcount; i++) {
+        /* <snapInfo> */
+        ret = xmlTextWriterStartElement(writer, (xmlChar *)"brick");
+        XML_RET_CHECK_AND_GOTO(ret, out);
 
-        /* Get status of every brick belonging to the snapshot volume */
-        for (i = 0 ; i < brickcount ; i++) {
-                /* <snapInfo> */
-                ret = xmlTextWriterStartElement (writer, (xmlChar *)"brick");
-                XML_RET_CHECK_AND_GOTO (ret, out);
+        ret = snprintf(key, sizeof(key), "%s.brick%d.path", keyprefix, i);
+        if (ret < 0)
+            goto out;
 
-                snprintf (key, sizeof (key), "%s.brick%d.path", keyprefix, i);
+        ret = dict_get_str(dict, key, &buffer);
+        if (ret) {
+            gf_log("cli", GF_LOG_ERROR, "Unable to get Brick Path");
+            /*
+             * If path itself is not present, then end *
+             * this brick's status and continue to the *
+             * brick                                   *
+             */
+            ret = xmlTextWriterEndElement(writer);
+            XML_RET_CHECK_AND_GOTO(ret, out);
+            continue;
+        }
+
+        ret = xmlTextWriterWriteFormatElement(writer, (xmlChar *)"path", "%s",
+                                              buffer);
+        XML_RET_CHECK_AND_GOTO(ret, out);
 
-                ret = dict_get_str (dict, key, &buffer);
-                if (ret) {
-                        gf_log ("cli", GF_LOG_INFO, "Unable to get Brick Path");
-                        goto out;
-                }
+        ret = snprintf(key, sizeof(key), "%s.brick%d.vgname", keyprefix, i);
+        if (ret < 0)
+            goto out;
 
-                ret = xmlTextWriterWriteFormatElement (writer,
-                                        (xmlChar *) "path", "%s", buffer);
-                XML_RET_CHECK_AND_GOTO (ret, out);
+        ret = dict_get_str(dict, key, &buffer);
+        if (ret) {
+            gf_log("cli", GF_LOG_ERROR, "Unable to get Volume Group");
+            ret = xmlTextWriterWriteFormatElement(
+                writer, (xmlChar *)"volumeGroup", "N/A");
+        } else
+            ret = xmlTextWriterWriteFormatElement(
+                writer, (xmlChar *)"volumeGroup", "%s", buffer);
 
-                snprintf (key, sizeof (key), "%s.brick%d.vgname",
-                          keyprefix, i);
+        XML_RET_CHECK_AND_GOTO(ret, out);
 
-                ret = dict_get_str (dict, key, &buffer);
-                if (ret) {
-                        gf_log ("cli", GF_LOG_INFO,
-                                "Unable to get Volume Group");
-                        goto out;
-                }
+        ret = snprintf(key, sizeof(key), "%s.brick%d.status", keyprefix, i);
+        if (ret < 0)
+            goto out;
 
-                ret = xmlTextWriterWriteFormatElement (writer,
-                                        (xmlChar *) "volumeGroup",
-                                        "%s", buffer);
-                XML_RET_CHECK_AND_GOTO (ret, out);
+        ret = dict_get_str(dict, key, &buffer);
+        if (ret) {
+            gf_log("cli", GF_LOG_INFO, "Unable to get Brick Running");
+            ret = xmlTextWriterWriteFormatElement(
+                writer, (xmlChar *)"brick_running", "N/A");
+        } else
+            ret = xmlTextWriterWriteFormatElement(
+                writer, (xmlChar *)"brick_running", "%s", buffer);
 
-                snprintf (key, sizeof (key), "%s.brick%d.status", keyprefix, i);
+        XML_RET_CHECK_AND_GOTO(ret, out);
 
-                ret = dict_get_str (dict, key, &buffer);
-                if (ret) {
-                        gf_log ("cli", GF_LOG_ERROR,
-                                "Unable to get Brick Running");
-                        goto out;
-                }
+        ret = snprintf(key, sizeof(key), "%s.brick%d.pid", keyprefix, i);
+        if (ret < 0)
+            goto out;
 
-                snprintf (key, sizeof (key), "%s.brick%d.pid", keyprefix, i);
+        ret = dict_get_int32(dict, key, &pid);
+        if (ret) {
+            gf_log("cli", GF_LOG_INFO, "Unable to get pid");
+            ret = xmlTextWriterWriteFormatElement(writer, (xmlChar *)"pid",
+                                                  "N/A");
+        } else
+            ret = xmlTextWriterWriteFormatElement(writer, (xmlChar *)"pid",
+                                                  "%d", pid);
 
-                ret = dict_get_int32 (dict, key, &pid);
-                if (ret) {
-                        gf_log ("cli", GF_LOG_ERROR, "Unable to get pid");
-                        goto out;
-                }
+        XML_RET_CHECK_AND_GOTO(ret, out);
 
-                ret = xmlTextWriterWriteFormatElement (writer,
-                                        (xmlChar *) "pid", "%d", pid);
-                XML_RET_CHECK_AND_GOTO (ret, out);
+        ret = snprintf(key, sizeof(key), "%s.brick%d.data", keyprefix, i);
+        if (ret < 0)
+            goto out;
 
-                snprintf (key, sizeof (key), "%s.brick%d.data", keyprefix, i);
+        ret = dict_get_str(dict, key, &buffer);
+        if (ret) {
+            gf_log("cli", GF_LOG_ERROR, "Unable to get Data Percent");
+            ret = xmlTextWriterWriteFormatElement(
+                writer, (xmlChar *)"data_percentage", "N/A");
+        } else
+            ret = xmlTextWriterWriteFormatElement(
+                writer, (xmlChar *)"data_percentage", "%s", buffer);
 
-                ret = dict_get_str (dict, key, &buffer);
-                if (ret) {
-                        gf_log ("cli", GF_LOG_ERROR,
-                                        "Unable to get Data Percent");
-                        goto out;
-                }
+        XML_RET_CHECK_AND_GOTO(ret, out);
 
-                ret = xmlTextWriterWriteFormatElement (writer,
-                                                       (xmlChar *) "lvUsage",
-                                                       "%s", buffer);
-                XML_RET_CHECK_AND_GOTO (ret, out);
+        ret = snprintf(key, sizeof(key), "%s.brick%d.lvsize", keyprefix, i);
+        if (ret < 0)
+            goto out;
 
-                snprintf (key, sizeof (key), "%s.brick%d.lvsize",
-                          keyprefix, i);
-                ret = dict_get_str (dict, key, &buffer);
-                if (ret) {
-                        gf_log ("cli", GF_LOG_INFO, "Unable to get LV Size");
-                        goto out;
-                }
+        ret = dict_get_str(dict, key, &buffer);
+        if (ret) {
+            gf_log("cli", GF_LOG_ERROR, "Unable to get LV Size");
+            ret = xmlTextWriterWriteFormatElement(writer, (xmlChar *)"lvSize",
+                                                  "N/A");
+        } else {
+            /* Truncate any newline character */
+            buffer = strtok(buffer, "\n");
 
-                /* Truncate any newline character */
-                buffer = strtok (buffer, "\n");
+            ret = xmlTextWriterWriteFormatElement(writer, (xmlChar *)"lvSize",
+                                                  "%s", buffer);
+        }
 
-                ret = xmlTextWriterWriteFormatElement (writer,
-                                                       (xmlChar *) "lvSize",
-                                                       "%s", buffer);
-                XML_RET_CHECK_AND_GOTO (ret, out);
+        XML_RET_CHECK_AND_GOTO(ret, out);
 
-                /* </brick> */
-                ret = xmlTextWriterEndElement (writer);
-                XML_RET_CHECK_AND_GOTO (ret, out);
-        }
+        /* </brick> */
+        ret = xmlTextWriterEndElement(writer);
+        XML_RET_CHECK_AND_GOTO(ret, out);
+    }
 
 out:
-        return ret;
+    return ret;
 }
 
 /* This function will generate snapshot status of individual
@@ -5125,88 +4626,85 @@ out:
  * @return 0 on success and -1 on failure
  */
 static int
-cli_xml_snapshot_status_per_snap (xmlTextWriterPtr writer, xmlDocPtr doc,
-                                  dict_t *dict, const char *keyprefix)
+cli_xml_snapshot_status_per_snap(xmlTextWriterPtr writer, xmlDocPtr doc,
+                                 dict_t *dict, const char *keyprefix)
 {
-        int     ret             = -1;
-        int     snapcount       = 0;
-        int     volcount        = 0;
-        int     i               = 0;
-        char   *buffer          = NULL;
-        char    key [PATH_MAX]  = "";
-
-        GF_ASSERT (writer);
-        GF_ASSERT (doc);
-        GF_ASSERT (dict);
-        GF_ASSERT (keyprefix);
-
-        ret = xmlTextWriterStartElement (writer, (xmlChar *)"snapshot");
-        XML_RET_CHECK_AND_GOTO (ret, out);
-
-        snprintf (key, sizeof (key), "%s.snapname", keyprefix);
-
-        ret = dict_get_str (dict, key, &buffer);
-        if (ret) {
-                gf_log ("cli", GF_LOG_ERROR, "Unable to get snapname");
-                goto out;
-        }
-
-        ret = xmlTextWriterWriteFormatElement (writer, (xmlChar *) "name",
-                                               "%s", buffer);
-        XML_RET_CHECK_AND_GOTO (ret, out);
-
-        snprintf (key, sizeof (key), "%s.uuid", keyprefix);
-
-        ret = dict_get_str (dict, key, &buffer);
-        if (ret) {
-                gf_log ("cli", GF_LOG_ERROR, "Unable to get snap UUID");
-                goto out;
-        }
-
-        ret = xmlTextWriterWriteFormatElement (writer, (xmlChar *) "uuid",
-                                               "%s", buffer);
-        XML_RET_CHECK_AND_GOTO (ret, out);
+    int ret = -1;
+    int volcount = 0;
+    int i = 0;
+    char *buffer = NULL;
+    char key[PATH_MAX] = "";
+
+    GF_ASSERT(writer);
+    GF_ASSERT(doc);
+    GF_ASSERT(dict);
+    GF_ASSERT(keyprefix);
+
+    ret = xmlTextWriterStartElement(writer, (xmlChar *)"snapshot");
+    XML_RET_CHECK_AND_GOTO(ret, out);
+
+    snprintf(key, sizeof(key), "%s.snapname", keyprefix);
+
+    ret = dict_get_str(dict, key, &buffer);
+    if (ret) {
+        gf_log("cli", GF_LOG_ERROR, "Unable to get snapname");
+        goto out;
+    }
+
+    ret = xmlTextWriterWriteFormatElement(writer, (xmlChar *)"name", "%s",
+                                          buffer);
+    XML_RET_CHECK_AND_GOTO(ret, out);
+
+    snprintf(key, sizeof(key), "%s.uuid", keyprefix);
+
+    ret = dict_get_str(dict, key, &buffer);
+    if (ret) {
+        gf_log("cli", GF_LOG_ERROR, "Unable to get snap UUID");
+        goto out;
+    }
+
+    ret = xmlTextWriterWriteFormatElement(writer, (xmlChar *)"uuid", "%s",
+                                          buffer);
+    XML_RET_CHECK_AND_GOTO(ret, out);
+
+    snprintf(key, sizeof(key), "%s.volcount", keyprefix);
+
+    ret = dict_get_int32(dict, key, &volcount);
+    if (ret) {
+        gf_log("cli", GF_LOG_ERROR, "Unable to get volume count");
+        goto out;
+    }
+
+    ret = xmlTextWriterWriteFormatElement(writer, (xmlChar *)"volCount", "%d",
+                                          volcount);
+    XML_RET_CHECK_AND_GOTO(ret, out);
+
+    /* Get snapshot status of individual snapshot volume */
+    for (i = 0; i < volcount; i++) {
+        /* <volume> */
+        ret = xmlTextWriterStartElement(writer, (xmlChar *)"volume");
+        XML_RET_CHECK_AND_GOTO(ret, out);
 
-        snprintf (key, sizeof (key), "%s.volcount", keyprefix);
+        snprintf(key, sizeof(key), "%s.vol%d", keyprefix, i);
 
-        ret = dict_get_int32 (dict, key, &volcount);
+        ret = cli_xml_snapshot_volume_status(writer, doc, dict, key);
         if (ret) {
-                gf_log ("cli", GF_LOG_ERROR, "Unable to get volume count");
-                goto out;
+            gf_log("cli", GF_LOG_ERROR, "Could not get snap volume status");
+            goto out;
         }
 
-        ret = xmlTextWriterWriteFormatElement (writer, (xmlChar *) "volCount",
-                                               "%d", volcount);
-        XML_RET_CHECK_AND_GOTO (ret, out);
-
-        /* Get snapshot status of individual snapshot volume */
-        for (i = 0 ; i < volcount ; i++) {
-                /* <volume> */
-                ret = xmlTextWriterStartElement (writer, (xmlChar *)"volume");
-                XML_RET_CHECK_AND_GOTO (ret, out);
-
-                snprintf (key, sizeof (key), "%s.vol%d", keyprefix, i);
-
-                ret = cli_xml_snapshot_volume_status (writer, doc,
-                                                             dict, key);
-                if (ret) {
-                        gf_log ("cli", GF_LOG_ERROR,
-                                        "Could not get snap volume status");
-                        goto out;
-                }
-
-                /* </volume> */
-                ret = xmlTextWriterEndElement (writer);
-                XML_RET_CHECK_AND_GOTO (ret, out);
-        }
+        /* </volume> */
+        ret = xmlTextWriterEndElement(writer);
+        XML_RET_CHECK_AND_GOTO(ret, out);
+    }
 
-        /* </snapshot> */
-        ret = xmlTextWriterEndElement (writer);
-        XML_RET_CHECK_AND_GOTO (ret, out);
+    /* </snapshot> */
+    ret = xmlTextWriterEndElement(writer);
+    XML_RET_CHECK_AND_GOTO(ret, out);
 
-        ret = 0;
+    ret = 0;
 out:
-        return ret;
+    return ret;
 }
 
 /* This function will generate snapshot status output in xml format.
@@ -5218,64 +4716,63 @@ out:
  * @return 0 on success and -1 on failure
  */
 static int
-cli_xml_snapshot_status (xmlTextWriterPtr writer, xmlDocPtr doc, dict_t *dict)
+cli_xml_snapshot_status(xmlTextWriterPtr writer, xmlDocPtr doc, dict_t *dict)
 {
-        int     ret             = -1;
-        int     snapcount       = 0;
-        int     i               = 0;
-        int     status_cmd      = 0;
-        char   *str_value       = NULL;
-        char    key [PATH_MAX]  = "";
-
-        GF_ASSERT (writer);
-        GF_ASSERT (doc);
-        GF_ASSERT (dict);
-
-        /* <snapStatus> */
-        ret = xmlTextWriterStartElement (writer, (xmlChar *)"snapStatus");
-        XML_RET_CHECK_AND_GOTO (ret, out);
-
-        ret = dict_get_int32 (dict, "sub-cmd", &status_cmd);
+    int ret = -1;
+    int snapcount = 0;
+    int i = 0;
+    int status_cmd = 0;
+    char key[PATH_MAX] = "";
+
+    GF_ASSERT(writer);
+    GF_ASSERT(doc);
+    GF_ASSERT(dict);
+
+    /* <snapStatus> */
+    ret = xmlTextWriterStartElement(writer, (xmlChar *)"snapStatus");
+    XML_RET_CHECK_AND_GOTO(ret, out);
+
+    ret = dict_get_int32(dict, "sub-cmd", &status_cmd);
+    if (ret) {
+        gf_log("cli", GF_LOG_ERROR, "Could not fetch status type");
+        goto out;
+    }
+
+    if ((GF_SNAP_STATUS_TYPE_SNAP == status_cmd) ||
+        (GF_SNAP_STATUS_TYPE_ITER == status_cmd)) {
+        snapcount = 1;
+    } else {
+        ret = dict_get_int32(dict, "status.snapcount", &snapcount);
         if (ret) {
-                gf_log ("cli", GF_LOG_ERROR, "Could not fetch status type");
-                goto out;
+            gf_log("cli", GF_LOG_ERROR, "Could not get snapcount");
+            goto out;
         }
 
-        if (GF_SNAP_STATUS_TYPE_SNAP == status_cmd) {
-                snapcount = 1;
-        } else {
-                ret = dict_get_int32 (dict, "status.snapcount", &snapcount);
-                if (ret) {
-                        gf_log ("cli", GF_LOG_ERROR, "Could not get snapcount");
-                        goto out;
-                }
-
-                ret = xmlTextWriterWriteFormatElement (writer,
-                                                       (xmlChar *) "count",
-                                                       "%d", snapcount);
-                XML_RET_CHECK_AND_GOTO (ret, out);
-        }
+        ret = xmlTextWriterWriteFormatElement(writer, (xmlChar *)"count", "%d",
+                                              snapcount);
+        XML_RET_CHECK_AND_GOTO(ret, out);
+    }
 
-        for (i = 0 ; i < snapcount; i++) {
-                snprintf (key, sizeof (key), "status.snap%d", i);
+    for (i = 0; i < snapcount; i++) {
+        snprintf(key, sizeof(key), "status.snap%d", i);
 
-                ret = cli_xml_snapshot_status_per_snap (writer, doc,
-                                                               dict, key);
-                if (ret < 0) {
-                        gf_log ("cli", GF_LOG_ERROR, "failed to create xml "
-                                "output for snapshot status");
-                        goto out;
-                }
+        ret = cli_xml_snapshot_status_per_snap(writer, doc, dict, key);
+        if (ret < 0) {
+            gf_log("cli", GF_LOG_ERROR,
+                   "failed to create xml "
+                   "output for snapshot status");
+            goto out;
         }
+    }
 
-        /* </snapStatus> */
-        ret = xmlTextWriterEndElement (writer);
-        XML_RET_CHECK_AND_GOTO (ret, out);
+    /* </snapStatus> */
+    ret = xmlTextWriterEndElement(writer);
+    XML_RET_CHECK_AND_GOTO(ret, out);
 
-        ret = 0;
+    ret = 0;
 out:
 
-        return ret;
+    return ret;
 }
 
 /* This function will generate snapshot config show output in xml format.
@@ -5287,153 +4784,149 @@ out:
  * @return 0 on success and -1 on failure
  */
 static int
-cli_xml_snapshot_config_show (xmlTextWriterPtr writer,
-                              xmlDocPtr doc, dict_t *dict)
+cli_xml_snapshot_config_show(xmlTextWriterPtr writer, xmlDocPtr doc,
+                             dict_t *dict)
 {
-        int             ret             = -1;
-        uint64_t        i               = 0;
-        uint64_t        value           = 0;
-        uint64_t        volcount        = 0;
-        char            buf[PATH_MAX]   = "";
-        char           *str_value       = NULL;
-
-        GF_ASSERT (writer);
-        GF_ASSERT (doc);
-        GF_ASSERT (dict);
-
-        /* <systemConfig> */
-        ret = xmlTextWriterStartElement (writer,
-                        (xmlChar *)"systemConfig");
-        XML_RET_CHECK_AND_GOTO (ret, out);
-
-        ret = dict_get_uint64 (dict, "snap-max-hard-limit", &value);
-        if (ret) {
-                gf_log ("cli", GF_LOG_ERROR, "Failed to get "
-                        "snap-max-hard-limit");
-                goto out;
-        }
-
-        ret = xmlTextWriterWriteFormatElement (writer,
-                        (xmlChar *) "hardLimit", "%"PRIu64, value);
-        XML_RET_CHECK_AND_GOTO (ret, out);
+    int ret = -1;
+    uint64_t i = 0;
+    uint64_t value = 0;
+    uint64_t volcount = 0;
+    char buf[PATH_MAX] = "";
+    char *str_value = NULL;
+
+    GF_ASSERT(writer);
+    GF_ASSERT(doc);
+    GF_ASSERT(dict);
+
+    /* <systemConfig> */
+    ret = xmlTextWriterStartElement(writer, (xmlChar *)"systemConfig");
+    XML_RET_CHECK_AND_GOTO(ret, out);
+
+    ret = dict_get_uint64(dict, "snap-max-hard-limit", &value);
+    if (ret) {
+        gf_log("cli", GF_LOG_ERROR,
+               "Failed to get "
+               "snap-max-hard-limit");
+        goto out;
+    }
+
+    ret = xmlTextWriterWriteFormatElement(writer, (xmlChar *)"hardLimit",
+                                          "%" PRIu64, value);
+    XML_RET_CHECK_AND_GOTO(ret, out);
+
+    ret = dict_get_uint64(dict, "snap-max-soft-limit", &value);
+    if (ret) {
+        gf_log("cli", GF_LOG_ERROR,
+               "Failed to get "
+               "snap-max-soft-limit");
+        goto out;
+    }
+
+    ret = xmlTextWriterWriteFormatElement(writer, (xmlChar *)"softLimit",
+                                          "%" PRIu64 "%%", value);
+    XML_RET_CHECK_AND_GOTO(ret, out);
+
+    ret = dict_get_str(dict, "auto-delete", &str_value);
+    if (ret) {
+        gf_log("cli", GF_LOG_ERROR, "Could not fetch auto-delete");
+        goto out;
+    }
+
+    ret = xmlTextWriterWriteFormatElement(writer, (xmlChar *)"autoDelete", "%s",
+                                          str_value);
+    XML_RET_CHECK_AND_GOTO(ret, out);
+
+    ret = dict_get_str(dict, "snap-activate-on-create", &str_value);
+    if (ret) {
+        gf_log("cli", GF_LOG_ERROR,
+               "Could not fetch snap-activate-on-create-delete");
+        goto out;
+    }
+
+    ret = xmlTextWriterWriteFormatElement(writer, (xmlChar *)"activateOnCreate",
+                                          "%s", str_value);
+    XML_RET_CHECK_AND_GOTO(ret, out);
+
+    /* </systemConfig> */
+    ret = xmlTextWriterEndElement(writer);
+    XML_RET_CHECK_AND_GOTO(ret, out);
+
+    /* <volumeConfig> */
+    ret = xmlTextWriterStartElement(writer, (xmlChar *)"volumeConfig");
+    XML_RET_CHECK_AND_GOTO(ret, out);
+
+    ret = dict_get_uint64(dict, "voldisplaycount", &volcount);
+    if (ret) {
+        gf_log("cli", GF_LOG_ERROR, "Could not fetch volcount");
+        goto out;
+    }
+
+    /* Get config of all the volumes */
+    for (i = 0; i < volcount; i++) {
+        /* <volume> */
+        ret = xmlTextWriterStartElement(writer, (xmlChar *)"volume");
+        XML_RET_CHECK_AND_GOTO(ret, out);
 
-        ret = dict_get_uint64 (dict, "snap-max-soft-limit", &value);
+        snprintf(buf, sizeof(buf), "volume%" PRIu64 "-volname", i);
+        ret = dict_get_str(dict, buf, &str_value);
         if (ret) {
-                gf_log ("cli", GF_LOG_ERROR, "Failed to get "
-                        "snap-max-soft-limit");
-                goto out;
+            gf_log("cli", GF_LOG_ERROR, "Could not fetch %s", buf);
+            goto out;
         }
 
-        ret = xmlTextWriterWriteFormatElement (writer,
-                        (xmlChar *) "softLimit",
-                        "%"PRIu64"%%", value);
-        XML_RET_CHECK_AND_GOTO (ret, out);
+        ret = xmlTextWriterWriteFormatElement(writer, (xmlChar *)"name", "%s",
+                                              str_value);
+        XML_RET_CHECK_AND_GOTO(ret, out);
 
-        ret = dict_get_str (dict, "auto-delete", &str_value);
+        snprintf(buf, sizeof(buf), "volume%" PRIu64 "-snap-max-hard-limit", i);
+        ret = dict_get_uint64(dict, buf, &value);
         if (ret) {
-                gf_log ("cli", GF_LOG_ERROR, "Could not fetch auto-delete");
-                goto out;
+            gf_log("cli", GF_LOG_ERROR, "Could not fetch %s", buf);
+            goto out;
         }
 
-        ret = xmlTextWriterWriteFormatElement (writer,
-                        (xmlChar *) "autoDelete", "%s", str_value);
-        XML_RET_CHECK_AND_GOTO (ret, out);
+        ret = xmlTextWriterWriteFormatElement(writer, (xmlChar *)"hardLimit",
+                                              "%" PRIu64, value);
+        XML_RET_CHECK_AND_GOTO(ret, out);
 
-        ret = dict_get_str (dict, "snap-activate-on-create", &str_value);
+        snprintf(buf, sizeof(buf), "volume%" PRIu64 "-active-hard-limit", i);
+        ret = dict_get_uint64(dict, buf, &value);
         if (ret) {
-                gf_log ("cli", GF_LOG_ERROR,
-                        "Could not fetch snap-activate-on-create-delete");
-                goto out;
+            gf_log("cli", GF_LOG_ERROR,
+                   "Could not fetch"
+                   " effective snap_max_hard_limit for "
+                   "%s",
+                   str_value);
+            goto out;
         }
 
-        ret = xmlTextWriterWriteFormatElement (writer,
-                        (xmlChar *) "activateOnCreate", "%s", str_value);
-        XML_RET_CHECK_AND_GOTO (ret, out);
-
-        /* </systemConfig> */
-        ret = xmlTextWriterEndElement (writer);
-        XML_RET_CHECK_AND_GOTO (ret, out);
-
-        /* <volumeConfig> */
-        ret = xmlTextWriterStartElement (writer, (xmlChar *)"volumeConfig");
-        XML_RET_CHECK_AND_GOTO (ret, out);
+        ret = xmlTextWriterWriteFormatElement(
+            writer, (xmlChar *)"effectiveHardLimit", "%" PRIu64, value);
+        XML_RET_CHECK_AND_GOTO(ret, out);
 
-        ret = dict_get_uint64 (dict, "voldisplaycount", &volcount);
+        snprintf(buf, sizeof(buf), "volume%" PRIu64 "-snap-max-soft-limit", i);
+        ret = dict_get_uint64(dict, buf, &value);
         if (ret) {
-                gf_log ("cli", GF_LOG_ERROR, "Could not fetch volcount");
-                goto out;
+            gf_log("cli", GF_LOG_ERROR, "Could not fetch %s", buf);
+            goto out;
         }
 
-        /* Get config of all the volumes */
-        for (i = 0; i < volcount; i++) {
-                /* <volume> */
-                ret = xmlTextWriterStartElement (writer, (xmlChar *)"volume");
-                XML_RET_CHECK_AND_GOTO (ret, out);
-
-                snprintf (buf, sizeof(buf), "volume%"PRIu64"-volname", i);
-                ret = dict_get_str (dict, buf, &str_value);
-                if (ret) {
-                        gf_log ("cli", GF_LOG_ERROR, "Could not fetch %s", buf);
-                        goto out;
-                }
-
-                ret = xmlTextWriterWriteFormatElement (writer,
-                                (xmlChar *) "name", "%s", str_value);
-                XML_RET_CHECK_AND_GOTO (ret, out);
-
-
-                snprintf (buf, sizeof(buf),
-                                "volume%"PRIu64"-snap-max-hard-limit", i);
-                ret = dict_get_uint64 (dict, buf, &value);
-                if (ret) {
-                        gf_log ("cli", GF_LOG_ERROR, "Could not fetch %s", buf);
-                        goto out;
-                }
-
-                ret = xmlTextWriterWriteFormatElement (writer,
-                                (xmlChar *) "hardLimit", "%"PRIu64, value);
-                XML_RET_CHECK_AND_GOTO (ret, out);
-
-                snprintf (buf, sizeof(buf),
-                                "volume%"PRIu64"-active-hard-limit", i);
-                ret = dict_get_uint64 (dict, buf, &value);
-                if (ret) {
-                        gf_log ("cli", GF_LOG_ERROR, "Could not fetch"
-                                " effective snap_max_hard_limit for "
-                                "%s", str_value);
-                        goto out;
-                }
-
-                ret = xmlTextWriterWriteFormatElement (writer,
-                                (xmlChar *) "effectiveHardLimit",
-                                "%"PRIu64, value);
-                XML_RET_CHECK_AND_GOTO (ret, out);
-
-                snprintf (buf, sizeof(buf),
-                                "volume%"PRIu64"-snap-max-soft-limit", i);
-                ret = dict_get_uint64 (dict, buf, &value);
-                if (ret) {
-                        gf_log ("cli", GF_LOG_ERROR, "Could not fetch %s", buf);
-                        goto out;
-                }
-
-                ret = xmlTextWriterWriteFormatElement (writer,
-                                        (xmlChar *) "softLimit",
-                                        "%"PRIu64, value);
-                XML_RET_CHECK_AND_GOTO (ret, out);
-
-                /* </volume> */
-                ret = xmlTextWriterEndElement (writer);
-                XML_RET_CHECK_AND_GOTO (ret, out);
-        }
+        ret = xmlTextWriterWriteFormatElement(writer, (xmlChar *)"softLimit",
+                                              "%" PRIu64, value);
+        XML_RET_CHECK_AND_GOTO(ret, out);
 
         /* </volume> */
-        ret = xmlTextWriterEndElement (writer);
-        XML_RET_CHECK_AND_GOTO (ret, out);
+        ret = xmlTextWriterEndElement(writer);
+        XML_RET_CHECK_AND_GOTO(ret, out);
+    }
 
-        ret = 0;
+    /* </volume> */
+    ret = xmlTextWriterEndElement(writer);
+    XML_RET_CHECK_AND_GOTO(ret, out);
+
+    ret = 0;
 out:
-        return ret;
+    return ret;
 }
 
 /* This function will generate snapshot config set output in xml format.
@@ -5445,87 +4938,84 @@ out:
  * @return 0 on success and -1 on failure
  */
 static int
-cli_xml_snapshot_config_set (xmlTextWriterPtr writer, xmlDocPtr doc,
-                             dict_t *dict)
+cli_xml_snapshot_config_set(xmlTextWriterPtr writer, xmlDocPtr doc,
+                            dict_t *dict)
 {
-        int             ret             = -1;
-        uint64_t        hard_limit      = 0;
-        uint64_t        soft_limit      = 0;
-        char           *volname         = NULL;
-        char           *auto_delete     = NULL;
-        char           *snap_activate   = NULL;
-
-        GF_ASSERT (writer);
-        GF_ASSERT (doc);
-        GF_ASSERT (dict);
-
-        /* This is optional parameter therefore ignore the error */
-        ret = dict_get_uint64 (dict, "snap-max-hard-limit", &hard_limit);
-        /* This is optional parameter therefore ignore the error */
-        ret = dict_get_uint64 (dict, "snap-max-soft-limit", &soft_limit);
-        ret = dict_get_str (dict, "auto-delete", &auto_delete);
-        ret = dict_get_str (dict, "snap-activate-on-create", &snap_activate);
-
-        if (!hard_limit && !soft_limit && !auto_delete && !snap_activate) {
-                ret = -1;
-                gf_log ("cli", GF_LOG_ERROR, "At least one option from "
-                        "snap-max-hard-limit, snap-max-soft-limit, auto-delete"
-                        " and snap-activate-on-create should be set");
-                goto out;
-        }
-
-        /* Ignore the error, as volname is optional */
-        ret = dict_get_str (dict, "volname", &volname);
-
-        if (NULL == volname) {
-                /* <systemConfig> */
-                ret = xmlTextWriterStartElement (writer,
-                                                 (xmlChar *)"systemConfig");
-        } else {
-                /* <volumeConfig> */
-                ret = xmlTextWriterStartElement (writer,
-                                                 (xmlChar *)"volumeConfig");
-                XML_RET_CHECK_AND_GOTO (ret, out);
+    int ret = -1;
+    uint64_t hard_limit = 0;
+    uint64_t soft_limit = 0;
+    char *volname = NULL;
+    char *auto_delete = NULL;
+    char *snap_activate = NULL;
+
+    GF_ASSERT(writer);
+    GF_ASSERT(doc);
+    GF_ASSERT(dict);
+
+    /* This is optional parameter therefore ignore the error */
+    ret = dict_get_uint64(dict, "snap-max-hard-limit", &hard_limit);
+    /* This is optional parameter therefore ignore the error */
+    ret = dict_get_uint64(dict, "snap-max-soft-limit", &soft_limit);
+    ret = dict_get_str(dict, "auto-delete", &auto_delete);
+    ret = dict_get_str(dict, "snap-activate-on-create", &snap_activate);
+
+    if (!hard_limit && !soft_limit && !auto_delete && !snap_activate) {
+        ret = -1;
+        gf_log("cli", GF_LOG_ERROR,
+               "At least one option from "
+               "snap-max-hard-limit, snap-max-soft-limit, auto-delete"
+               " and snap-activate-on-create should be set");
+        goto out;
+    }
+
+    /* Ignore the error, as volname is optional */
+    ret = dict_get_str(dict, "volname", &volname);
+
+    if (NULL == volname) {
+        /* <systemConfig> */
+        ret = xmlTextWriterStartElement(writer, (xmlChar *)"systemConfig");
+    } else {
+        /* <volumeConfig> */
+        ret = xmlTextWriterStartElement(writer, (xmlChar *)"volumeConfig");
+        XML_RET_CHECK_AND_GOTO(ret, out);
 
-                ret = xmlTextWriterWriteFormatElement (writer,
-                                (xmlChar *) "name", "%s", volname);
-        }
+        ret = xmlTextWriterWriteFormatElement(writer, (xmlChar *)"name", "%s",
+                                              volname);
+    }
 
-        XML_RET_CHECK_AND_GOTO (ret, out);
+    XML_RET_CHECK_AND_GOTO(ret, out);
 
-        if (hard_limit) {
-                ret = xmlTextWriterWriteFormatElement (writer,
-                                                (xmlChar *) "newHardLimit",
-                                                "%"PRIu64, hard_limit);
-                XML_RET_CHECK_AND_GOTO (ret, out);
-        }
+    if (hard_limit) {
+        ret = xmlTextWriterWriteFormatElement(writer, (xmlChar *)"newHardLimit",
+                                              "%" PRIu64, hard_limit);
+        XML_RET_CHECK_AND_GOTO(ret, out);
+    }
 
-        if (soft_limit) {
-                ret = xmlTextWriterWriteFormatElement (writer,
-                                (xmlChar *) "newSoftLimit",
-                                "%"PRIu64, soft_limit);
-                XML_RET_CHECK_AND_GOTO (ret, out);
-        }
+    if (soft_limit) {
+        ret = xmlTextWriterWriteFormatElement(writer, (xmlChar *)"newSoftLimit",
+                                              "%" PRIu64, soft_limit);
+        XML_RET_CHECK_AND_GOTO(ret, out);
+    }
 
-        if (auto_delete) {
-                ret = xmlTextWriterWriteFormatElement (writer,
-                                (xmlChar *) "autoDelete", "%s", auto_delete);
-                XML_RET_CHECK_AND_GOTO (ret, out);
-        }
+    if (auto_delete) {
+        ret = xmlTextWriterWriteFormatElement(writer, (xmlChar *)"autoDelete",
+                                              "%s", auto_delete);
+        XML_RET_CHECK_AND_GOTO(ret, out);
+    }
 
-        if (snap_activate) {
-                ret = xmlTextWriterWriteFormatElement (writer,
-                           (xmlChar *) "activateOnCreate", "%s", snap_activate);
-                XML_RET_CHECK_AND_GOTO (ret, out);
-        }
+    if (snap_activate) {
+        ret = xmlTextWriterWriteFormatElement(
+            writer, (xmlChar *)"activateOnCreate", "%s", snap_activate);
+        XML_RET_CHECK_AND_GOTO(ret, out);
+    }
 
-        /* </volumeConfig> or </systemConfig> */
-        ret = xmlTextWriterEndElement (writer);
-        XML_RET_CHECK_AND_GOTO (ret, out);
+    /* </volumeConfig> or </systemConfig> */
+    ret = xmlTextWriterEndElement(writer);
+    XML_RET_CHECK_AND_GOTO(ret, out);
 
-        ret = 0;
+    ret = 0;
 out:
-        return ret;
+    return ret;
 }
 
 /* This function will generate snapshot config output in xml format.
@@ -5537,58 +5027,60 @@ out:
  * @return 0 on success and -1 on failure
  */
 static int
-cli_xml_snapshot_config (xmlTextWriterPtr writer, xmlDocPtr doc, dict_t *dict)
+cli_xml_snapshot_config(xmlTextWriterPtr writer, xmlDocPtr doc, dict_t *dict)
 {
-        int             ret             = -1;
-        int             config_command  = 0;
+    int ret = -1;
+    int config_command = 0;
 
-        GF_ASSERT (writer);
-        GF_ASSERT (doc);
-        GF_ASSERT (dict);
+    GF_ASSERT(writer);
+    GF_ASSERT(doc);
+    GF_ASSERT(dict);
 
-        /* <snapConfig> */
-        ret = xmlTextWriterStartElement (writer, (xmlChar *)"snapConfig");
-        XML_RET_CHECK_AND_GOTO (ret, out);
+    /* <snapConfig> */
+    ret = xmlTextWriterStartElement(writer, (xmlChar *)"snapConfig");
+    XML_RET_CHECK_AND_GOTO(ret, out);
 
-        ret = dict_get_int32 (dict, "config-command", &config_command);
-        if (ret) {
-                gf_log ("cli", GF_LOG_ERROR, "Could not fetch config type");
-                goto out;
-        }
+    ret = dict_get_int32(dict, "config-command", &config_command);
+    if (ret) {
+        gf_log("cli", GF_LOG_ERROR, "Could not fetch config type");
+        goto out;
+    }
 
-        switch (config_command) {
+    switch (config_command) {
         case GF_SNAP_CONFIG_TYPE_SET:
-                ret = cli_xml_snapshot_config_set (writer, doc, dict);
-                if (ret) {
-                        gf_log ("cli", GF_LOG_ERROR, "Failed to create xml "
-                                "output for snapshot config set command");
-                        goto out;
-                }
+            ret = cli_xml_snapshot_config_set(writer, doc, dict);
+            if (ret) {
+                gf_log("cli", GF_LOG_ERROR,
+                       "Failed to create xml "
+                       "output for snapshot config set command");
+                goto out;
+            }
 
-                break;
+            break;
         case GF_SNAP_CONFIG_DISPLAY:
-                ret = cli_xml_snapshot_config_show (writer, doc, dict);
-                if (ret) {
-                        gf_log ("cli", GF_LOG_ERROR, "Failed to create xml "
-                                "output for snapshot config show command");
-                        goto out;
-                }
-                break;
+            ret = cli_xml_snapshot_config_show(writer, doc, dict);
+            if (ret) {
+                gf_log("cli", GF_LOG_ERROR,
+                       "Failed to create xml "
+                       "output for snapshot config show command");
+                goto out;
+            }
+            break;
         default:
-                gf_log ("cli", GF_LOG_ERROR, "Uknown config command :%d",
-                        config_command);
-                ret = -1;
-                goto out;
-        }
+            gf_log("cli", GF_LOG_ERROR, "Unknown config command :%d",
+                   config_command);
+            ret = -1;
+            goto out;
+    }
 
-        /* </snapConfig> */
-        ret = xmlTextWriterEndElement (writer);
-        XML_RET_CHECK_AND_GOTO (ret, out);
+    /* </snapConfig> */
+    ret = xmlTextWriterEndElement(writer);
+    XML_RET_CHECK_AND_GOTO(ret, out);
 
-        ret = 0;
+    ret = 0;
 out:
 
-        return ret;
+    return ret;
 }
 
 /* This function will generate snapshot activate or
@@ -5601,67 +5093,66 @@ out:
  * @return 0 on success and -1 on failure
  */
 static int
-cli_xml_snapshot_activate_deactivate (xmlTextWriterPtr writer, xmlDocPtr doc,
-                                      dict_t *dict, int cmd)
+cli_xml_snapshot_activate_deactivate(xmlTextWriterPtr writer, xmlDocPtr doc,
+                                     dict_t *dict, int cmd)
 {
-        int     ret     = -1;
-        char   *buffer  = NULL;
-        char   *tag     = NULL;
-
-        GF_ASSERT (writer);
-        GF_ASSERT (doc);
-        GF_ASSERT (dict);
-
-        if (GF_SNAP_OPTION_TYPE_ACTIVATE == cmd) {
-                tag = "snapActivate";
-        } else if (GF_SNAP_OPTION_TYPE_DEACTIVATE == cmd) {
-                tag = "snapDeactivate";
-        } else {
-                gf_log ("cli", GF_LOG_ERROR, "invalid command %d", cmd);
-                goto out;
-        }
-
-        /* <snapActivate> or <snapDeactivate> */
-        ret = xmlTextWriterStartElement (writer, (xmlChar *)tag);
-        XML_RET_CHECK_AND_GOTO (ret, out);
-
-        /* <snapshot> */
-        ret = xmlTextWriterStartElement (writer, (xmlChar *)"snapshot");
-        XML_RET_CHECK_AND_GOTO (ret, out);
-
-        ret = dict_get_str (dict, "snapname", &buffer);
-        if (ret) {
-                gf_log ("cli", GF_LOG_ERROR, "Failed to get snap name");
-                goto out;
-        }
-
-        ret = xmlTextWriterWriteFormatElement (writer, (xmlChar *) "name",
-                                               "%s", buffer);
-        XML_RET_CHECK_AND_GOTO (ret, out);
-
-        ret = dict_get_str (dict, "snapuuid", &buffer);
-        if (ret) {
-                gf_log ("cli", GF_LOG_ERROR, "Failed to get snap uuid");
-                goto out;
-        }
-
-        ret = xmlTextWriterWriteFormatElement (writer, (xmlChar *) "uuid",
-                                               "%s", buffer);
-        XML_RET_CHECK_AND_GOTO (ret, out);
-
-        /* </snapshot> */
-        ret = xmlTextWriterEndElement (writer);
-        XML_RET_CHECK_AND_GOTO (ret, out);
-
-
-        /* </snapActivate> or </snapDeactivate> */
-        ret = xmlTextWriterEndElement (writer);
-        XML_RET_CHECK_AND_GOTO (ret, out);
-
-        ret = 0;
+    int ret = -1;
+    char *buffer = NULL;
+    char *tag = NULL;
+
+    GF_ASSERT(writer);
+    GF_ASSERT(doc);
+    GF_ASSERT(dict);
+
+    if (GF_SNAP_OPTION_TYPE_ACTIVATE == cmd) {
+        tag = "snapActivate";
+    } else if (GF_SNAP_OPTION_TYPE_DEACTIVATE == cmd) {
+        tag = "snapDeactivate";
+    } else {
+        gf_log("cli", GF_LOG_ERROR, "invalid command %d", cmd);
+        goto out;
+    }
+
+    /* <snapActivate> or <snapDeactivate> */
+    ret = xmlTextWriterStartElement(writer, (xmlChar *)tag);
+    XML_RET_CHECK_AND_GOTO(ret, out);
+
+    /* <snapshot> */
+    ret = xmlTextWriterStartElement(writer, (xmlChar *)"snapshot");
+    XML_RET_CHECK_AND_GOTO(ret, out);
+
+    ret = dict_get_str(dict, "snapname", &buffer);
+    if (ret) {
+        gf_log("cli", GF_LOG_ERROR, "Failed to get snap name");
+        goto out;
+    }
+
+    ret = xmlTextWriterWriteFormatElement(writer, (xmlChar *)"name", "%s",
+                                          buffer);
+    XML_RET_CHECK_AND_GOTO(ret, out);
+
+    ret = dict_get_str(dict, "snapuuid", &buffer);
+    if (ret) {
+        gf_log("cli", GF_LOG_ERROR, "Failed to get snap uuid");
+        goto out;
+    }
+
+    ret = xmlTextWriterWriteFormatElement(writer, (xmlChar *)"uuid", "%s",
+                                          buffer);
+    XML_RET_CHECK_AND_GOTO(ret, out);
+
+    /* </snapshot> */
+    ret = xmlTextWriterEndElement(writer);
+    XML_RET_CHECK_AND_GOTO(ret, out);
+
+    /* </snapActivate> or </snapDeactivate> */
+    ret = xmlTextWriterEndElement(writer);
+    XML_RET_CHECK_AND_GOTO(ret, out);
+
+    ret = 0;
 out:
 
-        return ret;
+    return ret;
 }
 #endif /* HAVE_LIB_XML */
 
@@ -5675,191 +5166,183 @@ out:
  * @return 0 on success and -1 on failure
  */
 int
-cli_xml_snapshot_delete (xmlTextWriterPtr writer, xmlDocPtr doc, dict_t *dict,
-                         gf_cli_rsp *rsp)
+cli_xml_snapshot_delete(cli_local_t *local, dict_t *dict, gf_cli_rsp *rsp)
 {
-        int     ret             = -1;
+    int ret = -1;
 #ifdef HAVE_LIB_XML
-        char   *str_value       = NULL;
-
-        GF_ASSERT (writer);
-        GF_ASSERT (doc);
-        GF_ASSERT (dict);
-
-        /* <snapshot> */
-        ret = xmlTextWriterStartElement (writer, (xmlChar *)"snapshot");
-        XML_RET_CHECK_AND_GOTO (ret, out);
-
-        ret = dict_get_str (dict, "snapname", &str_value);
-        if (ret) {
-                gf_log ("cli", GF_LOG_ERROR, "Failed to get snap name");
-                goto xmlend;
-        }
-
-        if (!rsp->op_ret) {
-                ret = xmlTextWriterWriteFormatElement (writer,
-                                                       (xmlChar *) "status",
-                                                       "Success");
-                XML_RET_CHECK_AND_GOTO (ret, xmlend);
-        } else {
-                ret = xmlTextWriterWriteFormatElement (writer,
-                                                       (xmlChar *) "status",
-                                                       "Failure");
-                XML_RET_CHECK_AND_GOTO (ret, xmlend);
-
-                ret = cli_xml_output_common (writer, rsp->op_ret,
-                                             rsp->op_errno,
-                                             rsp->op_errstr);
-                XML_RET_CHECK_AND_GOTO (ret, xmlend);
-        }
-
-
-        ret = xmlTextWriterWriteFormatElement (writer, (xmlChar *) "name",
-                                               "%s", str_value);
-        XML_RET_CHECK_AND_GOTO (ret, xmlend);
-
-        ret = dict_get_str (dict, "snapuuid", &str_value);
-        if (ret) {
-                gf_log ("cli", GF_LOG_ERROR, "Failed to get snap uuid");
-                goto xmlend;
-        }
-
-        ret = xmlTextWriterWriteFormatElement (writer, (xmlChar *) "uuid",
-                                               "%s", str_value);
-        XML_RET_CHECK_AND_GOTO (ret, out);
+    char *str_value = NULL;
+    xmlTextWriterPtr writer = local->writer;
+    xmlDocPtr doc = local->doc;
+
+    GF_ASSERT(writer);
+    GF_ASSERT(doc);
+    GF_ASSERT(dict);
+
+    /* <snapshot> */
+    ret = xmlTextWriterStartElement(writer, (xmlChar *)"snapshot");
+    XML_RET_CHECK_AND_GOTO(ret, out);
+
+    ret = dict_get_str(dict, "snapname", &str_value);
+    if (ret) {
+        gf_log("cli", GF_LOG_ERROR, "Failed to get snap name");
+        goto xmlend;
+    }
+
+    if (!rsp->op_ret) {
+        ret = xmlTextWriterWriteFormatElement(writer, (xmlChar *)"status",
+                                              "Success");
+    } else {
+        ret = xmlTextWriterWriteFormatElement(writer, (xmlChar *)"status",
+                                              "Failure");
+        XML_RET_CHECK_AND_GOTO(ret, xmlend);
+
+        ret = cli_xml_output_common(writer, rsp->op_ret, rsp->op_errno,
+                                    rsp->op_errstr);
+    }
+    XML_RET_CHECK_AND_GOTO(ret, xmlend);
+
+    ret = xmlTextWriterWriteFormatElement(writer, (xmlChar *)"name", "%s",
+                                          str_value);
+    XML_RET_CHECK_AND_GOTO(ret, xmlend);
+
+    ret = dict_get_str(dict, "snapuuid", &str_value);
+    if (ret) {
+        gf_log("cli", GF_LOG_ERROR, "Failed to get snap uuid");
+        goto xmlend;
+    }
+
+    ret = xmlTextWriterWriteFormatElement(writer, (xmlChar *)"uuid", "%s",
+                                          str_value);
+    XML_RET_CHECK_AND_GOTO(ret, out);
 
 xmlend:
-        /* </snapshot> */
-        ret = xmlTextWriterEndElement (writer);
-        XML_RET_CHECK_AND_GOTO (ret, out);
+    /* </snapshot> */
+    ret = xmlTextWriterEndElement(writer);
+    XML_RET_CHECK_AND_GOTO(ret, out);
 
 #endif /* HAVE_LIB_XML */
-        ret = 0;
+    ret = 0;
 out:
 
-        return ret;
+    return ret;
 }
 
 int
-cli_xml_output_snap_status_begin (cli_local_t *local, int op_ret, int op_errno,
-                                  char *op_errstr)
+cli_xml_output_snap_status_begin(cli_local_t *local, int op_ret, int op_errno,
+                                 char *op_errstr)
 {
 #if (HAVE_LIB_XML)
-        int                     ret = -1;
+    int ret = -1;
 
-        GF_ASSERT (local);
+    GF_ASSERT(local);
 
-        ret = cli_begin_xml_output (&(local->writer), &(local->doc));
-        XML_RET_CHECK_AND_GOTO (ret, out);
+    ret = cli_begin_xml_output(&(local->writer), &(local->doc));
+    XML_RET_CHECK_AND_GOTO(ret, out);
 
-        ret = cli_xml_output_common (local->writer, op_ret, op_errno,
-                                     op_errstr);
-        XML_RET_CHECK_AND_GOTO (ret, out);
+    ret = cli_xml_output_common(local->writer, op_ret, op_errno, op_errstr);
+    XML_RET_CHECK_AND_GOTO(ret, out);
 
-        /* <snapStatus> */
-        ret = xmlTextWriterStartElement (local->writer,
-                                         (xmlChar *) "snapStatus");
-        XML_RET_CHECK_AND_GOTO (ret, out);
+    /* <snapStatus> */
+    ret = xmlTextWriterStartElement(local->writer, (xmlChar *)"snapStatus");
+    XML_RET_CHECK_AND_GOTO(ret, out);
 
-        /* <snapshots> */
-        ret = xmlTextWriterStartElement (local->writer, (xmlChar *)"snapshots");
-        XML_RET_CHECK_AND_GOTO (ret, out);
+    /* <snapshots> */
+    ret = xmlTextWriterStartElement(local->writer, (xmlChar *)"snapshots");
+    XML_RET_CHECK_AND_GOTO(ret, out);
 
 out:
-        gf_log ("cli", GF_LOG_TRACE, "Returning %d", ret);
-        return ret;
+    gf_log("cli", GF_LOG_TRACE, "Returning %d", ret);
+    return ret;
 #else
-        return 0;
+    return 0;
 #endif
 }
 
 int
-cli_xml_output_snap_status_end (cli_local_t *local)
+cli_xml_output_snap_status_end(cli_local_t *local)
 {
 #if (HAVE_LIB_XML)
-        int     ret = -1;
+    int ret = -1;
 
-        GF_ASSERT (local);
+    GF_ASSERT(local);
 
-        /* </snapshots> */
-        ret = xmlTextWriterEndElement (local->writer);
-        XML_RET_CHECK_AND_GOTO (ret, out);
+    /* </snapshots> */
+    ret = xmlTextWriterEndElement(local->writer);
+    XML_RET_CHECK_AND_GOTO(ret, out);
 
-        /* </snapStatus> */
-        ret = xmlTextWriterEndElement (local->writer);
-        XML_RET_CHECK_AND_GOTO(ret, out);
+    /* </snapStatus> */
+    ret = xmlTextWriterEndElement(local->writer);
+    XML_RET_CHECK_AND_GOTO(ret, out);
 
-        ret = cli_end_xml_output (local->writer, local->doc);
+    ret = cli_end_xml_output(local->writer, local->doc);
 out:
-        gf_log ("cli", GF_LOG_TRACE, "Returning %d", ret);
-        return ret;
+    gf_log("cli", GF_LOG_TRACE, "Returning %d", ret);
+    return ret;
 #else
-        return 0;
+    return 0;
 #endif
 }
 
 int
-cli_xml_output_snap_delete_begin (cli_local_t *local, int op_ret, int op_errno,
-                                  char *op_errstr)
+cli_xml_output_snap_delete_begin(cli_local_t *local, int op_ret, int op_errno,
+                                 char *op_errstr)
 {
 #if (HAVE_LIB_XML)
-        int ret = -1;
-        int delete_cmd = -1;
+    int ret = -1;
+    int delete_cmd = -1;
 
-        GF_ASSERT (local);
+    GF_ASSERT(local);
 
-        ret = cli_begin_xml_output (&(local->writer), &(local->doc));
-        XML_RET_CHECK_AND_GOTO (ret, out);
+    ret = cli_begin_xml_output(&(local->writer), &(local->doc));
+    XML_RET_CHECK_AND_GOTO(ret, out);
 
-        ret = dict_get_int32 (local->dict, "sub-cmd", &delete_cmd);
-        if (ret) {
-                gf_log ("cli", GF_LOG_ERROR, "Failed to get sub-cmd");
-                goto out;
-        }
+    ret = dict_get_int32(local->dict, "sub-cmd", &delete_cmd);
+    if (ret) {
+        gf_log("cli", GF_LOG_ERROR, "Failed to get sub-cmd");
+        goto out;
+    }
 
-        ret = cli_xml_output_common (local->writer, op_ret, op_errno,
-                                             op_errstr);
-        XML_RET_CHECK_AND_GOTO (ret, out);
+    ret = cli_xml_output_common(local->writer, op_ret, op_errno, op_errstr);
+    XML_RET_CHECK_AND_GOTO(ret, out);
 
-        /* <snapStatus> */
-        ret = xmlTextWriterStartElement (local->writer,
-                                         (xmlChar *) "snapDelete");
-        XML_RET_CHECK_AND_GOTO (ret, out);
+    /* <snapStatus> */
+    ret = xmlTextWriterStartElement(local->writer, (xmlChar *)"snapDelete");
+    XML_RET_CHECK_AND_GOTO(ret, out);
 
-        /* <snapshots> */
-        ret = xmlTextWriterStartElement (local->writer, (xmlChar *)"snapshots");
-        XML_RET_CHECK_AND_GOTO (ret, out);
+    /* <snapshots> */
+    ret = xmlTextWriterStartElement(local->writer, (xmlChar *)"snapshots");
+    XML_RET_CHECK_AND_GOTO(ret, out);
 
 out:
-        gf_log ("cli", GF_LOG_TRACE, "Returning %d", ret);
-        return ret;
+    gf_log("cli", GF_LOG_TRACE, "Returning %d", ret);
+    return ret;
 #else
-        return 0;
+    return 0;
 #endif
 }
 
 int
-cli_xml_output_snap_delete_end (cli_local_t *local)
+cli_xml_output_snap_delete_end(cli_local_t *local)
 {
 #if (HAVE_LIB_XML)
-        int     ret = -1;
+    int ret = -1;
 
-        GF_ASSERT (local);
+    GF_ASSERT(local);
 
-        /* </snapshots> */
-        ret = xmlTextWriterEndElement (local->writer);
-        XML_RET_CHECK_AND_GOTO (ret, out);
+    /* </snapshots> */
+    ret = xmlTextWriterEndElement(local->writer);
+    XML_RET_CHECK_AND_GOTO(ret, out);
 
-        /* </snapDelete> */
-        ret = xmlTextWriterEndElement (local->writer);
-        XML_RET_CHECK_AND_GOTO(ret, out);
+    /* </snapDelete> */
+    ret = xmlTextWriterEndElement(local->writer);
+    XML_RET_CHECK_AND_GOTO(ret, out);
 
-        ret = cli_end_xml_output (local->writer, local->doc);
+    ret = cli_end_xml_output(local->writer, local->doc);
 out:
-        gf_log ("cli", GF_LOG_TRACE, "Returning %d", ret);
-        return ret;
+    gf_log("cli", GF_LOG_TRACE, "Returning %d", ret);
+    return ret;
 #else
-        return 0;
+    return 0;
 #endif
 }
 /* This function will generate xml output for all the snapshot commands
@@ -5873,478 +5356,484 @@ out:
  * @return 0 on success and -1 on failure
  */
 int
-cli_xml_output_snapshot (int cmd_type, dict_t *dict, int op_ret,
-                         int op_errno, char *op_errstr)
+cli_xml_output_snapshot(int cmd_type, dict_t *dict, int op_ret, int op_errno,
+                        char *op_errstr)
 {
 #if (HAVE_LIB_XML)
-        int                     ret     = -1;
-        xmlTextWriterPtr        writer  = NULL;
-        xmlDocPtr               doc     = NULL;
-
-        GF_ASSERT (dict);
-
-        ret = cli_begin_xml_output (&writer, &doc);
-        if (ret) {
-                gf_log ("cli", GF_LOG_ERROR, "Failed to output "
-                        "xml begin block");
-                goto out;
-        }
-
-        ret = cli_xml_output_common (writer, op_ret, op_errno, op_errstr);
-        if (ret) {
-                gf_log ("cli", GF_LOG_ERROR, "Failed to output "
-                        "xml common block");
-                goto out;
-        }
-
-        /* In case of command failure just printing the error message is good
-         * enough */
-        if (0 != op_ret) {
-                goto end;
-        }
-
-        switch (cmd_type) {
+    int ret = -1;
+    xmlTextWriterPtr writer = NULL;
+    xmlDocPtr doc = NULL;
+
+    GF_ASSERT(dict);
+
+    ret = cli_begin_xml_output(&writer, &doc);
+    if (ret) {
+        gf_log("cli", GF_LOG_ERROR,
+               "Failed to output "
+               "xml begin block");
+        goto out;
+    }
+
+    ret = cli_xml_output_common(writer, op_ret, op_errno, op_errstr);
+    if (ret) {
+        gf_log("cli", GF_LOG_ERROR,
+               "Failed to output "
+               "xml common block");
+        goto out;
+    }
+
+    /* In case of command failure just printing the error message is good
+     * enough */
+    if (0 != op_ret) {
+        goto end;
+    }
+
+    switch (cmd_type) {
         case GF_SNAP_OPTION_TYPE_CREATE:
-                ret = cli_xml_snapshot_create (writer, doc, dict);
-                if (ret) {
-                        gf_log ("cli", GF_LOG_ERROR, "Failed to create "
-                                "xml output for snapshot create command");
-                        goto out;
-                }
-                break;
+            ret = cli_xml_snapshot_create(writer, doc, dict);
+            if (ret) {
+                gf_log("cli", GF_LOG_ERROR,
+                       "Failed to create "
+                       "xml output for snapshot create command");
+                goto out;
+            }
+            break;
+        case GF_SNAP_OPTION_TYPE_CLONE:
+            ret = cli_xml_snapshot_clone(writer, doc, dict);
+            if (ret) {
+                gf_log("cli", GF_LOG_ERROR,
+                       "Failed to create "
+                       "xml output for snapshot clone command");
+                goto out;
+            }
+            break;
         case GF_SNAP_OPTION_TYPE_RESTORE:
-                ret = cli_xml_snapshot_restore (writer, doc, dict);
-                if (ret) {
-                        gf_log ("cli", GF_LOG_ERROR, "Failed to create "
-                                "xml output for snapshot restore command");
-                        goto out;
-                }
-                break;
+            ret = cli_xml_snapshot_restore(writer, doc, dict);
+            if (ret) {
+                gf_log("cli", GF_LOG_ERROR,
+                       "Failed to create "
+                       "xml output for snapshot restore command");
+                goto out;
+            }
+            break;
         case GF_SNAP_OPTION_TYPE_LIST:
-                ret = cli_xml_snapshot_list (writer, doc, dict);
-                if (ret) {
-                        gf_log ("cli", GF_LOG_ERROR, "Failed to create "
-                                "xml output for snapshot list command");
-                        goto out;
-                }
-                break;
+            ret = cli_xml_snapshot_list(writer, doc, dict);
+            if (ret) {
+                gf_log("cli", GF_LOG_ERROR,
+                       "Failed to create "
+                       "xml output for snapshot list command");
+                goto out;
+            }
+            break;
         case GF_SNAP_OPTION_TYPE_STATUS:
-                ret = cli_xml_snapshot_status (writer, doc, dict);
-                if (ret) {
-                        gf_log ("cli", GF_LOG_ERROR, "Failed to create"
-                                "xml output for snapshot status command");
-                        goto out;
-                }
-                break;
+            ret = cli_xml_snapshot_status(writer, doc, dict);
+            if (ret) {
+                gf_log("cli", GF_LOG_ERROR,
+                       "Failed to create"
+                       "xml output for snapshot status command");
+                goto out;
+            }
+            break;
         case GF_SNAP_OPTION_TYPE_INFO:
-                ret = cli_xml_snapshot_info (writer, doc, dict);
-                if (ret) {
-                        gf_log ("cli", GF_LOG_ERROR, "Failed to create "
-                                "xml output for snapshot info command");
-                        goto out;
-                }
-                break;
+            ret = cli_xml_snapshot_info(writer, doc, dict);
+            if (ret) {
+                gf_log("cli", GF_LOG_ERROR,
+                       "Failed to create "
+                       "xml output for snapshot info command");
+                goto out;
+            }
+            break;
         case GF_SNAP_OPTION_TYPE_ACTIVATE:
         case GF_SNAP_OPTION_TYPE_DEACTIVATE:
-                ret = cli_xml_snapshot_activate_deactivate (writer, doc,
-                                                            dict, cmd_type);
-                if (ret) {
-                        gf_log ("cli", GF_LOG_ERROR, "Failed to create "
-                                "xml output for snapshot config command");
-                }
-                break;
+            ret = cli_xml_snapshot_activate_deactivate(writer, doc, dict,
+                                                       cmd_type);
+            if (ret) {
+                gf_log("cli", GF_LOG_ERROR,
+                       "Failed to create "
+                       "xml output for snapshot config command");
+            }
+            break;
         case GF_SNAP_OPTION_TYPE_CONFIG:
-                ret = cli_xml_snapshot_config (writer, doc, dict);
-                if (ret) {
-                        gf_log ("cli", GF_LOG_ERROR, "Failed to create "
-                                "xml output for snapshot config command");
-                }
-                break;
+            ret = cli_xml_snapshot_config(writer, doc, dict);
+            if (ret) {
+                gf_log("cli", GF_LOG_ERROR,
+                       "Failed to create "
+                       "xml output for snapshot config command");
+            }
+            break;
         default:
-                gf_log ("cli", GF_LOG_ERROR,
-                        "Unexpected snapshot command: %d", cmd_type);
-                goto out;
-        }
+            gf_log("cli", GF_LOG_ERROR, "Unexpected snapshot command: %d",
+                   cmd_type);
+            goto out;
+    }
 
 end:
-        ret = cli_end_xml_output (writer, doc);
-        if (ret) {
-                gf_log ("cli", GF_LOG_ERROR, "Failed to output "
-                        "xml end block");
-                goto out;
-        }
-
-        ret = 0;
+    ret = cli_end_xml_output(writer, doc);
+    if (ret) {
+        gf_log("cli", GF_LOG_ERROR,
+               "Failed to output "
+               "xml end block");
+        goto out;
+    }
+
+    ret = 0;
 out:
-        return ret;
+    return ret;
 #else
-        return 0;
+    return 0;
 #endif /* HAVE_LIB_XML */
 }
 
 int
-cli_xml_snapshot_begin_composite_op (cli_local_t *local)
+cli_xml_snapshot_begin_composite_op(cli_local_t *local)
 {
-        int ret         = -1;
+    int ret = -1;
 #ifdef HAVE_LIB_XML
-        int cmd         = -1;
-        int type        =  -1;
-
-        ret = dict_get_int32 (local->dict, "sub-cmd", &cmd);
-        if (ret) {
-                gf_log ("cli", GF_LOG_ERROR, "Failed to get "
-                        "sub-cmd");
-                ret = 0;
-                goto out;
-        }
-
-        if (cmd == GF_SNAP_STATUS_TYPE_SNAP ||
-            cmd == GF_SNAP_DELETE_TYPE_SNAP){
-                ret = 0;
-                goto out;
-        }
-
-        ret = dict_get_int32 (local->dict, "type", &type);
-        if (ret) {
-                gf_log ("cli", GF_LOG_ERROR, "Failed to get snapshot "
-                        "command type from dictionary");
-                goto out;
-        }
-
-        if (GF_SNAP_OPTION_TYPE_STATUS == type)
-                ret = cli_xml_output_snap_status_begin (local, 0, 0, NULL);
-        else if (GF_SNAP_OPTION_TYPE_DELETE == type)
-                ret = cli_xml_output_snap_delete_begin (local, 0, 0, NULL);
+    int cmd = -1;
+    int type = -1;
+
+    ret = dict_get_int32(local->dict, "sub-cmd", &cmd);
+    if (ret) {
+        gf_log("cli", GF_LOG_ERROR,
+               "Failed to get "
+               "sub-cmd");
+        ret = 0;
+        goto out;
+    }
 
-        if (ret) {
-                gf_log ("cli", GF_LOG_ERROR, "Error creating xml output");
-                goto out;
-        }
+    if (cmd == GF_SNAP_STATUS_TYPE_ITER || cmd == GF_SNAP_DELETE_TYPE_SNAP) {
+        ret = 0;
+        goto out;
+    }
+
+    ret = dict_get_int32(local->dict, "type", &type);
+    if (ret) {
+        gf_log("cli", GF_LOG_ERROR,
+               "Failed to get snapshot "
+               "command type from dictionary");
+        goto out;
+    }
+
+    if (GF_SNAP_OPTION_TYPE_STATUS == type)
+        ret = cli_xml_output_snap_status_begin(local, 0, 0, NULL);
+    else if (GF_SNAP_OPTION_TYPE_DELETE == type)
+        ret = cli_xml_output_snap_delete_begin(local, 0, 0, NULL);
+
+    if (ret) {
+        gf_log("cli", GF_LOG_ERROR, "Error creating xml output");
+        goto out;
+    }
 
 #endif /* HAVE_LIB_XML */
-        ret = 0;
+    ret = 0;
 out:
-        return ret;
+    return ret;
 }
 
 int
-cli_xml_snapshot_end_composite_op (cli_local_t *local)
+cli_xml_snapshot_end_composite_op(cli_local_t *local)
 {
-        int ret         = -1;
+    int ret = -1;
 #ifdef HAVE_LIB_XML
-        int cmd         = -1;
-        int type        = -1;
-
-        ret = dict_get_int32 (local->dict, "sub-cmd", &cmd);
-        if (ret) {
-                gf_log ("cli", GF_LOG_ERROR, "Failed to get "
-                        "sub-cmd");
-                ret = 0;
-                goto out;
-        }
-
-        if (cmd == GF_SNAP_STATUS_TYPE_SNAP ||
-            cmd == GF_SNAP_DELETE_TYPE_SNAP){
-                ret = 0;
-                goto out;
-        }
-
-        ret = dict_get_int32 (local->dict, "type", &type);
-        if (ret) {
-                gf_log ("cli", GF_LOG_ERROR, "Failed to get snapshot "
-                        "command type from dictionary");
-                goto out;
-        }
-
-        if (GF_SNAP_OPTION_TYPE_STATUS == type)
-                ret = cli_xml_output_snap_status_end (local);
-        else if (GF_SNAP_OPTION_TYPE_DELETE == type)
-                ret = cli_xml_output_snap_delete_end (local);
+    int cmd = -1;
+    int type = -1;
+
+    ret = dict_get_int32(local->dict, "sub-cmd", &cmd);
+    if (ret) {
+        gf_log("cli", GF_LOG_ERROR,
+               "Failed to get "
+               "sub-cmd");
+        ret = 0;
+        goto out;
+    }
 
-        if (ret) {
-                gf_log ("cli", GF_LOG_ERROR, "Error creating xml "
-                                "output");
-                goto out;
-        }
-#endif /* HAVE_LIB_XML */
+    if (cmd == GF_SNAP_STATUS_TYPE_ITER || cmd == GF_SNAP_DELETE_TYPE_SNAP) {
         ret = 0;
+        goto out;
+    }
+
+    ret = dict_get_int32(local->dict, "type", &type);
+    if (ret) {
+        gf_log("cli", GF_LOG_ERROR,
+               "Failed to get snapshot "
+               "command type from dictionary");
+        goto out;
+    }
+
+    if (GF_SNAP_OPTION_TYPE_STATUS == type)
+        ret = cli_xml_output_snap_status_end(local);
+    else if (GF_SNAP_OPTION_TYPE_DELETE == type)
+        ret = cli_xml_output_snap_delete_end(local);
+
+    if (ret) {
+        gf_log("cli", GF_LOG_ERROR,
+               "Error creating xml "
+               "output");
+        goto out;
+    }
+#endif /* HAVE_LIB_XML */
+    ret = 0;
 out:
-        return ret;
+    return ret;
 }
 
 int
-cli_xml_snapshot_status_single_snap (cli_local_t *local, dict_t *dict,
-                                     char *key)
+cli_xml_snapshot_status_single_snap(cli_local_t *local, dict_t *dict, char *key)
 {
 #if (HAVE_LIB_XML)
-        int ret = -1;
+    int ret = -1;
 
-        GF_VALIDATE_OR_GOTO ("cli", (local != NULL), out);
-        GF_VALIDATE_OR_GOTO ("cli", (dict != NULL), out);
-        GF_VALIDATE_OR_GOTO ("cli", (key != NULL), out);
+    GF_VALIDATE_OR_GOTO("cli", (local != NULL), out);
+    GF_VALIDATE_OR_GOTO("cli", (dict != NULL), out);
+    GF_VALIDATE_OR_GOTO("cli", (key != NULL), out);
 
-        ret = cli_xml_snapshot_status_per_snap (local->writer, local->doc, dict,
-                                                key);
+    ret = cli_xml_snapshot_status_per_snap(local->writer, local->doc, dict,
+                                           key);
 out:
-        return ret;
+    return ret;
 #else
-        return 0;
+    return 0;
 #endif /* HAVE_LIB_XML */
 }
 
 int
-cli_xml_output_vol_getopts (dict_t *dict, int op_ret, int op_errno,
-                            char *op_errstr)
+cli_xml_output_vol_getopts(dict_t *dict, int op_ret, int op_errno,
+                           char *op_errstr)
 {
 #if (HAVE_LIB_XML)
-        int                     i = 0;
-        int                     ret = -1;
-        int                     count = 0;
-        xmlTextWriterPtr        writer = NULL;
-        xmlDocPtr               doc = NULL;
-        char                    *key = NULL;
-        char                    *value = NULL;
-        char                    dict_key[50] = {0,};
-
-        ret = cli_begin_xml_output (&writer, &doc);
-        if (ret)
-                goto out;
+    int i = 0;
+    int ret = -1;
+    int count = 0;
+    xmlTextWriterPtr writer = NULL;
+    xmlDocPtr doc = NULL;
+    char *key = NULL;
+    char *value = NULL;
+    char dict_key[50] = {
+        0,
+    };
+
+    ret = cli_begin_xml_output(&writer, &doc);
+    if (ret)
+        goto out;
+
+    ret = cli_xml_output_common(writer, op_ret, op_errno, op_errstr);
+    if (ret)
+        goto out;
+
+    ret = xmlTextWriterStartElement(writer, (xmlChar *)"volGetopts");
+    XML_RET_CHECK_AND_GOTO(ret, out);
+
+    ret = dict_get_int32(dict, "count", &count);
+    if (ret) {
+        gf_log("cli", GF_LOG_ERROR,
+               "Failed to retrieve count "
+               "from the dictionary");
+        goto out;
+    }
+    if (count <= 0) {
+        gf_log("cli", GF_LOG_ERROR,
+               "Value of count :%d is "
+               "invalid",
+               count);
+        ret = -1;
+        goto out;
+    }
+    ret = xmlTextWriterWriteFormatElement(writer, (xmlChar *)"count", "%d",
+                                          count);
+
+    XML_RET_CHECK_AND_GOTO(ret, out);
+
+    for (i = 1; i <= count; i++) {
+        sprintf(dict_key, "key%d", i);
+        ret = dict_get_str(dict, dict_key, &key);
+        if (ret) {
+            gf_log("cli", GF_LOG_ERROR,
+                   "Failed to"
+                   " retrieve %s from the "
+                   "dictionary",
+                   dict_key);
+            goto out;
+        }
+        sprintf(dict_key, "value%d", i);
+        ret = dict_get_str(dict, dict_key, &value);
+        if (ret) {
+            gf_log("cli", GF_LOG_ERROR,
+                   "Failed to "
+                   "retrieve key value for %s from"
+                   "the dictionary",
+                   dict_key);
+            goto out;
+        }
+        ret = xmlTextWriterStartElement(writer, (xmlChar *)"Opt");
+        XML_RET_CHECK_AND_GOTO(ret, out);
 
-        ret = cli_xml_output_common (writer, op_ret, op_errno, op_errstr);
-        if (ret)
-                goto out;
+        ret = xmlTextWriterWriteFormatElement(writer, (xmlChar *)"Option", "%s",
+                                              key);
+        XML_RET_CHECK_AND_GOTO(ret, out);
 
-        ret = xmlTextWriterStartElement (writer, (xmlChar *)"volGetopts");
-        XML_RET_CHECK_AND_GOTO (ret, out);
+        ret = xmlTextWriterWriteFormatElement(writer, (xmlChar *)"Value", "%s",
+                                              value);
+        XML_RET_CHECK_AND_GOTO(ret, out);
 
-        ret = dict_get_int32 (dict, "count", &count);
-        if (ret) {
-                gf_log ("cli", GF_LOG_ERROR, "Failed to retrieve count "
-                        "from the dictionary");
-                goto out;
-        }
-        if (count <= 0) {
-                gf_log ("cli", GF_LOG_ERROR, "Value of count :%d is "
-                        "invalid", count);
-                ret = -1;
-                goto out;
-        }
-        ret = xmlTextWriterWriteFormatElement (writer, (xmlChar *)"count",
-                                               "%d", count);
-
-        XML_RET_CHECK_AND_GOTO (ret, out);
-
-        for (i=1; i<=count; i++) {
-                sprintf (dict_key, "key%d", i);
-                ret = dict_get_str (dict, dict_key, &key);
-                if (ret) {
-                        gf_log ("cli", GF_LOG_ERROR, "Failed to"
-                                " retrieve %s from the "
-                                "dictionary", dict_key);
-                        goto out;
-                }
-                sprintf (dict_key, "value%d", i);
-                ret = dict_get_str (dict, dict_key, &value);
-                if (ret) {
-                        gf_log ("cli", GF_LOG_ERROR, "Failed to "
-                                "retrieve key value for %s from"
-                                "the dictionary", dict_key);
-                        goto out;
-                }
-                ret = xmlTextWriterWriteFormatElement (writer,
-                                                       (xmlChar *)"Option",
-                                                       "%s", key);
-                XML_RET_CHECK_AND_GOTO (ret, out);
-
-                ret = xmlTextWriterWriteFormatElement (writer,
-                                                       (xmlChar *)"Value",
-                                                       "%s", value);
-                XML_RET_CHECK_AND_GOTO (ret, out);
-        }
-        ret = cli_end_xml_output (writer, doc);
+        ret = xmlTextWriterEndElement(writer);
+        XML_RET_CHECK_AND_GOTO(ret, out);
+    }
+    ret = cli_end_xml_output(writer, doc);
 
 out:
-        gf_log ("cli", GF_LOG_DEBUG, "Returning %d", ret);
-        return ret;
+    gf_log("cli", GF_LOG_DEBUG, "Returning %d", ret);
+    return ret;
 #else
-        return 0;
+    return 0;
 #endif /* HAVE_LIB_XML */
 }
 
 int
-cli_quota_list_xml_error (cli_local_t *local, char *path,
-                          char *errstr)
+cli_quota_list_xml_error(cli_local_t *local, char *path, char *errstr)
 {
 #if (HAVE_LIB_XML)
-        int     ret     =       -1;
+    int ret = -1;
 
-        ret = xmlTextWriterStartElement (local->writer, (xmlChar *)"limit");
-        XML_RET_CHECK_AND_GOTO (ret, out);
+    ret = xmlTextWriterStartElement(local->writer, (xmlChar *)"limit");
+    XML_RET_CHECK_AND_GOTO(ret, out);
 
-        ret = xmlTextWriterWriteFormatElement (local->writer,
-                                              (xmlChar *)"path",
-                                               "%s", path);
-        XML_RET_CHECK_AND_GOTO (ret, out);
+    ret = xmlTextWriterWriteFormatElement(local->writer, (xmlChar *)"path",
+                                          "%s", path);
+    XML_RET_CHECK_AND_GOTO(ret, out);
 
-        ret = xmlTextWriterWriteFormatElement (local->writer,
-                                              (xmlChar *)"errstr",
-                                               "%s", errstr);
-        XML_RET_CHECK_AND_GOTO (ret, out);
+    ret = xmlTextWriterWriteFormatElement(local->writer, (xmlChar *)"errstr",
+                                          "%s", errstr);
+    XML_RET_CHECK_AND_GOTO(ret, out);
 
-        ret = xmlTextWriterEndElement (local->writer);
-        XML_RET_CHECK_AND_GOTO (ret, out);
+    ret = xmlTextWriterEndElement(local->writer);
+    XML_RET_CHECK_AND_GOTO(ret, out);
 
 out:
-        return ret;
+    return ret;
 #else
-        return 0;
+    return 0;
 #endif
 }
 
 int
-cli_quota_xml_output (cli_local_t *local, char *path, int64_t hl_str,
-                      char *sl_final, int64_t sl_num, int64_t used,
-                      int64_t avail, char *sl, char *hl,
-                      gf_boolean_t limit_set)
+cli_quota_xml_output(cli_local_t *local, char *path, int64_t hl_str,
+                     char *sl_final, int64_t sl_num, int64_t used,
+                     int64_t avail, char *sl, char *hl, gf_boolean_t limit_set)
 {
 #if (HAVE_LIB_XML)
-        int     ret             = -1;
-
-        ret = xmlTextWriterStartElement (local->writer, (xmlChar *)"limit");
-        XML_RET_CHECK_AND_GOTO (ret, out);
-
-        ret = xmlTextWriterWriteFormatElement (local->writer,
-                                              (xmlChar *)"path",
-                                              "%s", path);
-        XML_RET_CHECK_AND_GOTO (ret, out);
-
-        ret = xmlTextWriterWriteFormatElement (local->writer,
-                                              (xmlChar *)"hard_limit",
-                                               !limit_set ? "N/A" :
-                                               "%"PRIu64, hl_str);
-        XML_RET_CHECK_AND_GOTO (ret, out);
-
-        ret = xmlTextWriterWriteFormatElement (local->writer,
-                                              (xmlChar *)"soft_limit_percent",
-                                               !limit_set ? "N/A" :
-                                               "%s", sl_final);
-        XML_RET_CHECK_AND_GOTO (ret, out);
-
-        ret = xmlTextWriterWriteFormatElement (local->writer,
-                                              (xmlChar *)"soft_limit_value",
-                                               !limit_set ? "N/A" :
-                                               "%"PRIu64, sl_num);
-        XML_RET_CHECK_AND_GOTO (ret, out);
-
-        ret = xmlTextWriterWriteFormatElement (local->writer,
-                                               (xmlChar *)"used_space",
-                                               "%"PRIu64, used);
-        XML_RET_CHECK_AND_GOTO (ret, out);
-
-        ret = xmlTextWriterWriteFormatElement (local->writer,
-                                               (xmlChar *)"avail_space",
-                                               !limit_set ? "N/A" :
-                                               "%"PRIu64, avail);
-        XML_RET_CHECK_AND_GOTO (ret, out);
-
-        ret = xmlTextWriterWriteFormatElement (local->writer,
-                                              (xmlChar *)"sl_exceeded",
-                                               !limit_set ? "N/A" :
-                                               "%s", sl);
-        XML_RET_CHECK_AND_GOTO (ret, out);
-
-        ret = xmlTextWriterWriteFormatElement (local->writer,
-                                               (xmlChar *)"hl_exceeded",
-                                               !limit_set ? "N/A" :
-                                               "%s", hl);
-        XML_RET_CHECK_AND_GOTO (ret, out);
-
-
-        ret = xmlTextWriterEndElement (local->writer);
-        XML_RET_CHECK_AND_GOTO (ret, out);
+    int ret = -1;
+
+    ret = xmlTextWriterStartElement(local->writer, (xmlChar *)"limit");
+    XML_RET_CHECK_AND_GOTO(ret, out);
+
+    ret = xmlTextWriterWriteFormatElement(local->writer, (xmlChar *)"path",
+                                          "%s", path);
+    XML_RET_CHECK_AND_GOTO(ret, out);
+
+    ret = xmlTextWriterWriteFormatElement(
+        local->writer, (xmlChar *)"hard_limit", !limit_set ? "N/A" : "%" PRId64,
+        hl_str);
+    XML_RET_CHECK_AND_GOTO(ret, out);
+
+    ret = xmlTextWriterWriteFormatElement(local->writer,
+                                          (xmlChar *)"soft_limit_percent",
+                                          !limit_set ? "N/A" : "%s", sl_final);
+    XML_RET_CHECK_AND_GOTO(ret, out);
+
+    ret = xmlTextWriterWriteFormatElement(
+        local->writer, (xmlChar *)"soft_limit_value",
+        !limit_set ? "N/A" : "%" PRId64, sl_num);
+    XML_RET_CHECK_AND_GOTO(ret, out);
+
+    ret = xmlTextWriterWriteFormatElement(
+        local->writer, (xmlChar *)"used_space", "%" PRId64, used);
+    XML_RET_CHECK_AND_GOTO(ret, out);
+
+    ret = xmlTextWriterWriteFormatElement(
+        local->writer, (xmlChar *)"avail_space",
+        !limit_set ? "N/A" : "%" PRId64, avail);
+    XML_RET_CHECK_AND_GOTO(ret, out);
+
+    ret = xmlTextWriterWriteFormatElement(
+        local->writer, (xmlChar *)"sl_exceeded", !limit_set ? "N/A" : "%s", sl);
+    XML_RET_CHECK_AND_GOTO(ret, out);
+
+    ret = xmlTextWriterWriteFormatElement(
+        local->writer, (xmlChar *)"hl_exceeded", !limit_set ? "N/A" : "%s", hl);
+    XML_RET_CHECK_AND_GOTO(ret, out);
+
+    ret = xmlTextWriterEndElement(local->writer);
+    XML_RET_CHECK_AND_GOTO(ret, out);
 
 out:
-        return ret;
+    return ret;
 #else
-        return 0;
+    return 0;
 #endif /* HAVE_LIB_XML */
 }
 
 int
-cli_quota_object_xml_output (cli_local_t *local, char *path, char *sl_str,
-                             int64_t sl_val, quota_limits_t *limits,
-                             quota_meta_t *used_space, int64_t avail,
-                             char *sl, char *hl, gf_boolean_t limit_set)
+cli_quota_object_xml_output(cli_local_t *local, char *path, char *sl_str,
+                            int64_t sl_val, quota_limits_t *limits,
+                            quota_meta_t *used_space, int64_t avail, char *sl,
+                            char *hl, gf_boolean_t limit_set)
 {
 #if (HAVE_LIB_XML)
-        int     ret             = -1;
-
-        ret = xmlTextWriterStartElement (local->writer, (xmlChar *)"limit");
-        XML_RET_CHECK_AND_GOTO (ret, out);
-
-        ret = xmlTextWriterWriteFormatElement (local->writer,
-                                              (xmlChar *)"path",
-                                              "%s", path);
-        XML_RET_CHECK_AND_GOTO (ret, out);
+    int ret = -1;
 
-        ret = xmlTextWriterWriteFormatElement (local->writer,
-                                              (xmlChar *)"hard_limit",
-                                               !limit_set ? "N/A" :
-                                               "%"PRIu64, limits->hl);
-        XML_RET_CHECK_AND_GOTO (ret, out);
+    ret = xmlTextWriterStartElement(local->writer, (xmlChar *)"limit");
+    XML_RET_CHECK_AND_GOTO(ret, out);
 
-        ret = xmlTextWriterWriteFormatElement (local->writer,
-                                              (xmlChar *)"soft_limit_percent",
-                                               !limit_set ? "N/A" :
-                                               "%s", sl_str);
-        XML_RET_CHECK_AND_GOTO (ret, out);
+    ret = xmlTextWriterWriteFormatElement(local->writer, (xmlChar *)"path",
+                                          "%s", path);
+    XML_RET_CHECK_AND_GOTO(ret, out);
 
-        ret = xmlTextWriterWriteFormatElement (local->writer,
-                                              (xmlChar *)"soft_limit_value",
-                                               !limit_set ? "N/A" :
-                                               "%"PRIu64, sl_val);
-        XML_RET_CHECK_AND_GOTO (ret, out);
+    ret = xmlTextWriterWriteFormatElement(
+        local->writer, (xmlChar *)"hard_limit", !limit_set ? "N/A" : "%" PRId64,
+        limits->hl);
+    XML_RET_CHECK_AND_GOTO(ret, out);
 
-        ret = xmlTextWriterWriteFormatElement (local->writer,
-                                              (xmlChar *)"file_count",
-                                               "%"PRIu64,
-                                               used_space->file_count);
+    ret = xmlTextWriterWriteFormatElement(local->writer,
+                                          (xmlChar *)"soft_limit_percent",
+                                          !limit_set ? "N/A" : "%s", sl_str);
+    XML_RET_CHECK_AND_GOTO(ret, out);
 
-        XML_RET_CHECK_AND_GOTO (ret, out);
+    ret = xmlTextWriterWriteFormatElement(
+        local->writer, (xmlChar *)"soft_limit_value",
+        !limit_set ? "N/A" : "%" PRIu64, sl_val);
+    XML_RET_CHECK_AND_GOTO(ret, out);
 
-        ret = xmlTextWriterWriteFormatElement (local->writer,
-                                              (xmlChar *)"dir_count", "%"PRIu64,
-                                               used_space->dir_count);
+    ret = xmlTextWriterWriteFormatElement(local->writer,
+                                          (xmlChar *)"file_count", "%" PRId64,
+                                          used_space->file_count);
 
-        XML_RET_CHECK_AND_GOTO (ret, out);
+    XML_RET_CHECK_AND_GOTO(ret, out);
 
+    ret = xmlTextWriterWriteFormatElement(local->writer, (xmlChar *)"dir_count",
+                                          "%" PRIu64, used_space->dir_count);
 
-        ret = xmlTextWriterWriteFormatElement (local->writer,
-                                              (xmlChar *)"available",
-                                               !limit_set ? "N/A" :
-                                              "%"PRIu64, avail);
+    XML_RET_CHECK_AND_GOTO(ret, out);
 
-        XML_RET_CHECK_AND_GOTO (ret, out);
+    ret = xmlTextWriterWriteFormatElement(local->writer, (xmlChar *)"available",
+                                          !limit_set ? "N/A" : "%" PRId64,
+                                          avail);
 
-        ret = xmlTextWriterWriteFormatElement (local->writer,
-                                              (xmlChar *)"sl_exceeded",
-                                               !limit_set ? "N/A" :
-                                               "%s", sl);
-        XML_RET_CHECK_AND_GOTO (ret, out);
+    XML_RET_CHECK_AND_GOTO(ret, out);
 
-        ret = xmlTextWriterWriteFormatElement (local->writer,
-                                               (xmlChar *)"hl_exceeded",
-                                               !limit_set ? "N/A" :
-                                               "%s", hl);
-        XML_RET_CHECK_AND_GOTO (ret, out);
+    ret = xmlTextWriterWriteFormatElement(
+        local->writer, (xmlChar *)"sl_exceeded", !limit_set ? "N/A" : "%s", sl);
+    XML_RET_CHECK_AND_GOTO(ret, out);
 
+    ret = xmlTextWriterWriteFormatElement(
+        local->writer, (xmlChar *)"hl_exceeded", !limit_set ? "N/A" : "%s", hl);
+    XML_RET_CHECK_AND_GOTO(ret, out);
 
-        ret = xmlTextWriterEndElement (local->writer);
-        XML_RET_CHECK_AND_GOTO (ret, out);
+    ret = xmlTextWriterEndElement(local->writer);
+    XML_RET_CHECK_AND_GOTO(ret, out);
 
 out:
-        return ret;
+    return ret;
 #else
-        return 0;
+    return 0;
 #endif /* HAVE_LIB_XML */
 }
diff --git a/cli/src/cli.c b/cli/src/cli.c
index 46736bbf76d..a52b39c5fb8 100644
--- a/cli/src/cli.c
+++ b/cli/src/cli.c
@@ -1,5 +1,5 @@
 /*
-   Copyright (c) 2010-2012 Red Hat, Inc. <http://www.redhat.com>
+   Copyright (c) 2010-2016 Red Hat, Inc. <http://www.redhat.com>
    This file is part of GlusterFS.
 
    This file is licensed to you under your choice of the GNU Lesser
@@ -33,755 +33,880 @@
 #include <malloc.h>
 #endif
 
-#ifdef HAVE_MALLOC_STATS
-#ifdef DEBUG
-#include <mcheck.h>
-#endif
-#endif
-
 #include "cli.h"
 #include "cli-quotad-client.h"
 #include "cli-cmd.h"
 #include "cli-mem-types.h"
 
-#include "xlator.h"
-#include "glusterfs.h"
-#include "compat.h"
-#include "logging.h"
-#include "dict.h"
-#include "list.h"
-#include "timer.h"
-#include "stack.h"
-#include "revision.h"
-#include "common-utils.h"
-#include "event.h"
-#include "globals.h"
-#include "syscall.h"
-#include "call-stub.h"
+#include <glusterfs/xlator.h>
+#include <glusterfs/glusterfs.h>
+#include <glusterfs/compat.h>
+#include <glusterfs/logging.h>
+#include <glusterfs/dict.h>
+#include <glusterfs/list.h>
+#include <glusterfs/timer.h>
+#include <glusterfs/stack.h>
+#include <glusterfs/revision.h>
+#include <glusterfs/common-utils.h>
+#include <glusterfs/gf-event.h>
+#include <glusterfs/syscall.h>
+#include <glusterfs/call-stub.h>
 #include <fnmatch.h>
 
 #include "xdr-generic.h"
 
-extern int connected;
 /* using argp for command line parsing */
 
-const char *argp_program_version = ""                                 \
-        PACKAGE_NAME" "PACKAGE_VERSION" built on "__DATE__" "__TIME__ \
-        "\nRepository revision: " GLUSTERFS_REPOSITORY_REVISION "\n"  \
-        "Copyright (c) 2006-2011 Gluster Inc. "                       \
-        "<http://www.gluster.com>\n"                                  \
-        "GlusterFS comes with ABSOLUTELY NO WARRANTY.\n"              \
-        "You may redistribute copies of GlusterFS under the terms of "\
-        "the GNU General Public License.";
-
+const char *argp_program_version =
+    "" PACKAGE_NAME " " PACKAGE_VERSION
+    "\nRepository revision: " GLUSTERFS_REPOSITORY_REVISION
+    "\n"
+    "Copyright (c) 2006-2016 Red Hat, Inc. "
+    "<https://www.gluster.org/>\n"
+    "GlusterFS comes with ABSOLUTELY NO WARRANTY.\n"
+    "It is licensed to you under your choice of the GNU Lesser\n"
+    "General Public License, version 3 or any later version (LGPLv3\n"
+    "or later), or the GNU General Public License, version 2 (GPLv2),\n"
+    "in all cases as published by the Free Software Foundation.";
 const char *argp_program_bug_address = "<" PACKAGE_BUGREPORT ">";
 
-
-
 struct rpc_clnt *global_quotad_rpc;
+
 struct rpc_clnt *global_rpc;
 
 rpc_clnt_prog_t *cli_rpc_prog;
 
-
 extern struct rpc_clnt_program cli_prog;
 
+int cli_default_conn_timeout = 120;
+int cli_ten_minutes_timeout = 600;
+
 static int
-glusterfs_ctx_defaults_init (glusterfs_ctx_t *ctx)
+glusterfs_ctx_defaults_init(glusterfs_ctx_t *ctx)
 {
-        cmd_args_t    *cmd_args = NULL;
-        struct rlimit  lim = {0, };
-        call_pool_t   *pool = NULL;
-        int            ret         = -1;
-
-        ret = xlator_mem_acct_init (THIS, cli_mt_end);
-        if (ret != 0) {
-                return ret;
-        }
-
-        ctx->process_uuid = generate_glusterfs_ctx_id ();
-        if (!ctx->process_uuid)
-                return -1;
-
-        ctx->page_size  = 128 * GF_UNIT_KB;
-
-        ctx->iobuf_pool = iobuf_pool_new ();
-        if (!ctx->iobuf_pool)
-                return -1;
-
-        ctx->event_pool = event_pool_new (DEFAULT_EVENT_POOL_SIZE,
-                                          STARTING_EVENT_THREADS);
-        if (!ctx->event_pool)
-                return -1;
-
-        pool = GF_CALLOC (1, sizeof (call_pool_t),
-                          cli_mt_call_pool_t);
-        if (!pool)
-                return -1;
-
-        /* frame_mem_pool size 112 * 64 */
-        pool->frame_mem_pool = mem_pool_new (call_frame_t, 32);
-        if (!pool->frame_mem_pool)
-                return -1;
-
-        /* stack_mem_pool size 256 * 128 */
-        pool->stack_mem_pool = mem_pool_new (call_stack_t, 16);
-
-        if (!pool->stack_mem_pool)
-                return -1;
-
-        ctx->stub_mem_pool = mem_pool_new (call_stub_t, 16);
-        if (!ctx->stub_mem_pool)
-                return -1;
-
-        ctx->dict_pool = mem_pool_new (dict_t, 32);
-        if (!ctx->dict_pool)
-                return -1;
-
-        ctx->dict_pair_pool = mem_pool_new (data_pair_t, 512);
-        if (!ctx->dict_pair_pool)
-                return -1;
-
-        ctx->dict_data_pool = mem_pool_new (data_t, 512);
-        if (!ctx->dict_data_pool)
-                return -1;
-
-        ctx->logbuf_pool = mem_pool_new (log_buf_t, 256);
-        if (!ctx->logbuf_pool)
-                return -1;
-
-        INIT_LIST_HEAD (&pool->all_frames);
-        LOCK_INIT (&pool->lock);
-        ctx->pool = pool;
-
-        pthread_mutex_init (&(ctx->lock), NULL);
-
-        cmd_args = &ctx->cmd_args;
-
-        INIT_LIST_HEAD (&cmd_args->xlator_options);
+    cmd_args_t *cmd_args = NULL;
+    struct rlimit lim = {
+        0,
+    };
+    call_pool_t *pool = NULL;
+    int ret = -1;
+
+    if (!ctx)
+        return ret;
 
-        lim.rlim_cur = RLIM_INFINITY;
-        lim.rlim_max = RLIM_INFINITY;
-        setrlimit (RLIMIT_CORE, &lim);
+    ret = xlator_mem_acct_init(THIS, cli_mt_end);
+    if (ret != 0) {
+        gf_log("cli", GF_LOG_ERROR, "Memory accounting init failed.");
+        return ret;
+    }
+
+    /* Resetting ret to -1 to so in case of failure
+     * we can relese allocated resource.
+     */
+    ret = -1;
+
+    ctx->process_uuid = generate_glusterfs_ctx_id();
+    if (!ctx->process_uuid) {
+        gf_log("cli", GF_LOG_ERROR, "Failed to generate uuid.");
+        goto out;
+    }
+
+    ctx->page_size = 128 * GF_UNIT_KB;
+
+    ctx->iobuf_pool = iobuf_pool_new();
+    if (!ctx->iobuf_pool) {
+        gf_log("cli", GF_LOG_ERROR, "Failed to create iobuf pool.");
+        goto out;
+    }
+
+    ctx->event_pool = gf_event_pool_new(DEFAULT_EVENT_POOL_SIZE,
+                                        STARTING_EVENT_THREADS);
+    if (!ctx->event_pool) {
+        gf_log("cli", GF_LOG_ERROR, "Failed to create event pool.");
+        goto out;
+    }
+
+    pool = GF_CALLOC(1, sizeof(call_pool_t), cli_mt_call_pool_t);
+    if (!pool) {
+        gf_log("cli", GF_LOG_ERROR, "Failed to create call pool.");
+        goto out;
+    }
+
+    /* frame_mem_pool size 112 * 64 */
+    pool->frame_mem_pool = mem_pool_new(call_frame_t, 32);
+    if (!pool->frame_mem_pool) {
+        gf_log("cli", GF_LOG_ERROR, "Failed to create frame mem pool.");
+        goto out;
+    }
+
+    /* stack_mem_pool size 256 * 128 */
+    pool->stack_mem_pool = mem_pool_new(call_stack_t, 16);
+
+    if (!pool->stack_mem_pool) {
+        gf_log("cli", GF_LOG_ERROR, "Failed to create stack mem pool.");
+        goto out;
+    }
+
+    ctx->stub_mem_pool = mem_pool_new(call_stub_t, 16);
+    if (!ctx->stub_mem_pool) {
+        gf_log("cli", GF_LOG_ERROR, "Failed to stub mem pool.");
+        goto out;
+    }
+
+    ctx->dict_pool = mem_pool_new(dict_t, 32);
+    if (!ctx->dict_pool) {
+        gf_log("cli", GF_LOG_ERROR, "Failed to create dict pool.");
+        goto out;
+    }
+
+    ctx->dict_pair_pool = mem_pool_new(data_pair_t, 512);
+    if (!ctx->dict_pair_pool) {
+        gf_log("cli", GF_LOG_ERROR, "Failed to create dict pair pool.");
+        goto out;
+    }
+
+    ctx->dict_data_pool = mem_pool_new(data_t, 512);
+    if (!ctx->dict_data_pool) {
+        gf_log("cli", GF_LOG_ERROR, "Failed to create dict data pool.");
+        goto out;
+    }
+
+    ctx->logbuf_pool = mem_pool_new(log_buf_t, 256);
+    if (!ctx->logbuf_pool) {
+        gf_log("cli", GF_LOG_ERROR, "Failed to create logbuf pool.");
+        goto out;
+    }
+
+    INIT_LIST_HEAD(&pool->all_frames);
+    LOCK_INIT(&pool->lock);
+    ctx->pool = pool;
+
+    cmd_args = &ctx->cmd_args;
+
+    INIT_LIST_HEAD(&cmd_args->xlator_options);
+
+    lim.rlim_cur = RLIM_INFINITY;
+    lim.rlim_max = RLIM_INFINITY;
+    setrlimit(RLIMIT_CORE, &lim);
+
+    ret = 0;
 
-        return 0;
+out:
+    if (ret != 0) {
+        if (pool) {
+            mem_pool_destroy(pool->frame_mem_pool);
+            mem_pool_destroy(pool->stack_mem_pool);
+        }
+        GF_FREE(pool);
+        pool = NULL;
+        GF_FREE(ctx->process_uuid);
+        mem_pool_destroy(ctx->stub_mem_pool);
+        mem_pool_destroy(ctx->dict_pool);
+        mem_pool_destroy(ctx->dict_pair_pool);
+        mem_pool_destroy(ctx->dict_data_pool);
+        mem_pool_destroy(ctx->logbuf_pool);
+    }
+
+    return ret;
 }
 
-
 static int
-logging_init (glusterfs_ctx_t *ctx, struct cli_state *state)
+logging_init(glusterfs_ctx_t *ctx, struct cli_state *state)
 {
-        char *log_file = state->log_file ? state->log_file :
-                         DEFAULT_CLI_LOG_FILE_DIRECTORY "/cli.log";
-
-        /* passing ident as NULL means to use default ident for syslog */
-        if (gf_log_init (ctx, log_file, NULL) == -1) {
-                fprintf (stderr, "ERROR: failed to open logfile %s\n",
-                         log_file);
-                return -1;
-        }
-
-        /* CLI should not have something to DEBUG after the release,
-           hence defaulting to INFO loglevel */
-        gf_log_set_loglevel ((state->log_level == GF_LOG_NONE) ? GF_LOG_INFO :
-                             state->log_level);
-
-        return 0;
+    char *log_file = state->log_file ? state->log_file
+                                     : DEFAULT_CLI_LOG_FILE_DIRECTORY
+                         "/cli.log";
+
+    /* passing ident as NULL means to use default ident for syslog */
+    if (gf_log_init(ctx, log_file, NULL) == -1) {
+        fprintf(stderr, "ERROR: failed to open logfile %s\n", log_file);
+    }
+
+    /* CLI should not have something to DEBUG after the release,
+       hence defaulting to INFO loglevel */
+    gf_log_set_loglevel(ctx, (state->log_level == GF_LOG_NONE)
+                                 ? GF_LOG_INFO
+                                 : state->log_level);
+
+    return 0;
 }
 
 int
-cli_submit_request (struct rpc_clnt *rpc, void *req, call_frame_t *frame,
-                    rpc_clnt_prog_t *prog,
-                    int procnum, struct iobref *iobref,
-                    xlator_t *this, fop_cbk_fn_t cbkfn, xdrproc_t xdrproc)
+cli_submit_request(struct rpc_clnt *rpc, void *req, call_frame_t *frame,
+                   rpc_clnt_prog_t *prog, int procnum, struct iobref *iobref,
+                   xlator_t *this, fop_cbk_fn_t cbkfn, xdrproc_t xdrproc)
 {
-        int                     ret         = -1;
-        int                     count      = 0;
-        struct iovec            iov         = {0, };
-        struct iobuf            *iobuf = NULL;
-        char                    new_iobref = 0;
-        ssize_t                 xdr_size   = 0;
-
-        GF_ASSERT (this);
-
-        if (req) {
-                xdr_size = xdr_sizeof (xdrproc, req);
-                iobuf = iobuf_get2 (this->ctx->iobuf_pool, xdr_size);
-                if (!iobuf) {
-                        goto out;
-                };
-
-                if (!iobref) {
-                        iobref = iobref_new ();
-                        if (!iobref) {
-                                goto out;
-                        }
-
-                        new_iobref = 1;
-                }
-
-                iobref_add (iobref, iobuf);
-
-                iov.iov_base = iobuf->ptr;
-                iov.iov_len  = iobuf_size (iobuf);
-
-
-                /* Create the xdr payload */
-                ret = xdr_serialize_generic (iov, req, xdrproc);
-                if (ret == -1) {
-                        goto out;
-                }
-                iov.iov_len = ret;
-                count = 1;
+    int ret = -1;
+    int count = 0;
+    struct iovec iov = {
+        0,
+    };
+    struct iobuf *iobuf = NULL;
+    char new_iobref = 0;
+    ssize_t xdr_size = 0;
+
+    GF_ASSERT(this);
+
+    if (req) {
+        xdr_size = xdr_sizeof(xdrproc, req);
+        iobuf = iobuf_get2(this->ctx->iobuf_pool, xdr_size);
+        if (!iobuf) {
+            goto out;
+        };
+
+        if (!iobref) {
+            iobref = iobref_new();
+            if (!iobref) {
+                goto out;
+            }
+
+            new_iobref = 1;
+        }
+
+        iobref_add(iobref, iobuf);
+
+        iov.iov_base = iobuf->ptr;
+        iov.iov_len = iobuf_size(iobuf);
+
+        /* Create the xdr payload */
+        ret = xdr_serialize_generic(iov, req, xdrproc);
+        if (ret == -1) {
+            goto out;
         }
+        iov.iov_len = ret;
+        count = 1;
+    }
 
-        if (!rpc)
-                rpc = global_rpc;
-        /* Send the msg */
-        ret = rpc_clnt_submit (rpc, prog, procnum, cbkfn,
-                               &iov, count,
-                               NULL, 0, iobref, frame, NULL, 0, NULL, 0, NULL);
-        ret = 0;
+    if (!rpc)
+        rpc = global_rpc;
+    /* Send the msg */
+    ret = rpc_clnt_submit(rpc, prog, procnum, cbkfn, &iov, count, NULL, 0,
+                          iobref, frame, NULL, 0, NULL, 0, NULL);
+    ret = 0;
 
 out:
-        if (new_iobref)
-                iobref_unref (iobref);
-        if (iobuf)
-                iobuf_unref (iobuf);
-        return ret;
+    if (new_iobref)
+        iobref_unref(iobref);
+    if (iobuf)
+        iobuf_unref(iobuf);
+    return ret;
 }
 
 int
-cli_rpc_notify (struct rpc_clnt *rpc, void *mydata, rpc_clnt_event_t event,
-                void *data)
+cli_rpc_notify(struct rpc_clnt *rpc, void *mydata, rpc_clnt_event_t event,
+               void *data)
 {
-        xlator_t                *this = NULL;
-        int                     ret = 0;
-
-        this = mydata;
+    xlator_t *this = NULL;
+    int ret = 0;
 
-        switch (event) {
-        case RPC_CLNT_CONNECT:
-        {
+    this = mydata;
 
-                cli_cmd_broadcast_connected ();
-                gf_log (this->name, GF_LOG_TRACE, "got RPC_CLNT_CONNECT");
-               break;
+    switch (event) {
+        case RPC_CLNT_CONNECT: {
+            cli_cmd_broadcast_connected(_gf_true);
+            gf_log(this->name, GF_LOG_TRACE, "got RPC_CLNT_CONNECT");
+            break;
         }
 
-        case RPC_CLNT_DISCONNECT:
-        {
-                gf_log (this->name, GF_LOG_TRACE, "got RPC_CLNT_DISCONNECT");
-                connected = 0;
-                if (!global_state->prompt && global_state->await_connected) {
-                        ret = 1;
-                        cli_out ("Connection failed. Please check if gluster "
-                                  "daemon is operational.");
-                        exit (ret);
-                }
-                break;
+        case RPC_CLNT_DISCONNECT: {
+            cli_cmd_broadcast_connected(_gf_false);
+            gf_log(this->name, GF_LOG_TRACE, "got RPC_CLNT_DISCONNECT");
+            if (!global_state->prompt && global_state->await_connected) {
+                ret = 1;
+                cli_out(
+                    "Connection failed. Please check if gluster "
+                    "daemon is operational.");
+                exit(ret);
+            }
+            break;
         }
 
         default:
-                gf_log (this->name, GF_LOG_TRACE,
-                        "got some other RPC event %d", event);
-                ret = 0;
-                break;
-        }
+            gf_log(this->name, GF_LOG_TRACE, "got some other RPC event %d",
+                   event);
+            ret = 0;
+            break;
+    }
 
-        return ret;
+    return ret;
 }
 
+static gf_boolean_t
+is_valid_int(char *str)
+{
+    if (*str == '-')
+        ++str;
+
+    /* Handle empty string or just "-".*/
+    if (!*str)
+        return _gf_false;
+
+    /* Check for non-digit chars in the rest of the string */
+    while (*str) {
+        if (!isdigit(*str))
+            return _gf_false;
+        else
+            ++str;
+    }
+    return _gf_true;
+}
 
 /*
  * ret: 0: option successfully processed
  *      1: signalling end of option list
- *     -1: unknown option or other issue
+ *     -1: unknown option
+ *     -2: parsing issue (avoid unknown option error)
  */
 int
-cli_opt_parse (char *opt, struct cli_state *state)
+cli_opt_parse(char *opt, struct cli_state *state)
 {
-        char            *oarg           = NULL;
-        gf_boolean_t    secure_mgmt_tmp = 0;
-
-        if (strcmp (opt, "") == 0)
-                return 1;
-
-        if (strcmp (opt, "version") == 0) {
-                cli_out ("%s", argp_program_version);
-                exit (0);
-        }
-
-        if (strcmp (opt, "print-logdir") == 0) {
-                cli_out ("%s", DEFAULT_LOG_FILE_DIRECTORY);
-                exit (0);
-        }
-
-        if (strcmp (opt, "print-statedumpdir") == 0) {
-                cli_out ("%s", DEFAULT_VAR_RUN_DIRECTORY);
-                exit (0);
-        }
-
-        if (strcmp (opt, "xml") == 0) {
+    char *oarg = NULL;
+    gf_boolean_t secure_mgmt_tmp = 0;
+
+    if (strcmp(opt, "") == 0)
+        return 1;
+    if (strcmp(opt, "help") == 0) {
+        cli_out(
+            " peer help                - display help for peer commands\n"
+            " volume help              - display help for volume commands\n"
+            " volume bitrot help       - display help for volume"
+            " bitrot commands\n"
+            " volume quota help        - display help for volume"
+            " quota commands\n"
+            " snapshot help            - display help for snapshot commands\n"
+            " global help              - list global commands\n");
+        exit(0);
+    }
+
+    if (strcmp(opt, "version") == 0) {
+        cli_out("%s", argp_program_version);
+        exit(0);
+    }
+
+    if (strcmp(opt, "print-logdir") == 0) {
+        cli_out("%s", DEFAULT_LOG_FILE_DIRECTORY);
+        exit(0);
+    }
+
+    if (strcmp(opt, "print-statedumpdir") == 0) {
+        cli_out("%s", DEFAULT_VAR_RUN_DIRECTORY);
+        exit(0);
+    }
+
+    if (strcmp(opt, "xml") == 0) {
 #if (HAVE_LIB_XML)
-                state->mode |= GLUSTER_MODE_XML;
+        state->mode |= GLUSTER_MODE_XML;
 #else
-                cli_err ("XML output not supported. Ignoring '--xml' option");
+        cli_err("XML output not supported. Ignoring '--xml' option");
 #endif
-                return 0;
-        }
+        return 0;
+    }
 
-        if (strcmp (opt, "wignore") == 0) {
-                state->mode |= GLUSTER_MODE_WIGNORE;
-                return 0;
-        }
+    if (strcmp(opt, "nolog") == 0) {
+        state->mode |= GLUSTER_MODE_GLFSHEAL_NOLOG;
+        return 0;
+    }
 
-        oarg = strtail (opt, "mode=");
-        if (oarg) {
-                if (strcmp (oarg, "script") == 0) {
-                        state->mode |= GLUSTER_MODE_SCRIPT;
-                        return 0;
-                }
+    if (strcmp(opt, "wignore-partition") == 0) {
+        state->mode |= GLUSTER_MODE_WIGNORE_PARTITION;
+        return 0;
+    }
 
-                if (strcmp (oarg, "interactive") == 0)
-                        return 0;
+    if (strcmp(opt, "wignore") == 0) {
+        state->mode |= GLUSTER_MODE_WIGNORE;
+        return 0;
+    }
 
-                return -1;
+    oarg = strtail(opt, "mode=");
+    if (oarg) {
+        if (strcmp(oarg, "script") == 0) {
+            state->mode |= GLUSTER_MODE_SCRIPT;
+            return 0;
         }
 
-        oarg = strtail (opt, "remote-host=");
-        if (oarg) {
-                state->remote_host = oarg;
-                return 0;
-        }
+        if (strcmp(oarg, "interactive") == 0)
+            return 0;
 
-        oarg = strtail (opt, "log-file=");
-        if (oarg) {
-                state->log_file = oarg;
-                return 0;
-        }
+        return -1;
+    }
 
-        oarg = strtail (opt, "log-level=");
-        if (oarg) {
-                state->log_level = glusterd_check_log_level(oarg);
-                if (state->log_level == -1)
-                        return -1;
-                return 0;
-        }
+    oarg = strtail(opt, "remote-host=");
+    if (oarg) {
+        state->remote_host = oarg;
+        return 0;
+    }
 
-        oarg = strtail (opt, "glusterd-sock=");
-        if (oarg) {
-                state->glusterd_sock = oarg;
-                return 0;
+    oarg = strtail(opt, "inet6");
+    if (oarg) {
+        state->address_family = "inet6";
+        return 0;
+    }
+
+    oarg = strtail(opt, "log-file=");
+    if (oarg) {
+        state->log_file = oarg;
+        return 0;
+    }
+    oarg = strtail(opt, "timeout=");
+    if (oarg) {
+        if (!is_valid_int(oarg) || atoi(oarg) <= 0) {
+            cli_err("timeout value should be a positive integer");
+            return -2; /* -2 instead of -1 to avoid unknown option
+                          error */
         }
+        cli_default_conn_timeout = atoi(oarg);
+        return 0;
+    }
+
+    oarg = strtail(opt, "log-level=");
+    if (oarg) {
+        int log_level = glusterd_check_log_level(oarg);
+        if (log_level == -1)
+            return -1;
+        state->log_level = (gf_loglevel_t)log_level;
+        return 0;
+    }
 
-        oarg = strtail (opt, "secure-mgmt=");
-        if (oarg) {
-                if (gf_string2boolean(oarg,&secure_mgmt_tmp) == 0) {
-                        if (secure_mgmt_tmp) {
-                                /* See declaration for why this is an int. */
-                                state->ctx->secure_mgmt = 1;
-                        }
-                }
-                else {
-                        cli_err ("invalide secure-mgmt value (ignored)");
-                }
-                return 0;
+    oarg = strtail(opt, "glusterd-sock=");
+    if (oarg) {
+        state->glusterd_sock = oarg;
+        return 0;
+    }
+
+    oarg = strtail(opt, "secure-mgmt=");
+    if (oarg) {
+        if (gf_string2boolean(oarg, &secure_mgmt_tmp) == 0) {
+            if (secure_mgmt_tmp) {
+                /* See declaration for why this is an int. */
+                state->ctx->secure_mgmt = 1;
+            }
+        } else {
+            cli_err("invalid secure-mgmt value (ignored)");
         }
+        return 0;
+    }
 
-        return -1;
+    return -1;
 }
 
 int
-parse_cmdline (int argc, char *argv[], struct cli_state *state)
+parse_cmdline(int argc, char *argv[], struct cli_state *state)
 {
-        int                 ret            = 0;
-        int                 i              = 0;
-        int                 j              = 0;
-        char               *opt            = NULL;
-        gf_boolean_t       geo_rep_config  = _gf_false;
-
-        state->argc=argc-1;
-        state->argv=&argv[1];
-
-        /* Do this first so that an option can override. */
-        if (sys_access (SECURE_ACCESS_FILE, F_OK) == 0) {
-                state->ctx->secure_mgmt = 1;
+    int ret = 0;
+    int i = 0;
+    int j = 0;
+    char *opt = NULL;
+
+    state->argc = argc - 1;
+    state->argv = &argv[1];
+
+    /* Do this first so that an option can override. */
+    if (sys_access(SECURE_ACCESS_FILE, F_OK) == 0) {
+        state->ctx->secure_mgmt = 1;
+        state->ctx->ssl_cert_depth = glusterfs_read_secure_access_file();
+    }
+
+    if (state->argc > GEO_REP_CMD_CONFIG_INDEX &&
+        strtail(state->argv[GEO_REP_CMD_INDEX], "geo") &&
+        strtail(state->argv[GEO_REP_CMD_CONFIG_INDEX], "co"))
+        goto done;
+
+    for (i = 0; i < state->argc; i++) {
+        opt = strtail(state->argv[i], "--");
+        if (!opt)
+            continue;
+        ret = cli_opt_parse(opt, state);
+        if (ret == -1) {
+            cli_out("unrecognized option --%s\n", opt);
+            usage();
+            return ret;
+        } else if (ret == -2) {
+            return ret;
         }
-
-        if (state->argc > GEO_REP_CMD_CONFIG_INDEX &&
-            strtail (state->argv[GEO_REP_CMD_INDEX], "geo") &&
-            strtail (state->argv[GEO_REP_CMD_CONFIG_INDEX], "co"))
-                geo_rep_config = _gf_true;
-
-        for (i = 0; i < state->argc; i++) {
-                opt = strtail (state->argv[i], "--");
-                if (opt && !geo_rep_config) {
-                        ret = cli_opt_parse (opt, state);
-                        if (ret == -1) {
-                                cli_out ("unrecognized option --%s", opt);
-                                return ret;
-                        }
-                        for (j = i; j < state->argc - 1; j++)
-                                state->argv[j] = state->argv[j + 1];
-                        state->argc--;
-                        /* argv shifted, next check should be at i again */
-                        i--;
-                        if (ret == 1) {
-                                /* end of cli options */
-                                ret = 0;
-                                break;
-                        }
-                }
+        for (j = i; j < state->argc - 1; j++)
+            state->argv[j] = state->argv[j + 1];
+        state->argc--;
+        /* argv shifted, next check should be at i again */
+        i--;
+        if (ret == 1) {
+            /* end of cli options */
+            ret = 0;
+            break;
         }
+    }
 
-        state->argv[state->argc] = NULL;
+done:
+    state->argv[state->argc] = NULL;
 
-        return ret;
+    return ret;
 }
 
-
 int
-cli_cmd_tree_init (struct cli_cmd_tree *tree)
+cli_cmd_tree_init(struct cli_cmd_tree *tree)
 {
-        struct cli_cmd_word  *root = NULL;
-        int                   ret = 0;
+    struct cli_cmd_word *root = NULL;
+    int ret = 0;
 
-        root = &tree->root;
-        root->tree = tree;
+    root = &tree->root;
+    root->tree = tree;
 
-        return ret;
+    return ret;
 }
 
-
 int
-cli_state_init (struct cli_state *state)
+cli_state_init(struct cli_state *state)
 {
-        struct cli_cmd_tree  *tree = NULL;
-        int                   ret = 0;
+    struct cli_cmd_tree *tree = NULL;
+    int ret = 0;
 
+    state->log_level = GF_LOG_NONE;
 
-        state->log_level = GF_LOG_NONE;
+    tree = &state->tree;
+    tree->state = state;
 
-        tree = &state->tree;
-        tree->state = state;
+    ret = cli_cmd_tree_init(tree);
 
-        ret = cli_cmd_tree_init (tree);
-
-        return ret;
+    return ret;
 }
 
 int
-cli_usage_out (const char *usage)
+cli_usage_out(const char *usage)
 {
-        GF_ASSERT (usage);
-        GF_ASSERT (usage[0] != '\0');
+    GF_ASSERT(usage);
 
-        if (!usage || usage[0] == '\0')
-                return -1;
+    if (!usage || usage[0] == '\0')
+        return -1;
 
-        cli_err ("Usage: %s", usage);
-        return 0;
+    cli_err("\nUsage:\n%s\n", usage);
+    return 0;
 }
 
 int
-_cli_err (const char *fmt, ...)
+_cli_err(const char *fmt, ...)
 {
-        struct cli_state *state = NULL;
-        va_list           ap;
-        int               ret = 0;
-
-        state = global_state;
+    va_list ap;
+    int ret = 0;
+#ifdef HAVE_READLINE
+    struct cli_state *state = global_state;
+#endif
 
-        va_start (ap, fmt);
+    va_start(ap, fmt);
 
 #ifdef HAVE_READLINE
-        if (state->rl_enabled && !state->rl_processing)
-                return cli_rl_err(state, fmt, ap);
+    if (state->rl_enabled && !state->rl_processing) {
+        ret = cli_rl_err(state, fmt, ap);
+        va_end(ap);
+        return ret;
+    }
 #endif
 
-        ret = vfprintf (stderr, fmt, ap);
-        fprintf (stderr, "\n");
-        va_end (ap);
+    ret = vfprintf(stderr, fmt, ap);
+    fprintf(stderr, "\n");
+    va_end(ap);
 
-        return ret;
+    return ret;
 }
 
-
 int
-_cli_out (const char *fmt, ...)
+_cli_out(const char *fmt, ...)
 {
-        struct cli_state *state = NULL;
-        va_list           ap;
-        int               ret = 0;
-
-        state = global_state;
-
-        va_start (ap, fmt);
+    va_list ap;
+    int ret = 0;
+#ifdef HAVE_READLINE
+    struct cli_state *state = global_state;
+#endif
 
+    va_start(ap, fmt);
 #ifdef HAVE_READLINE
-        if (state->rl_enabled && !state->rl_processing)
-                return cli_rl_out(state, fmt, ap);
+    if (state->rl_enabled && !state->rl_processing) {
+        ret = cli_rl_out(state, fmt, ap);
+        va_end(ap);
+        return ret;
+    }
 #endif
 
-        ret = vprintf (fmt, ap);
-        printf ("\n");
-        va_end (ap);
+    ret = vprintf(fmt, ap);
+    printf("\n");
+    va_end(ap);
 
-        return ret;
+    return ret;
 }
 
 struct rpc_clnt *
-cli_quotad_clnt_rpc_init (void)
+cli_quotad_clnt_rpc_init(void)
 {
-        struct rpc_clnt *rpc = NULL;
-        dict_t          *rpc_opts = NULL;
-        int             ret = -1;
-
-        rpc_opts = dict_new ();
-        if (!rpc_opts) {
-                        ret = -1;
-                        goto out;
-                }
+    struct rpc_clnt *rpc = NULL;
+    dict_t *rpc_opts = NULL;
+    int ret = -1;
+
+    rpc_opts = dict_new();
+    if (!rpc_opts) {
+        ret = -1;
+        goto out;
+    }
+
+    ret = dict_set_str(rpc_opts, "transport.address-family", "unix");
+    if (ret)
+        goto out;
+
+    ret = dict_set_str(rpc_opts, "transport-type", "socket");
+    if (ret)
+        goto out;
+
+    ret = dict_set_str(rpc_opts, "transport.socket.connect-path",
+                       "/var/run/gluster/quotad.socket");
+    if (ret)
+        goto out;
+
+    rpc = cli_quotad_clnt_init(THIS, rpc_opts);
+    if (!rpc)
+        goto out;
+
+    global_quotad_rpc = rpc;
+out:
+    if (rpc_opts) {
+        dict_unref(rpc_opts);
+    }
+    return rpc;
+}
 
-        ret = dict_set_str (rpc_opts, "transport.address-family", "unix");
-        if (ret)
-                goto out;
+struct rpc_clnt *
+cli_rpc_init(struct cli_state *state)
+{
+    struct rpc_clnt *rpc = NULL;
+    dict_t *options = NULL;
+    int ret = -1;
+    int port = CLI_GLUSTERD_PORT;
+    xlator_t *this = NULL;
+#ifdef IPV6_DEFAULT
+    char *addr_family = "inet6";
+#else
+    char *addr_family = "inet";
+#endif
 
-        ret = dict_set_str (rpc_opts, "transport-type", "socket");
+    this = THIS;
+    cli_rpc_prog = &cli_prog;
+
+    options = dict_new();
+    if (!options)
+        goto out;
+
+    /* If address family specified in CLI */
+    if (state->address_family) {
+        addr_family = state->address_family;
+    }
+
+    /* Connect to glusterd using the specified method, giving preference
+     * to a unix socket connection.  If nothing is specified, connect to
+     * the default glusterd socket.
+     */
+    if (state->glusterd_sock) {
+        gf_log("cli", GF_LOG_INFO,
+               "Connecting to glusterd using "
+               "sockfile %s",
+               state->glusterd_sock);
+        ret = rpc_transport_unix_options_build(options, state->glusterd_sock,
+                                               0);
         if (ret)
-                goto out;
-
-        ret = dict_set_str (rpc_opts, "transport.socket.connect-path",
-                                            "/var/run/gluster/quotad.socket");
+            goto out;
+    } else if (state->remote_host) {
+        gf_log("cli", GF_LOG_INFO,
+               "Connecting to remote glusterd at "
+               "%s",
+               state->remote_host);
+
+        ret = dict_set_str(options, "remote-host", state->remote_host);
         if (ret)
-                goto out;
-
-        rpc = cli_quotad_clnt_init (THIS, rpc_opts);
-        if (!rpc)
-                goto out;
+            goto out;
 
-        global_quotad_rpc = rpc;
-out:
-        if (ret) {
-                if (rpc_opts)
-                        dict_destroy(rpc_opts);
-        }
-        return rpc;
-}
+        if (state->remote_port)
+            port = state->remote_port;
 
-struct rpc_clnt *
-cli_rpc_init (struct cli_state *state)
-{
-        struct rpc_clnt         *rpc = NULL;
-        dict_t                  *options = NULL;
-        int                     ret = -1;
-        int                     port = CLI_GLUSTERD_PORT;
-        xlator_t                *this = NULL;
-
-        this = THIS;
-        cli_rpc_prog = &cli_prog;
-        options = dict_new ();
-        if (!options)
-                goto out;
+        ret = dict_set_int32(options, "remote-port", port);
+        if (ret)
+            goto out;
 
-        /* Connect to glusterd using the specified method, giving preference
-         * to a unix socket connection.  If nothing is specified, connect to
-         * the default glusterd socket.
-         */
-        if (state->glusterd_sock) {
-                gf_log ("cli", GF_LOG_INFO, "Connecting to glusterd using "
-                        "sockfile %s", state->glusterd_sock);
-                ret = rpc_transport_unix_options_build (&options,
-                                                        state->glusterd_sock,
-                                                        0);
-                if (ret)
-                        goto out;
-        }
-        else if (state->remote_host) {
-                gf_log ("cli", GF_LOG_INFO, "Connecting to remote glusterd at "
-                        "%s", state->remote_host);
-                ret = dict_set_str (options, "remote-host", state->remote_host);
-                if (ret)
-                        goto out;
-
-                if (state->remote_port)
-                        port = state->remote_port;
-
-                ret = dict_set_int32 (options, "remote-port", port);
-                if (ret)
-                        goto out;
-
-                ret = dict_set_str (options, "transport.address-family",
-                                    "inet");
-                if (ret)
-                        goto out;
-        }
-        else {
-                gf_log ("cli", GF_LOG_DEBUG, "Connecting to glusterd using "
-                        "default socket");
-                ret = rpc_transport_unix_options_build
-                        (&options, DEFAULT_GLUSTERD_SOCKFILE, 0);
-                if (ret)
-                        goto out;
-        }
+        ret = dict_set_str(options, "transport.address-family", addr_family);
+        if (ret)
+            goto out;
+    } else {
+        gf_log("cli", GF_LOG_DEBUG,
+               "Connecting to glusterd using "
+               "default socket");
+        ret = rpc_transport_unix_options_build(options,
+                                               DEFAULT_GLUSTERD_SOCKFILE, 0);
+        if (ret)
+            goto out;
+    }
 
-        rpc = rpc_clnt_new (options, this, this->name, 16);
-        if (!rpc)
-                goto out;
+    rpc = rpc_clnt_new(options, this, this->name, 16);
+    if (!rpc)
+        goto out;
 
-        ret = rpc_clnt_register_notify (rpc, cli_rpc_notify, this);
-        if (ret) {
-                gf_log ("cli", GF_LOG_ERROR, "failed to register notify");
-                goto out;
-        }
+    ret = rpc_clnt_register_notify(rpc, cli_rpc_notify, this);
+    if (ret) {
+        gf_log("cli", GF_LOG_ERROR, "failed to register notify");
+        goto out;
+    }
 
-        ret = rpc_clnt_start (rpc);
+    ret = rpc_clnt_start(rpc);
 out:
-        if (ret) {
-                if (rpc)
-                        rpc_clnt_unref (rpc);
-                rpc = NULL;
-        }
-        return rpc;
+    if (options)
+        dict_unref(options);
+
+    if (ret) {
+        if (rpc)
+            rpc_clnt_unref(rpc);
+        rpc = NULL;
+    }
+    return rpc;
 }
 
 cli_local_t *
-cli_local_get ()
+cli_local_get()
 {
-        cli_local_t     *local = NULL;
+    cli_local_t *local = NULL;
 
-        local = GF_CALLOC (1, sizeof (*local), cli_mt_cli_local_t);
-        LOCK_INIT (&local->lock);
+    local = GF_CALLOC(1, sizeof(*local), cli_mt_cli_local_t);
+    LOCK_INIT(&local->lock);
+    INIT_LIST_HEAD(&local->dict_list);
 
-        return local;
+    return local;
 }
 
 void
-cli_local_wipe (cli_local_t *local)
+cli_local_wipe(cli_local_t *local)
 {
-        if (local) {
-                GF_FREE (local->get_vol.volname);
-                if (local->dict)
-                        dict_unref (local->dict);
-                GF_FREE (local);
-        }
-
-        return;
+    if (local) {
+        GF_FREE(local->get_vol.volname);
+        if (local->dict)
+            dict_unref(local->dict);
+        GF_FREE(local);
+    }
+
+    return;
 }
 
 struct cli_state *global_state;
 
 int
-main (int argc, char *argv[])
+main(int argc, char *argv[])
 {
-        struct cli_state   state = {0, };
-        int                ret = -1;
-        glusterfs_ctx_t   *ctx = NULL;
+    struct cli_state state = {
+        0,
+    };
+    int ret = -1;
+    glusterfs_ctx_t *ctx = NULL;
 
-        ctx = glusterfs_ctx_new ();
-        if (!ctx)
-                return ENOMEM;
+    mem_pools_init();
+
+    ctx = glusterfs_ctx_new();
+    if (!ctx)
+        return ENOMEM;
 
 #ifdef DEBUG
-        gf_mem_acct_enable_set (ctx);
+    gf_mem_acct_enable_set(ctx);
 #endif
 
-        ret = glusterfs_globals_init (ctx);
-        if (ret)
-                return ret;
+    ret = glusterfs_globals_init(ctx);
+    if (ret)
+        return ret;
 
-	THIS->ctx = ctx;
+    THIS->ctx = ctx;
 
-        ret = glusterfs_ctx_defaults_init (ctx);
-        if (ret)
-                goto out;
+    ret = glusterfs_ctx_defaults_init(ctx);
+    if (ret)
+        goto out;
 
-        ret = cli_state_init (&state);
-        if (ret)
-                goto out;
+    cli_default_conn_timeout = 120;
+    cli_ten_minutes_timeout = 600;
 
-        state.ctx = ctx;
-        global_state = &state;
+    ret = cli_state_init(&state);
+    if (ret)
+        goto out;
 
-        ret = parse_cmdline (argc, argv, &state);
-        if (ret)
-                goto out;
+    state.ctx = ctx;
+    global_state = &state;
 
-        ret = logging_init (ctx, &state);
-        if (ret)
-                goto out;
+    ret = parse_cmdline(argc, argv, &state);
+    if (ret)
+        goto out;
 
-        gf_log ("cli", GF_LOG_INFO, "Started running %s with version %s",
-                argv[0], PACKAGE_VERSION);
+    ret = logging_init(ctx, &state);
+    if (ret)
+        goto out;
 
-        global_rpc = cli_rpc_init (&state);
-        if (!global_rpc)
-                goto out;
+    gf_log("cli", GF_LOG_INFO, "Started running %s with version %s", argv[0],
+           PACKAGE_VERSION);
 
-        global_quotad_rpc = cli_quotad_clnt_rpc_init ();
-        if (!global_quotad_rpc)
-                goto out;
+    global_rpc = cli_rpc_init(&state);
+    if (!global_rpc)
+        goto out;
 
-        ret = cli_cmds_register (&state);
-        if (ret)
-                goto out;
+    global_quotad_rpc = cli_quotad_clnt_rpc_init();
+    if (!global_quotad_rpc)
+        goto out;
 
-        ret = cli_cmd_cond_init ();
-        if (ret)
-                goto out;
+    ret = cli_cmds_register(&state);
+    if (ret)
+        goto out;
 
-        ret = cli_input_init (&state);
-        if (ret)
-                goto out;
+    ret = cli_input_init(&state);
+    if (ret)
+        goto out;
 
-        ret = event_dispatch (ctx->event_pool);
+    ret = gf_event_dispatch(ctx->event_pool);
 
 out:
-//        glusterfs_ctx_destroy (ctx);
+    //        glusterfs_ctx_destroy (ctx);
 
-        return ret;
+    mem_pools_fini();
+
+    return ret;
 }
 
 void
-cli_print_line (int len)
+cli_print_line(int len)
 {
-        GF_ASSERT (len > 0);
+    GF_ASSERT(len > 0);
 
-        while (len--)
-                printf ("-");
+    while (len--)
+        printf("-");
 
-        printf ("\n");
+    printf("\n");
 }
 
 void
-print_quota_list_header (int type)
+print_quota_list_header(int type)
 {
-        if (type == GF_QUOTA_OPTION_TYPE_LIST) {
-                cli_out ("                  Path                   Hard-limit "
-                         " Soft-limit      Used  Available  Soft-limit "
-                         "exceeded? Hard-limit exceeded?");
-                cli_out ("-----------------------------------------------------"
-                         "-----------------------------------------------------"
-                         "---------------------");
-        } else {
-                cli_out ("                  Path                   Hard-limit  "
-                         " Soft-limit      Files       Dirs     Available  "
-                         "Soft-limit exceeded? Hard-limit exceeded?");
-                cli_out ("-----------------------------------------------------"
-                         "-----------------------------------------------------"
-                         "-------------------------------------");
-        }
+    if (type == GF_QUOTA_OPTION_TYPE_LIST) {
+        cli_out(
+            "                  Path                   Hard-limit "
+            " Soft-limit      Used  Available  Soft-limit "
+            "exceeded? Hard-limit exceeded?");
+        cli_out(
+            "-----------------------------------------------------"
+            "-----------------------------------------------------"
+            "---------------------");
+    } else {
+        cli_out(
+            "                  Path                   Hard-limit  "
+            " Soft-limit      Files       Dirs     Available  "
+            "Soft-limit exceeded? Hard-limit exceeded?");
+        cli_out(
+            "-----------------------------------------------------"
+            "-----------------------------------------------------"
+            "-------------------------------------");
+    }
 }
 
 void
-print_quota_list_empty (char *path, int type)
+print_quota_list_empty(char *path, int type)
 {
-        if (type == GF_QUOTA_OPTION_TYPE_LIST)
-                cli_out ("%-40s %7s %9s %10s %7s %15s %20s", path,
-                         "N/A", "N/A", "N/A", "N/A", "N/A", "N/A");
-        else
-                cli_out ("%-40s %9s %9s %12s %10s %10s %15s %20s", path,
-                         "N/A", "N/A", "N/A", "N/A", "N/A", "N/A", "N/A");
+    if (type == GF_QUOTA_OPTION_TYPE_LIST)
+        cli_out("%-40s %7s %9s %10s %7s %15s %20s", path, "N/A", "N/A", "N/A",
+                "N/A", "N/A", "N/A");
+    else
+        cli_out("%-40s %9s %9s %12s %10s %10s %15s %20s", path, "N/A", "N/A",
+                "N/A", "N/A", "N/A", "N/A", "N/A");
 }
-
diff --git a/cli/src/cli.h b/cli/src/cli.h
index 68cdb671628..c0d933e8f8a 100644
--- a/cli/src/cli.h
+++ b/cli/src/cli.h
@@ -11,483 +11,506 @@
 #define __CLI_H__
 
 #include "rpc-clnt.h"
-#include "glusterfs.h"
+#include <glusterfs/glusterfs.h>
 #include "protocol-common.h"
-#include "logging.h"
-#include "quota-common-utils.h"
+#include <glusterfs/logging.h>
+#include <glusterfs/quota-common-utils.h>
 
 #include "cli1-xdr.h"
+#include "gd-common-utils.h"
 
 #if (HAVE_LIB_XML)
 #include <libxml/encoding.h>
 #include <libxml/xmlwriter.h>
 #endif
 
-#define DEFAULT_EVENT_POOL_SIZE            16384
-#define CLI_GLUSTERD_PORT                  24007
-#define CLI_DEFAULT_CONN_TIMEOUT             120
-#define CLI_DEFAULT_CMD_TIMEOUT              120
-#define CLI_TEN_MINUTES_TIMEOUT              600 //Longer timeout for volume top
-#define DEFAULT_CLI_LOG_FILE_DIRECTORY     DATADIR "/log/glusterfs"
-#define CLI_VOL_STATUS_BRICK_LEN              43
-#define CLI_TAB_LENGTH                         8
-#define CLI_BRICK_STATUS_LINE_LEN             78
+#define DEFAULT_EVENT_POOL_SIZE 16384
+#define CLI_GLUSTERD_PORT 24007
+#define DEFAULT_CLI_LOG_FILE_DIRECTORY DATADIR "/log/glusterfs"
+#define CLI_VOL_STATUS_BRICK_LEN 43
+#define CLI_TAB_LENGTH 8
+#define CLI_BRICK_STATUS_LINE_LEN 78
 
 /* Geo-rep command positional arguments' index  */
-#define GEO_REP_CMD_INDEX                      1
-#define GEO_REP_CMD_CONFIG_INDEX               4
+#define GEO_REP_CMD_INDEX 1
+#define GEO_REP_CMD_CONFIG_INDEX 4
 
 enum argp_option_keys {
-	ARGP_DEBUG_KEY = 133,
-	ARGP_PORT_KEY = 'p',
+    ARGP_DEBUG_KEY = 133,
+    ARGP_PORT_KEY = 'p',
 };
 
+extern int cli_default_conn_timeout;
+extern int cli_ten_minutes_timeout;
+
 typedef enum {
-        COLD_BRICK_COUNT,
-        COLD_TYPE,
-        COLD_DIST_COUNT,
-        COLD_REPLICA_COUNT,
-        COLD_DISPERSE_COUNT,
-        COLD_REDUNDANCY_COUNT,
-        HOT_BRICK_COUNT,
-        HOT_TYPE,
-        HOT_REPLICA_COUNT,
-        MAX
+    COLD_BRICK_COUNT,
+    COLD_TYPE,
+    COLD_DIST_COUNT,
+    COLD_REPLICA_COUNT,
+    COLD_ARBITER_COUNT,
+    COLD_DISPERSE_COUNT,
+    COLD_REDUNDANCY_COUNT,
+    HOT_BRICK_COUNT,
+    HOT_TYPE,
+    HOT_REPLICA_COUNT,
+    MAX
 } values;
 
-#define GLUSTER_MODE_SCRIPT    (1 << 0)
+#define GLUSTER_MODE_SCRIPT (1 << 0)
 #define GLUSTER_MODE_ERR_FATAL (1 << 1)
-#define GLUSTER_MODE_XML       (1 << 2)
-#define GLUSTER_MODE_WIGNORE   (1 << 3)
-
+#define GLUSTER_MODE_XML (1 << 2)
+#define GLUSTER_MODE_WIGNORE (1 << 3)
+#define GLUSTER_MODE_WIGNORE_PARTITION (1 << 4)
+#define GLUSTER_MODE_GLFSHEAL_NOLOG (1 << 5)
 
-#define GLUSTERD_GET_QUOTA_AUX_MOUNT_PATH(abspath, volname, path)      \
-        snprintf (abspath, sizeof (abspath)-1,                          \
-                  DEFAULT_VAR_RUN_DIRECTORY"/%s%s", volname, path);
-
-#define GLUSTERFS_GET_AUX_MOUNT_PIDFILE(pidfile,volname) {               \
-                snprintf (pidfile, PATH_MAX-1,                             \
-                          DEFAULT_VAR_RUN_DIRECTORY"/%s.pid", volname);  \
-        }
+#define GLUSTERD_GET_QUOTA_LIST_MOUNT_PATH(abspath, volname, path)             \
+    do {                                                                       \
+        snprintf(abspath, sizeof(abspath) - 1,                                 \
+                 DEFAULT_VAR_RUN_DIRECTORY "/%s_quota_list%s", volname, path); \
+    } while (0)
 
 struct cli_state;
 struct cli_cmd_word;
 struct cli_cmd_tree;
 struct cli_cmd;
 
-extern char *cli_vol_type_str[];
 extern char *cli_vol_status_str[];
 extern char *cli_vol_task_status_str[];
 
-typedef int (cli_cmd_cbk_t)(struct cli_state *state,
-                            struct cli_cmd_word *word,
-                            const char **words,
-                            int wordcount);
-typedef void (cli_cmd_reg_cbk_t)( struct cli_cmd *this);
+typedef int(cli_cmd_cbk_t)(struct cli_state *state, struct cli_cmd_word *word,
+                           const char **words, int wordcount);
+typedef void(cli_cmd_reg_cbk_t)(struct cli_cmd *this);
 
-typedef int (cli_cmd_match_t)(struct cli_cmd_word *word);
-typedef int (cli_cmd_filler_t)(struct cli_cmd_word *word);
+typedef int(cli_cmd_match_t)(struct cli_cmd_word *word);
+typedef int(cli_cmd_filler_t)(struct cli_cmd_word *word);
 
 struct cli_cmd_word {
-        struct cli_cmd_tree   *tree;
-        const char            *word;
-        cli_cmd_filler_t      *filler;
-        cli_cmd_match_t       *match;
-        cli_cmd_cbk_t         *cbkfn;
-        const char            *desc;
-        const char            *pattern;
-        int                    nextwords_cnt;
-        struct cli_cmd_word  **nextwords;
+    struct cli_cmd_tree *tree;
+    const char *word;
+    cli_cmd_filler_t *filler;
+    cli_cmd_match_t *match;
+    cli_cmd_cbk_t *cbkfn;
+    const char *desc;
+    const char *pattern;
+    int nextwords_cnt;
+    struct cli_cmd_word **nextwords;
 };
 
-
 struct cli_cmd_tree {
-        struct cli_state      *state;
-        struct cli_cmd_word    root;
+    struct cli_state *state;
+    struct cli_cmd_word root;
 };
 
-
 struct cli_state {
-        int                   argc;
-        char                **argv;
+    int argc;
+    char **argv;
 
-        char                  debug;
+    char debug;
 
-        /* for events dispatching */
-        glusterfs_ctx_t      *ctx;
+    /* for events dispatching */
+    glusterfs_ctx_t *ctx;
 
-        /* registry of known commands */
-        struct cli_cmd_tree   tree;
+    /* registry of known commands */
+    struct cli_cmd_tree tree;
 
-        /* the thread which "executes" the command in non-interactive mode */
-        /* also the thread which reads from stdin in non-readline mode */
-        pthread_t             input;
+    /* the thread which "executes" the command in non-interactive mode */
+    /* also the thread which reads from stdin in non-readline mode */
+    pthread_t input;
 
-        /* terminal I/O */
-        const char           *prompt;
-        int                   rl_enabled;
-        int                   rl_async;
-        int                   rl_processing;
+    /* terminal I/O */
+    const char *prompt;
+    int rl_enabled;
+    int rl_async;
+    int rl_processing;
 
-        /* autocompletion state */
-        char                **matches;
-        char                **matchesp;
+    /* autocompletion state */
+    char **matches;
+    char **matchesp;
 
-        char                 *remote_host;
-        int                   remote_port;
-        int                   mode;
-        int                   await_connected;
+    char *remote_host;
+    int remote_port;
+    int mode;
+    int await_connected;
 
-        char                 *log_file;
-        gf_loglevel_t         log_level;
+    char *log_file;
+    gf_loglevel_t log_level;
 
-        char                 *glusterd_sock;
+    char *glusterd_sock;
+    char *address_family;
 };
 
 struct cli_local {
-        struct {
-                char    *volname;
-                int     flags;
-        } get_vol;
-
-        dict_t          *dict;
-        const char      **words;
-        /* Marker for volume status all */
-        gf_boolean_t    all;
+    struct {
+        char *volname;
+        int flags;
+    } get_vol;
+
+    dict_t *dict;
+    const char **words;
+    /* Marker for volume status all */
+    gf_boolean_t all;
 #if (HAVE_LIB_XML)
-        xmlTextWriterPtr        writer;
-        xmlDocPtr               doc;
-        int                     vol_count;
+    xmlTextWriterPtr writer;
+    xmlDocPtr doc;
+    int vol_count;
 #endif
-        gf_lock_t               lock;
+    gf_lock_t lock;
+    struct list_head dict_list;
 };
 
 struct cli_volume_status {
-        int            port;
-        int            rdma_port;
-        int            online;
-        uint64_t       block_size;
-        uint64_t       total_inodes;
-        uint64_t       free_inodes;
-        char          *brick;
-        char          *pid_str;
-        char          *free;
-        char          *total;
-#ifdef GF_LINUX_HOST_OS
-        char          *fs_name;
-        char          *mount_options;
-        char          *device;
-        char          *inode_size;
-#endif
+    int port;
+    int rdma_port;
+    int online;
+    uint64_t block_size;
+    uint64_t total_inodes;
+    uint64_t free_inodes;
+    char *brick;
+    char *pid_str;
+    char *free;
+    char *total;
+    char *fs_name;
+    char *mount_options;
+    char *device;
+    char *inode_size;
 };
 
 struct snap_config_opt_vals_ {
-        char           *op_name;
-        char           *question;
+    char *op_name;
+    char *question;
 };
 
 typedef struct cli_volume_status cli_volume_status_t;
 
 typedef struct cli_local cli_local_t;
 
-typedef ssize_t (*cli_serialize_t) (struct iovec outmsg, void *args);
+typedef ssize_t (*cli_serialize_t)(struct iovec outmsg, void *args);
 
 extern struct cli_state *global_state; /* use only in readline callback */
 
-typedef const char *(*cli_selector_t) (void *wcon);
+extern struct rpc_clnt *global_quotad_rpc;
 
-char *get_struct_variable (int mem_num, gf_gsync_status_t *sts_val);
+extern struct rpc_clnt *global_rpc;
 
-void *cli_getunamb (const char *tok, void **choices, cli_selector_t sel);
+extern rpc_clnt_prog_t *cli_rpc_prog;
 
-int cli_cmd_register (struct cli_cmd_tree *tree, struct cli_cmd *cmd);
-int cli_cmds_register (struct cli_state *state);
+typedef const char *(*cli_selector_t)(void *wcon);
 
-int cli_input_init (struct cli_state *state);
+char *
+get_struct_variable(int mem_num, gf_gsync_status_t *sts_val);
 
-int cli_cmd_process (struct cli_state *state, int argc, char *argv[]);
-int cli_cmd_process_line (struct cli_state *state, const char *line);
+void *
+cli_getunamb(const char *tok, void **choices, cli_selector_t sel);
 
-int cli_rl_enable (struct cli_state *state);
-int cli_rl_out (struct cli_state *state, const char *fmt, va_list ap);
-int cli_rl_err (struct cli_state *state, const char *fmt, va_list ap);
+int
+cli_cmd_register(struct cli_cmd_tree *tree, struct cli_cmd *cmd);
+int
+cli_cmds_register(struct cli_state *state);
 
-int cli_usage_out (const char *usage);
+int
+cli_input_init(struct cli_state *state);
 
-int _cli_out (const char *fmt, ...);
-int _cli_err (const char *fmt, ...);
+int
+cli_cmd_process(struct cli_state *state, int argc, char *argv[]);
+int
+cli_cmd_process_line(struct cli_state *state, const char *line);
 
-#define cli_out(fmt...) do {                       \
-                FMT_WARN (fmt);                    \
-                                                   \
-                _cli_out(fmt);                     \
-                                                   \
-        } while (0)
+int
+cli_rl_enable(struct cli_state *state);
+int
+cli_rl_out(struct cli_state *state, const char *fmt, va_list ap);
+int
+cli_rl_err(struct cli_state *state, const char *fmt, va_list ap);
 
-#define cli_err(fmt...) do {                       \
-                FMT_WARN (fmt);                    \
-                                                   \
-                _cli_err(fmt);                     \
-                                                   \
-        } while (0)
+int
+cli_usage_out(const char *usage);
 
 int
-cli_submit_request (struct rpc_clnt *rpc, void *req, call_frame_t *frame,
-                    rpc_clnt_prog_t *prog,
-                    int procnum, struct iobref *iobref,
-                    xlator_t *this, fop_cbk_fn_t cbkfn, xdrproc_t xdrproc);
+_cli_out(const char *fmt, ...);
+int
+_cli_err(const char *fmt, ...);
 
-int32_t
-cli_cmd_volume_create_parse (struct cli_state *state, const char **words,
-                             int wordcount, dict_t **options);
+#define cli_out(fmt...)                                                        \
+    do {                                                                       \
+        FMT_WARN(fmt);                                                         \
+                                                                               \
+        _cli_out(fmt);                                                         \
+                                                                               \
+    } while (0)
+
+#define cli_err(fmt...)                                                        \
+    do {                                                                       \
+        FMT_WARN(fmt);                                                         \
+                                                                               \
+        _cli_err(fmt);                                                         \
+                                                                               \
+    } while (0)
+
+#define usage()                                                                \
+    do {                                                                       \
+        cli_out(                                                               \
+            " Usage: gluster [options] <help> <peer>"                          \
+            " <pool> <volume>\n"                                               \
+            " Options:\n"                                                      \
+            " --help  Shows the help information\n"                            \
+            " --version  Shows the version\n"                                  \
+            " --print-logdir  Shows the log directory\n"                       \
+            " --print-statedumpdir Shows the state dump directory\n");         \
+                                                                               \
+    } while (0)
+
+int
+cli_submit_request(struct rpc_clnt *rpc, void *req, call_frame_t *frame,
+                   rpc_clnt_prog_t *prog, int procnum, struct iobref *iobref,
+                   xlator_t *this, fop_cbk_fn_t cbkfn, xdrproc_t xdrproc);
 
 int32_t
-cli_cmd_volume_reset_parse (const char **words, int wordcount, dict_t **opt);
+cli_cmd_volume_create_parse(struct cli_state *state, const char **words,
+                            int wordcount, dict_t **options, char **bricks);
 
 int32_t
-cli_cmd_gsync_set_parse (const char **words, int wordcount, dict_t **opt);
+cli_cmd_volume_reset_parse(const char **words, int wordcount, dict_t **opt);
 
 int32_t
-cli_cmd_quota_parse (const char **words, int wordcount, dict_t **opt);
+cli_cmd_gsync_set_parse(struct cli_state *state, const char **words,
+                        int wordcount, dict_t **opt, char **errstr);
 
 int32_t
-cli_cmd_inode_quota_parse (const char **words, int wordcount, dict_t **opt);
+cli_cmd_quota_parse(const char **words, int wordcount, dict_t **opt);
 
 int32_t
-cli_cmd_bitrot_parse (const char **words, int wordcount, dict_t **opt);
+cli_cmd_inode_quota_parse(const char **words, int wordcount, dict_t **opt);
 
 int32_t
-cli_cmd_volume_set_parse (struct cli_state *state, const char **words,
-                          int wordcount, dict_t **options, char **op_errstr);
+cli_cmd_bitrot_parse(const char **words, int wordcount, dict_t **opt);
+
 int32_t
-cli_cmd_ganesha_parse (struct cli_state *state, const char **words,
-                       int wordcount, dict_t **options, char **op_errstr);
+cli_cmd_volume_set_parse(struct cli_state *state, const char **words,
+                         int wordcount, dict_t **options, char **op_errstr);
 
 int32_t
-cli_cmd_volume_add_brick_parse (const char **words, int wordcount,
-                                dict_t **options, int *type);
+cli_cmd_ganesha_parse(struct cli_state *state, const char **words,
+                      int wordcount, dict_t **options, char **op_errstr);
 
 int32_t
-cli_cmd_volume_detach_tier_parse (const char **words, int wordcount,
-                                  dict_t **options, int *question);
+cli_cmd_get_state_parse(struct cli_state *state, const char **words,
+                        int wordcount, dict_t **options, char **op_errstr);
 
 int32_t
-cli_cmd_volume_tier_parse (const char **words, int wordcount,
-                           dict_t **options);
+cli_cmd_volume_add_brick_parse(struct cli_state *state, const char **words,
+                               int wordcount, dict_t **options, int *type);
 
 int32_t
-cli_cmd_volume_remove_brick_parse (const char **words, int wordcount,
-                                   dict_t **options, int *question);
+cli_cmd_volume_remove_brick_parse(struct cli_state *state, const char **words,
+                                  int wordcount, dict_t **options,
+                                  int *question, int *brick_count,
+                                  int32_t *command);
 
 int32_t
-cli_cmd_volume_replace_brick_parse (const char **words, int wordcount,
+cli_cmd_volume_replace_brick_parse(const char **words, int wordcount,
                                    dict_t **options);
 
 int32_t
-cli_cmd_log_rotate_parse (const char **words, int wordcount, dict_t **options);
+cli_cmd_volume_reset_brick_parse(const char **words, int wordcount,
+                                 dict_t **options);
+
+int32_t
+cli_cmd_log_rotate_parse(const char **words, int wordcount, dict_t **options);
 int32_t
-cli_cmd_log_locate_parse (const char **words, int wordcount, dict_t **options);
+cli_cmd_log_locate_parse(const char **words, int wordcount, dict_t **options);
 int32_t
-cli_cmd_log_filename_parse (const char **words, int wordcount, dict_t **options);
+cli_cmd_log_filename_parse(const char **words, int wordcount, dict_t **options);
 
 int32_t
-cli_cmd_volume_statedump_options_parse (const char **words, int wordcount,
-                                        dict_t **options);
+cli_cmd_volume_statedump_options_parse(const char **words, int wordcount,
+                                       dict_t **options);
 int32_t
-cli_cmd_volume_clrlks_opts_parse (const char **words, int wordcount,
-                                  dict_t **options);
+cli_cmd_volume_clrlks_opts_parse(const char **words, int wordcount,
+                                 dict_t **options);
 
-cli_local_t * cli_local_get ();
+cli_local_t *
+cli_local_get();
 
 void
-cli_local_wipe (cli_local_t *local);
+cli_local_wipe(cli_local_t *local);
+
+gf_boolean_t
+cli_cmd_connected();
 
 int32_t
-cli_cmd_await_connected ();
+cli_cmd_await_connected(unsigned timeout);
 
 int32_t
-cli_cmd_broadcast_connected ();
+cli_cmd_broadcast_connected(gf_boolean_t status);
 
 int
-cli_rpc_notify (struct rpc_clnt *rpc, void *mydata, rpc_clnt_event_t event,
-                void *data);
+cli_rpc_notify(struct rpc_clnt *rpc, void *mydata, rpc_clnt_event_t event,
+               void *data);
 
 int32_t
-cli_cmd_volume_profile_parse (const char **words, int wordcount,
-                              dict_t **options);
+cli_cmd_volume_profile_parse(const char **words, int wordcount,
+                             dict_t **options);
 int32_t
-cli_cmd_volume_top_parse (const char **words, int wordcount,
-                              dict_t **options);
+cli_cmd_volume_top_parse(const char **words, int wordcount, dict_t **options);
 
 int32_t
-cli_cmd_log_level_parse (const char **words, int wordcount,
-                         dict_t **options);
+cli_cmd_log_level_parse(const char **words, int wordcount, dict_t **options);
 
 int32_t
-cli_cmd_volume_status_parse (const char **words, int wordcount,
-                             dict_t **options);
+cli_cmd_volume_status_parse(const char **words, int wordcount,
+                            dict_t **options);
 
 int
-cli_cmd_volume_heal_options_parse (const char **words, int wordcount,
-                                   dict_t **options);
+cli_cmd_volume_heal_options_parse(const char **words, int wordcount,
+                                  dict_t **options);
 
 int
-cli_cmd_volume_defrag_parse (const char **words, int wordcount,
-                             dict_t **options);
+cli_cmd_volume_defrag_parse(const char **words, int wordcount,
+                            dict_t **options);
 
 int
-cli_print_brick_status (cli_volume_status_t *status);
+cli_print_brick_status(cli_volume_status_t *status);
 
 void
-cli_print_detailed_status (cli_volume_status_t *status);
+cli_print_detailed_status(cli_volume_status_t *status);
 
 int
-cli_get_detail_status (dict_t *dict, int i, cli_volume_status_t *status);
+cli_get_detail_status(dict_t *dict, int i, cli_volume_status_t *status);
 
 void
-cli_print_line (int len);
+cli_print_line(int len);
 
 int
-cli_xml_output_str (char *op, char *str, int op_ret, int op_errno,
-                    char *op_errstr);
+cli_xml_output_str(char *op, char *str, int op_ret, int op_errno,
+                   char *op_errstr);
 
 int
-cli_xml_output_dict (char *op, dict_t *dict, int op_ret, int op_errno,
-                     char *op_errstr);
+cli_xml_output_dict(char *op, dict_t *dict, int op_ret, int op_errno,
+                    char *op_errstr);
 
 int
-cli_xml_output_vol_top (dict_t *dict, int op_ret, int op_errno,
-                        char *op_errstr);
+cli_xml_output_vol_top(dict_t *dict, int op_ret, int op_errno, char *op_errstr);
 
 int
-cli_xml_output_vol_profile (dict_t *dict, int op_ret, int op_errno,
-                            char *op_errstr);
+cli_xml_output_vol_profile(dict_t *dict, int op_ret, int op_errno,
+                           char *op_errstr);
 
 int
-cli_xml_output_vol_status_begin (cli_local_t *local, int op_ret, int op_errno,
-                                 char *op_errstr);
+cli_xml_output_vol_status_begin(cli_local_t *local, int op_ret, int op_errno,
+                                char *op_errstr);
 
 int
-cli_xml_output_vol_status_end (cli_local_t *local);
+cli_xml_output_vol_status_end(cli_local_t *local);
 
 int
-cli_xml_output_vol_status (cli_local_t *local, dict_t *dict);
+cli_xml_output_vol_status(cli_local_t *local, dict_t *dict);
 
 int
-cli_xml_output_vol_list (dict_t *dict, int op_ret, int op_errno,
-                         char *op_errstr);
+cli_xml_output_vol_list(dict_t *dict, int op_ret, int op_errno,
+                        char *op_errstr);
 
 int
-cli_xml_output_vol_info_begin (cli_local_t *local, int op_ret, int op_errno,
-                               char *op_errstr);
+cli_xml_output_vol_info_begin(cli_local_t *local, int op_ret, int op_errno,
+                              char *op_errstr);
 
 int
-cli_xml_output_vol_info_end (cli_local_t *local);
+cli_xml_output_vol_info_end(cli_local_t *local);
 
 int
-cli_xml_output_vol_info (cli_local_t *local, dict_t *dict);
+cli_xml_output_vol_info(cli_local_t *local, dict_t *dict);
 
 int
-cli_xml_output_vol_quota_limit_list_begin (cli_local_t *local, int op_ret,
-                                           int op_errno, char *op_errstr);
+cli_xml_output_vol_quota_limit_list_begin(cli_local_t *local, int op_ret,
+                                          int op_errno, char *op_errstr);
 int
-cli_xml_output_vol_quota_limit_list_end (cli_local_t *local);
+cli_xml_output_vol_quota_limit_list_end(cli_local_t *local);
 
 int
-cli_quota_list_xml_error (cli_local_t *local, char *path,
-                          char *errstr);
+cli_quota_list_xml_error(cli_local_t *local, char *path, char *errstr);
 
 int
-cli_quota_xml_output (cli_local_t *local, char *path, int64_t hl_str,
-                      char *sl_final, int64_t sl_num, int64_t used,
-                      int64_t avail, char *sl, char *hl,
-                      gf_boolean_t limit_set);
+cli_quota_xml_output(cli_local_t *local, char *path, int64_t hl_str,
+                     char *sl_final, int64_t sl_num, int64_t used,
+                     int64_t avail, char *sl, char *hl, gf_boolean_t limit_set);
 
 int
-cli_quota_object_xml_output (cli_local_t *local, char *path, char *sl_str,
-                             int64_t sl_val, quota_limits_t *limits,
-                             quota_meta_t *used_space, int64_t avail,
-                             char *sl, char *hl, gf_boolean_t limit_set);
+cli_quota_object_xml_output(cli_local_t *local, char *path, char *sl_str,
+                            int64_t sl_val, quota_limits_t *limits,
+                            quota_meta_t *used_space, int64_t avail, char *sl,
+                            char *hl, gf_boolean_t limit_set);
 
 int
-cli_xml_output_peer_status (dict_t *dict, int op_ret, int op_errno,
-                            char *op_errstr);
+cli_xml_output_peer_status(dict_t *dict, int op_ret, int op_errno,
+                           char *op_errstr);
 
 int
-cli_xml_output_vol_rebalance (gf_cli_defrag_type op, dict_t *dict, int op_ret,
-                              int op_errno, char *op_errstr);
+cli_xml_output_vol_rebalance(gf_cli_defrag_type op, dict_t *dict, int op_ret,
+                             int op_errno, char *op_errstr);
 
 int
-cli_xml_output_vol_remove_brick_detach_tier (gf_boolean_t status_op,
-                                             dict_t *dict, int op_ret,
-                                             int op_errno, char *op_errstr,
-                                             const char *op);
+cli_xml_output_vol_remove_brick(gf_boolean_t status_op, dict_t *dict,
+                                int op_ret, int op_errno, char *op_errstr,
+                                const char *op);
 
 int
-cli_xml_output_vol_replace_brick (char *op, dict_t *dict, int op_ret,
-                                  int op_errno, char *op_errstr);
+cli_xml_output_vol_replace_brick(dict_t *dict, int op_ret, int op_errno,
+                                 char *op_errstr);
 
 int
-cli_xml_output_vol_create (dict_t *dict, int op_ret, int op_errno,
-                           char *op_errstr);
+cli_xml_output_vol_create(dict_t *dict, int op_ret, int op_errno,
+                          char *op_errstr);
 
 int
-cli_xml_output_generic_volume (char *op, dict_t *dict, int op_ret, int op_errno,
-                               char *op_errstr);
+cli_xml_output_generic_volume(char *op, dict_t *dict, int op_ret, int op_errno,
+                              char *op_errstr);
 
 int
-cli_xml_output_vol_gsync (dict_t *dict, int op_ret, int op_errno,
-                          char *op_errstr);
+cli_xml_output_vol_gsync(dict_t *dict, int op_ret, int op_errno,
+                         char *op_errstr);
 int
-cli_xml_output_vol_status_tasks_detail (cli_local_t *local, dict_t *dict);
+cli_xml_output_vol_status_tasks_detail(cli_local_t *local, dict_t *dict);
 
 int
-cli_xml_output_common (xmlTextWriterPtr writer, int op_ret, int op_errno,
-                       char *op_errstr);
-int
-cli_xml_snapshot_delete (xmlTextWriterPtr writer, xmlDocPtr doc, dict_t *dict,
-                        gf_cli_rsp *rsp);
+cli_xml_snapshot_delete(cli_local_t *local, dict_t *dict, gf_cli_rsp *rsp);
+
 int
-cli_xml_snapshot_begin_composite_op (cli_local_t *local);
+cli_xml_snapshot_begin_composite_op(cli_local_t *local);
 
 int
-cli_xml_snapshot_end_composite_op (cli_local_t *local);
+cli_xml_snapshot_end_composite_op(cli_local_t *local);
 
 int
-cli_xml_output_snap_delete_begin (cli_local_t *local, int op_ret, int op_errno,
-                                  char *op_errstr);
+cli_xml_output_snap_delete_begin(cli_local_t *local, int op_ret, int op_errno,
+                                 char *op_errstr);
 int
-cli_xml_output_snap_delete_end (cli_local_t *local);
+cli_xml_output_snap_delete_end(cli_local_t *local);
 
 int
-cli_xml_output_snap_status_begin (cli_local_t *local, int op_ret, int op_errno,
-                                  char *op_errstr);
+cli_xml_output_snap_status_begin(cli_local_t *local, int op_ret, int op_errno,
+                                 char *op_errstr);
 int
-cli_xml_output_snap_status_end (cli_local_t *local);
+cli_xml_output_snap_status_end(cli_local_t *local);
 int
-cli_xml_output_snapshot (int cmd_type, dict_t *dict, int op_ret,
-                         int op_errno, char *op_errstr);
+cli_xml_output_snapshot(int cmd_type, dict_t *dict, int op_ret, int op_errno,
+                        char *op_errstr);
 int
-cli_xml_snapshot_status_single_snap (cli_local_t *local, dict_t *dict,
-                                     char *key);
-char *
-is_server_debug_xlator (void *myframe);
-
+cli_xml_snapshot_status_single_snap(cli_local_t *local, dict_t *dict,
+                                    char *key);
 int32_t
-cli_cmd_snapshot_parse (const char **words, int wordcount, dict_t **options,
-                        struct cli_state *state);
+cli_cmd_snapshot_parse(const char **words, int wordcount, dict_t **options,
+                       struct cli_state *state);
 
 int
-cli_xml_output_vol_getopts (dict_t *dict, int op_ret, int op_errno,
-                             char *op_errstr);
+cli_xml_output_vol_getopts(dict_t *dict, int op_ret, int op_errno,
+                           char *op_errstr);
 
 void
-print_quota_list_header (int type);
+print_quota_list_header(int type);
 
 void
-print_quota_list_empty (char *path, int type);
+print_quota_list_empty(char *path, int type);
 
 int
-gf_gsync_status_t_comparator (const void *p, const void *q);
-
+gf_gsync_status_t_comparator(const void *p, const void *q);
 #endif /* __CLI_H__ */
diff --git a/cli/src/input.c b/cli/src/input.c
index 10fc39cd6b4..5ac1a20edb1 100644
--- a/cli/src/input.c
+++ b/cli/src/input.c
@@ -19,74 +19,75 @@
 #define CMDBUFSIZ 1024
 
 void *
-cli_batch (void *d)
+cli_batch(void *d)
 {
-        struct cli_state *state = NULL;
-        int               ret = 0;
+    struct cli_state *state = NULL;
+    int ret = 0;
 
-        state = d;
+    state = d;
 
-        ret = cli_cmd_process (state, state->argc, state->argv);
+    ret = cli_cmd_process(state, state->argc, state->argv);
 
-        gf_log ("", GF_LOG_INFO, "Exiting with: %d", ret);
-        exit (-ret);
+    gf_log("", GF_LOG_INFO, "Exiting with: %d", ret);
+    exit(-ret);
 
-        return NULL;
+    return NULL;
 }
 
-
 void *
-cli_input (void *d)
+cli_input(void *d)
 {
-        struct cli_state *state = NULL;
-        int               ret = 0;
-        char              cmdbuf[CMDBUFSIZ];
-        char             *cmd = NULL;
-        size_t            len = 0;
-
-        state = d;
-
-        for (;;) {
-                printf ("%s", state->prompt);
-
-                cmd = fgets (cmdbuf, CMDBUFSIZ, stdin);
-                if (!cmd)
-                        break;
-                len = strlen(cmd);
-                if (len > 0 && cmd[len - 1] == '\n') //strip trailing \n
-                        cmd[len - 1] = '\0';
-                ret = cli_cmd_process_line (state, cmd);
-                if (ret != 0 && state->mode & GLUSTER_MODE_ERR_FATAL)
-                        break;
-        }
-
-        exit (-ret);
-
-        return NULL;
+    struct cli_state *state = NULL;
+    int ret = 0;
+    char cmdbuf[CMDBUFSIZ];
+    char *cmd = NULL;
+    size_t len = 0;
+
+    state = d;
+
+    fprintf(stderr,
+            "Welcome to gluster prompt, type 'help' to see the available "
+            "commands.\n");
+    for (;;) {
+        printf("%s", state->prompt);
+
+        cmd = fgets(cmdbuf, CMDBUFSIZ, stdin);
+        if (!cmd)
+            break;
+        len = strlen(cmd);
+        if (len > 0 && cmd[len - 1] == '\n')  // strip trailing \n
+            cmd[len - 1] = '\0';
+        ret = cli_cmd_process_line(state, cmd);
+        if (ret != 0 && state->mode & GLUSTER_MODE_ERR_FATAL)
+            break;
+    }
+
+    exit(-ret);
+
+    return NULL;
 }
 
-
 int
-cli_input_init (struct cli_state *state)
+cli_input_init(struct cli_state *state)
 {
-        int  ret = 0;
+    int ret = 0;
 
-        if (state->argc) {
-                ret = pthread_create (&state->input, NULL, cli_batch, state);
-                return ret;
-        }
+    if (state->argc) {
+        ret = pthread_create(&state->input, NULL, cli_batch, state);
+        return ret;
+    }
 
-        if (isatty (STDIN_FILENO)) {
-                state->prompt = "gluster> ";
+    if (isatty(STDIN_FILENO)) {
+        state->prompt = "gluster> ";
 
-                cli_rl_enable (state);
-        } else {
-                state->prompt = "";
-                state->mode |= GLUSTER_MODE_SCRIPT | GLUSTER_MODE_ERR_FATAL;
-        }
+        cli_rl_enable(state);
+    } else {
+        state->prompt = "";
+        state->mode |= GLUSTER_MODE_SCRIPT | GLUSTER_MODE_ERR_FATAL;
+    }
 
-        if (!state->rl_enabled)
-                ret = pthread_create (&state->input, NULL, cli_input, state);
+    if (!state->rl_enabled)
+        ret = pthread_create(&state->input, NULL, cli_input, state);
 
-        return ret;
+    return ret;
 }
diff --git a/cli/src/registry.c b/cli/src/registry.c
index 1adf7d6e572..85f7686ade1 100644
--- a/cli/src/registry.c
+++ b/cli/src/registry.c
@@ -15,20 +15,18 @@
 #include "cli.h"
 #include "cli-cmd.h"
 
-
 static int
-__is_spc (int ch)
+__is_spc(int ch)
 {
-        if (ch == ' ')
-                return 1;
-        return 0;
+    if (ch == ' ')
+        return 1;
+    return 0;
 }
 
-
 static int
-__is_div (int ch)
+__is_div(int ch)
 {
-        switch (ch) {
+    switch (ch) {
         case '(':
         case ')':
         case '<':
@@ -38,369 +36,356 @@ __is_div (int ch)
         case '{':
         case '}':
         case '|':
-                return 1;
-        }
+            return 1;
+    }
 
-        return 0;
+    return 0;
 }
 
-
 static int
-__is_word (const char *word)
+__is_word(const char *word)
 {
-        return (!__is_div (*word) && !__is_spc (*word));
+    return (!__is_div(*word) && !__is_spc(*word));
 }
 
-
 int
-counter_char (int ch)
+counter_char(int ch)
 {
-        switch (ch) {
+    switch (ch) {
         case '(':
-                return ')';
+            return ')';
         case '<':
-                return '>';
+            return '>';
         case '[':
-                return ']';
+            return ']';
         case '{':
-                return '}';
-        }
+            return '}';
+    }
 
-        return -1;
+    return -1;
 }
 
-
 const char *
-__is_template_balanced (const char *template)
+__is_template_balanced(const char *template)
 {
-        const char *trav = NULL;
-        int   ch = 0;
-
-        trav = template;
-
-        while (*trav) {
-                ch = *trav;
-
-                switch (ch) {
-                case '<':
-                case '(':
-                case '[':
-                        trav = __is_template_balanced (trav+1);
-                        if (!trav)
-                                return NULL;
-                        if (*trav != counter_char (ch))
-                                return NULL;
-                        break;
-                case '>':
-                case ')':
-                case ']':
-                        return trav;
-                }
+    const char *trav = NULL;
+    int ch = 0;
 
-                trav++;
+    trav = template;
+
+    while (*trav) {
+        ch = *trav;
+
+        switch (ch) {
+            case '<':
+            case '(':
+            case '[':
+                trav = __is_template_balanced(trav + 1);
+                if (!trav)
+                    return NULL;
+                if (*trav != counter_char(ch))
+                    return NULL;
+                break;
+            case '>':
+            case ')':
+            case ']':
+                return trav;
         }
 
-        return trav;
-}
+        trav++;
+    }
 
+    return trav;
+}
 
 int
-is_template_balanced (const char *template)
+is_template_balanced(const char *template)
 {
-        const char *trav = NULL;
+    const char *trav = NULL;
 
-        trav = __is_template_balanced (template);
-        if (!trav || *trav)
-                return -1;
+    trav = __is_template_balanced(template);
+    if (!trav || *trav)
+        return -1;
 
-        return 0;
+    return 0;
 }
 
-
 int
-cli_cmd_token_count (const char *template)
+cli_cmd_token_count(const char *template)
 {
-        int         count = 0;
-        const char *trav = NULL;
-        int         is_alnum = 0;
-
-        for (trav = template; *trav; trav++) {
-                switch (*trav) {
-                case '<':
-                case '>':
-                case '(':
-                case ')':
-                case '[':
-                case ']':
-                case '{':
-                case '}':
-                case '|':
-                        count++;
-                        /* fall through */
-                case ' ':
-                        is_alnum = 0;
-                        break;
-                default:
-                        if (!is_alnum) {
-                                is_alnum = 1;
-                                count++;
-                        }
+    int count = 0;
+    const char *trav = NULL;
+    int is_alnum = 0;
+
+    for (trav = template; *trav; trav++) {
+        switch (*trav) {
+            case '<':
+            case '>':
+            case '(':
+            case ')':
+            case '[':
+            case ']':
+            case '{':
+            case '}':
+            case '|':
+                count++;
+                /* fall through */
+            case ' ':
+                is_alnum = 0;
+                break;
+            default:
+                if (!is_alnum) {
+                    is_alnum = 1;
+                    count++;
                 }
         }
+    }
 
-        return count + 1;
+    return count + 1;
 }
 
-
 void
-cli_cmd_tokens_destroy (char **tokens)
+cli_cmd_tokens_destroy(char **tokens)
 {
-        char **tokenp = NULL;
+    char **tokenp = NULL;
 
-        if (!tokens)
-                return;
+    if (!tokens)
+        return;
 
-        tokenp = tokens;
-        while (*tokenp) {
-                free (*tokenp);
-                tokenp++;
-        }
+    tokenp = tokens;
+    while (*tokenp) {
+        free(*tokenp);
+        tokenp++;
+    }
 
-        free (tokens);
+    free(tokens);
 }
 
-
 int
-cli_cmd_tokens_fill (char **tokens, const char *template)
+cli_cmd_tokens_fill(char **tokens, const char *template)
 {
-        const char  *trav = NULL;
-        char       **tokenp = NULL;
-        char        *token = NULL;
-        int          ret = 0;
-        int          ch = 0;
+    const char *trav = NULL;
+    char **tokenp = NULL;
+    char *token = NULL;
+    int ret = 0;
+    int ch = 0;
 
-        tokenp = tokens;
+    tokenp = tokens;
 
-        for (trav = template; *trav; trav++) {
-                ch = *trav;
+    for (trav = template; *trav; trav++) {
+        ch = *trav;
 
-                if (__is_spc (ch))
-                        continue;
+        if (__is_spc(ch))
+            continue;
 
-                if (__is_div (ch)) {
-                        token = calloc (2, 1);
-                        if (!token)
-                                return -1;
-                        token[0] = ch;
+        if (__is_div(ch)) {
+            token = calloc(2, 1);
+            if (!token)
+                return -1;
+            token[0] = ch;
 
-                        *tokenp = token;
-                        tokenp++;
+            *tokenp = token;
+            tokenp++;
 
-                        continue;
-                }
+            continue;
+        }
 
-                token = strdup (trav);
-                *tokenp = token;
-                tokenp++;
+        token = strdup(trav);
+        *tokenp = token;
+        tokenp++;
 
-                for (token++; *token; token++) {
-                        if (__is_spc (*token) || __is_div (*token)) {
-                                *token = 0;
-                                break;
-                        }
-                        trav++;
-                }
+        for (token++; *token; token++) {
+            if (__is_spc(*token) || __is_div(*token)) {
+                *token = 0;
+                break;
+            }
+            trav++;
         }
+    }
 
-        return ret;
+    return ret;
 }
 
-
 char **
-cli_cmd_tokenize (const char *template)
+cli_cmd_tokenize(const char *template)
 {
-        char **tokens = NULL;
-        int    ret = 0;
-        int    count = 0;
+    char **tokens = NULL;
+    int ret = 0;
+    int count = 0;
 
-        ret = is_template_balanced (template);
-        if (ret)
-                return NULL;
+    ret = is_template_balanced(template);
+    if (ret)
+        return NULL;
 
-        count = cli_cmd_token_count (template);
-        if (count <= 0)
-                return NULL;
+    count = cli_cmd_token_count(template);
+    if (count <= 0)
+        return NULL;
 
-        tokens = calloc (count + 1, sizeof (char *));
-        if (!tokens)
-                return NULL;
+    tokens = calloc(count + 1, sizeof(char *));
+    if (!tokens)
+        return NULL;
 
-        ret = cli_cmd_tokens_fill (tokens, template);
-        if (ret)
-                goto err;
+    ret = cli_cmd_tokens_fill(tokens, template);
+    if (ret)
+        goto err;
 
-        return tokens;
+    return tokens;
 err:
-        cli_cmd_tokens_destroy (tokens);
-        return NULL;
+    cli_cmd_tokens_destroy(tokens);
+    return NULL;
 }
 
 void *
-cli_getunamb (const char *tok, void **choices, cli_selector_t sel)
+cli_getunamb(const char *tok, void **choices, cli_selector_t sel)
 {
-        void  **wcon = NULL;
-        char      *w = NULL;
-        unsigned  mn = 0;
-        void    *ret = NULL;
-
-        if (!choices || !tok || !*tok)
-                return NULL;
-
-        for (wcon = choices; *wcon; wcon++) {
-                w = strtail ((char *)sel (*wcon), tok);
-                if (!w)
-                        /* no match */
-                        continue;
-                if (!*w)
-                        /* exact match */
-                        return *wcon;
-
-                ret = *wcon;
-                mn++;
-        }
+    void **wcon = NULL;
+    char *w = NULL;
+    unsigned mn = 0;
+    void *ret = NULL;
 
-#ifdef FORCE_MATCH_EXACT
+    if (!choices || !tok || !*tok)
         return NULL;
+
+    for (wcon = choices; *wcon; wcon++) {
+        w = strtail((char *)sel(*wcon), tok);
+        if (!w)
+            /* no match */
+            continue;
+        if (!*w)
+            /* exact match */
+            return *wcon;
+
+        ret = *wcon;
+        mn++;
+    }
+
+#ifdef FORCE_MATCH_EXACT
+    return NULL;
 #else
-        return (mn == 1) ? ret : NULL;
+    return (mn == 1) ? ret : NULL;
 #endif
 }
 
 static const char *
-sel_cmd_word (void *wcon)
+sel_cmd_word(void *wcon)
 {
-        return ((struct cli_cmd_word *)wcon)->word;
+    return ((struct cli_cmd_word *)wcon)->word;
 }
 
 struct cli_cmd_word *
-cli_cmd_nextword (struct cli_cmd_word *word, const char *token)
+cli_cmd_nextword(struct cli_cmd_word *word, const char *token)
 {
-        return (struct cli_cmd_word *)cli_getunamb (token,
-                                                    (void **)word->nextwords,
-                                                    sel_cmd_word);
+    return (struct cli_cmd_word *)cli_getunamb(token, (void **)word->nextwords,
+                                               sel_cmd_word);
 }
 
-
 struct cli_cmd_word *
-cli_cmd_newword (struct cli_cmd_word *word, const char *token)
+cli_cmd_newword(struct cli_cmd_word *word, const char *token)
 {
-        struct cli_cmd_word **nextwords = NULL;
-        struct cli_cmd_word  *nextword = NULL;
+    struct cli_cmd_word **nextwords = NULL;
+    struct cli_cmd_word *nextword = NULL;
 
-        nextwords = realloc (word->nextwords,
-                             (word->nextwords_cnt + 2) * sizeof (*nextwords));
-        if (!nextwords)
-                return NULL;
+    nextwords = realloc(word->nextwords,
+                        (word->nextwords_cnt + 2) * sizeof(*nextwords));
+    if (!nextwords)
+        return NULL;
 
-        word->nextwords = nextwords;
+    word->nextwords = nextwords;
 
-        nextword = calloc (1, sizeof (*nextword));
-        if (!nextword)
-                return NULL;
+    nextword = calloc(1, sizeof(*nextword));
+    if (!nextword)
+        return NULL;
 
-        nextword->word = strdup (token);
-        if (!nextword->word) {
-                free (nextword);
-                return NULL;
-        }
+    nextword->word = strdup(token);
+    if (!nextword->word) {
+        free(nextword);
+        return NULL;
+    }
 
-        nextword->tree = word->tree;
-        nextwords[word->nextwords_cnt++] = nextword;
-        nextwords[word->nextwords_cnt] = NULL;
+    nextword->tree = word->tree;
+    nextwords[word->nextwords_cnt++] = nextword;
+    nextwords[word->nextwords_cnt] = NULL;
 
-        return nextword;
+    return nextword;
 }
 
-
 int
-cli_cmd_ingest (struct cli_cmd_tree *tree, char **tokens, cli_cmd_cbk_t *cbkfn,
-                const char *desc, const char *pattern)
+cli_cmd_ingest(struct cli_cmd_tree *tree, char **tokens, cli_cmd_cbk_t *cbkfn,
+               const char *desc, const char *pattern)
 {
-        int                    ret = 0;
-        char                 **tokenp = NULL;
-        char                  *token = NULL;
-        struct cli_cmd_word   *word = NULL;
-        struct cli_cmd_word   *next = NULL;
-
-        word = &tree->root;
+    int ret = 0;
+    char **tokenp = NULL;
+    char *token = NULL;
+    struct cli_cmd_word *word = NULL;
+    struct cli_cmd_word *next = NULL;
 
-        for (tokenp = tokens; (token = *tokenp); tokenp++) {
-                if (!__is_word (token))
-                        break;
+    word = &tree->root;
 
-                next = cli_cmd_nextword (word, token);
-                if (!next)
-                        next = cli_cmd_newword (word, token);
+    for (tokenp = tokens; (token = *tokenp); tokenp++) {
+        if (!__is_word(token))
+            break;
 
-                word = next;
-                if (!word)
-                        break;
-        }
+        next = cli_cmd_nextword(word, token);
+        if (!next)
+            next = cli_cmd_newword(word, token);
 
+        word = next;
         if (!word)
-                return -1;
+            break;
+    }
 
-        if (word->cbkfn) {
-                /* warning - command already registered */
-        }
+    if (!word)
+        return -1;
 
-        word->cbkfn = cbkfn;
-        word->desc  = desc;
-        word->pattern = pattern;
+    if (word->cbkfn) {
+        /* warning - command already registered */
+    }
 
-        /* end of static strings in command template */
+    word->cbkfn = cbkfn;
+    word->desc = desc;
+    word->pattern = pattern;
 
-        /* TODO: autocompletion beyond this point is just "nice to have" */
+    /* end of static strings in command template */
 
-        return ret;
-}
+    /* TODO: autocompletion beyond this point is just "nice to have" */
 
+    return ret;
+}
 
 int
-cli_cmd_register (struct cli_cmd_tree *tree, struct cli_cmd *cmd)
+cli_cmd_register(struct cli_cmd_tree *tree, struct cli_cmd *cmd)
 {
-        char **tokens = NULL;
-        int    ret = 0;
+    char **tokens = NULL;
+    int ret = 0;
 
-        GF_ASSERT (cmd);
+    GF_ASSERT(cmd);
 
-        if (cmd->reg_cbk)
-                cmd->reg_cbk (cmd);
+    if (cmd->reg_cbk)
+        cmd->reg_cbk(cmd);
 
-        if (cmd->disable) {
-                ret = 0;
-                goto out;
-        }
+    if (cmd->disable) {
+        ret = 0;
+        goto out;
+    }
 
-        tokens = cli_cmd_tokenize (cmd->pattern);
-        if (!tokens) {
-                ret = -1;
-                goto out;
-        }
+    tokens = cli_cmd_tokenize(cmd->pattern);
+    if (!tokens) {
+        ret = -1;
+        goto out;
+    }
 
-        ret = cli_cmd_ingest (tree, tokens, cmd->cbk, cmd->desc, cmd->pattern);
-        if (ret) {
-                ret = -1;
-                goto out;
-        }
+    ret = cli_cmd_ingest(tree, tokens, cmd->cbk, cmd->desc, cmd->pattern);
+    if (ret) {
+        ret = -1;
+        goto out;
+    }
 
-        ret = 0;
+    ret = 0;
 
 out:
-        if (tokens)
-                cli_cmd_tokens_destroy (tokens);
+    if (tokens)
+        cli_cmd_tokens_destroy(tokens);
 
-        gf_log ("cli", GF_LOG_DEBUG, "Returning %d", ret);
-        return ret;
+    gf_log("cli", GF_LOG_DEBUG, "Returning %d", ret);
+    return ret;
 }
-
diff --git a/configure.ac b/configure.ac
index 5c89394f91d..e2d6fd66cec 100644
--- a/configure.ac
+++ b/configure.ac
@@ -1,4 +1,4 @@
-dnl  Copyright (c) 2006-2012 Red Hat, Inc. <http://www.redhat.com>
+dnl  Copyright (c) 2006-2016 Red Hat, Inc. <http://www.redhat.com>
 dnl  This file is part of GlusterFS.
 dnl
 dnl  This file is licensed to you under your choice of the GNU Lesser
@@ -13,7 +13,7 @@ AC_INIT([glusterfs],
 AC_SUBST([PACKAGE_RELEASE],
          [m4_esyscmd([build-aux/pkg-version --release])])
 
-AM_INIT_AUTOMAKE(tar-pax)
+AM_INIT_AUTOMAKE([tar-pax foreign])
 
 # Removes warnings when using automake 1.14 around (...but option 'subdir-objects' is disabled )
 #but libglusterfs fails to build with contrib (Then are not set up that way?)
@@ -21,28 +21,21 @@ AM_INIT_AUTOMAKE(tar-pax)
 
 m4_ifdef([AM_SILENT_RULES], [AM_SILENT_RULES(yes)])
 
-if make --help 2>&1 | grep -q no-print-directory; then
-   AM_MAKEFLAGS="$AM_MAKEFLAGS --no-print-directory";
-fi
-
-if make --help 2>&1 | grep -q quiet; then
-   AM_MAKEFLAGS="$AM_MAKEFLAGS --quiet"
-fi
-
-if libtool --help 2>&1 | grep -q quiet; then
-   AM_LIBTOOLFLAGS="--quiet";
-fi
-
-AC_CONFIG_HEADERS([config.h])
+AC_CONFIG_HEADERS([config.h site.h])
 
 AC_CONFIG_FILES([Makefile
                 libglusterfs/Makefile
                 libglusterfs/src/Makefile
-                libglusterfs/src/gfdb/Makefile
+                libglusterd/Makefile
+                libglusterd/src/Makefile
                 geo-replication/src/peer_gsec_create
                 geo-replication/src/peer_mountbroker
+                geo-replication/src/peer_mountbroker.py
+                geo-replication/src/peer_georep-sshkey.py
                 extras/peer_add_secret_pub
-                geo-replication/syncdaemon/configinterface.py
+                geo-replication/syncdaemon/conf.py
+                geo-replication/gsyncd.conf
+                extras/snap_scheduler/conf.py
                 glusterfsd/Makefile
                 glusterfsd/src/Makefile
                 rpc/Makefile
@@ -51,8 +44,6 @@ AC_CONFIG_FILES([Makefile
                 rpc/rpc-transport/Makefile
                 rpc/rpc-transport/socket/Makefile
                 rpc/rpc-transport/socket/src/Makefile
-                rpc/rpc-transport/rdma/Makefile
-                rpc/rpc-transport/rdma/src/Makefile
                 rpc/xdr/Makefile
                 rpc/xdr/src/Makefile
                 xlators/Makefile
@@ -67,13 +58,9 @@ AC_CONFIG_FILES([Makefile
                 xlators/storage/Makefile
                 xlators/storage/posix/Makefile
                 xlators/storage/posix/src/Makefile
-                xlators/storage/bd/Makefile
-                xlators/storage/bd/src/Makefile
                 xlators/cluster/Makefile
                 xlators/cluster/afr/Makefile
                 xlators/cluster/afr/src/Makefile
-                xlators/cluster/stripe/Makefile
-                xlators/cluster/stripe/src/Makefile
                 xlators/cluster/dht/Makefile
                 xlators/cluster/dht/src/Makefile
                 xlators/cluster/ec/Makefile
@@ -89,19 +76,23 @@ AC_CONFIG_FILES([Makefile
                 xlators/performance/io-threads/src/Makefile
                 xlators/performance/io-cache/Makefile
                 xlators/performance/io-cache/src/Makefile
-                xlators/performance/symlink-cache/Makefile
-                xlators/performance/symlink-cache/src/Makefile
                 xlators/performance/quick-read/Makefile
                 xlators/performance/quick-read/src/Makefile
                 xlators/performance/open-behind/Makefile
                 xlators/performance/open-behind/src/Makefile
                 xlators/performance/md-cache/Makefile
                 xlators/performance/md-cache/src/Makefile
+                xlators/performance/nl-cache/Makefile
+                xlators/performance/nl-cache/src/Makefile
                 xlators/debug/Makefile
+                xlators/debug/sink/Makefile
+                xlators/debug/sink/src/Makefile
                 xlators/debug/trace/Makefile
                 xlators/debug/trace/src/Makefile
                 xlators/debug/error-gen/Makefile
                 xlators/debug/error-gen/src/Makefile
+                xlators/debug/delay-gen/Makefile
+                xlators/debug/delay-gen/src/Makefile
                 xlators/debug/io-stats/Makefile
                 xlators/debug/io-stats/src/Makefile
                 xlators/protocol/Makefile
@@ -117,40 +108,34 @@ AC_CONFIG_FILES([Makefile
                 xlators/features/Makefile
                 xlators/features/arbiter/Makefile
                 xlators/features/arbiter/src/Makefile
+                xlators/features/thin-arbiter/Makefile
+                xlators/features/thin-arbiter/src/Makefile
                 xlators/features/changelog/Makefile
                 xlators/features/changelog/src/Makefile
                 xlators/features/changelog/lib/Makefile
                 xlators/features/changelog/lib/src/Makefile
-                xlators/features/changetimerecorder/Makefile
-                xlators/features/changetimerecorder/src/Makefile
-                xlators/features/glupy/Makefile
-                xlators/features/glupy/examples/Makefile
-                xlators/features/glupy/src/Makefile
-                xlators/features/glupy/src/setup.py
-                xlators/features/glupy/src/__init__.py
-                xlators/features/glupy/src/glupy/Makefile
                 xlators/features/locks/Makefile
                 xlators/features/locks/src/Makefile
                 xlators/features/quota/Makefile
                 xlators/features/quota/src/Makefile
                 xlators/features/marker/Makefile
                 xlators/features/marker/src/Makefile
+                xlators/features/selinux/Makefile
+                xlators/features/selinux/src/Makefile
+                xlators/features/sdfs/Makefile
+                xlators/features/sdfs/src/Makefile
                 xlators/features/read-only/Makefile
                 xlators/features/read-only/src/Makefile
                 xlators/features/compress/Makefile
                 xlators/features/compress/src/Makefile
-                xlators/features/mac-compat/Makefile
-                xlators/features/mac-compat/src/Makefile
+                xlators/features/namespace/Makefile
+                xlators/features/namespace/src/Makefile
                 xlators/features/quiesce/Makefile
                 xlators/features/quiesce/src/Makefile
                 xlators/features/barrier/Makefile
                 xlators/features/barrier/src/Makefile
-                xlators/features/ganesha/Makefile
-                xlators/features/ganesha/src/Makefile
                 xlators/features/index/Makefile
                 xlators/features/index/src/Makefile
-                xlators/features/protect/Makefile
-                xlators/features/protect/src/Makefile
                 xlators/features/gfid-access/Makefile
                 xlators/features/gfid-access/src/Makefile
                 xlators/features/trash/Makefile
@@ -167,16 +152,23 @@ AC_CONFIG_FILES([Makefile
                 xlators/features/bit-rot/src/Makefile
                 xlators/features/bit-rot/src/stub/Makefile
                 xlators/features/bit-rot/src/bitd/Makefile
+                xlators/features/leases/Makefile
+                xlators/features/leases/src/Makefile
+                xlators/features/cloudsync/Makefile
+                xlators/features/cloudsync/src/Makefile
+                xlators/features/utime/Makefile
+                xlators/features/utime/src/Makefile
+                xlators/features/cloudsync/src/cloudsync-plugins/Makefile
+                xlators/features/cloudsync/src/cloudsync-plugins/src/Makefile
+                xlators/features/cloudsync/src/cloudsync-plugins/src/cloudsyncs3/Makefile
+                xlators/features/cloudsync/src/cloudsync-plugins/src/cloudsyncs3/src/Makefile
+                xlators/features/cloudsync/src/cloudsync-plugins/src/cvlt/Makefile
+                xlators/features/cloudsync/src/cloudsync-plugins/src/cvlt/src/Makefile
+                xlators/features/metadisp/Makefile
+                xlators/features/metadisp/src/Makefile
                 xlators/playground/Makefile
                 xlators/playground/template/Makefile
                 xlators/playground/template/src/Makefile
-                xlators/encryption/Makefile
-                xlators/encryption/rot-13/Makefile
-                xlators/encryption/rot-13/src/Makefile
-                xlators/encryption/crypt/Makefile
-                xlators/encryption/crypt/src/Makefile
-                xlators/features/qemu-block/Makefile
-                xlators/features/qemu-block/src/Makefile
                 xlators/system/Makefile
                 xlators/system/posix-acl/Makefile
                 xlators/system/posix-acl/src/Makefile
@@ -186,24 +178,30 @@ AC_CONFIG_FILES([Makefile
                 xlators/mgmt/Makefile
                 xlators/mgmt/glusterd/Makefile
                 xlators/mgmt/glusterd/src/Makefile
-                xlators/experimental/Makefile
                 cli/Makefile
                 cli/src/Makefile
                 doc/Makefile
                 extras/Makefile
                 extras/glusterd.vol
+                extras/cliutils/Makefile
                 extras/init.d/Makefile
                 extras/init.d/glusterd.plist
                 extras/init.d/glusterd-Debian
                 extras/init.d/glusterd-Redhat
                 extras/init.d/glusterd-FreeBSD
                 extras/init.d/glusterd-SuSE
+                extras/init.d/glustereventsd-Debian
+                extras/init.d/glustereventsd-Redhat
+                extras/init.d/glustereventsd-FreeBSD
                 extras/ganesha/Makefile
                 extras/ganesha/config/Makefile
                 extras/ganesha/scripts/Makefile
                 extras/ganesha/ocf/Makefile
                 extras/systemd/Makefile
                 extras/systemd/glusterd.service
+                extras/systemd/glustereventsd.service
+                extras/systemd/glusterfssharedstorage.service
+                extras/systemd/gluster-ta-volume.service
                 extras/run-gluster.tmpfiles
                 extras/benchmarking/Makefile
                 extras/hook-scripts/Makefile
@@ -212,10 +210,15 @@ AC_CONFIG_FILES([Makefile
                 extras/ocf/volume
                 extras/LinuxRPM/Makefile
                 extras/geo-rep/Makefile
+                extras/geo-rep/schedule_georep.py
                 extras/firewalld/Makefile
                 extras/hook-scripts/add-brick/Makefile
                 extras/hook-scripts/add-brick/pre/Makefile
                 extras/hook-scripts/add-brick/post/Makefile
+                extras/hook-scripts/create/Makefile
+                extras/hook-scripts/create/post/Makefile
+                extras/hook-scripts/delete/Makefile
+                extras/hook-scripts/delete/pre/Makefile
                 extras/hook-scripts/start/Makefile
                 extras/hook-scripts/start/post/Makefile
                 extras/hook-scripts/set/Makefile
@@ -225,13 +228,16 @@ AC_CONFIG_FILES([Makefile
                 extras/hook-scripts/reset/Makefile
                 extras/hook-scripts/reset/post/Makefile
                 extras/hook-scripts/reset/pre/Makefile
+                extras/python/Makefile
                 extras/snap_scheduler/Makefile
+                events/Makefile
+                events/src/Makefile
+                events/src/eventsapiconf.py
+                events/tools/Makefile
                 contrib/fuse-util/Makefile
                 contrib/umountd/Makefile
-                contrib/uuid/uuid_types.h
                 glusterfs-api.pc
                 libgfchangelog.pc
-                libgfdb.pc
                 api/Makefile
                 api/src/Makefile
                 api/examples/Makefile
@@ -243,10 +249,12 @@ AC_CONFIG_FILES([Makefile
                 heal/Makefile
                 heal/src/Makefile
                 glusterfs.spec
-		tools/glusterfind/src/tool.conf
-	        tools/glusterfind/glusterfind
-	        tools/glusterfind/Makefile
-                tools/glusterfind/src/Makefile])
+                tools/glusterfind/src/tool.conf
+                tools/glusterfind/glusterfind
+                tools/glusterfind/Makefile
+                tools/glusterfind/src/Makefile
+                tools/setgfid2path/Makefile
+                tools/setgfid2path/src/Makefile])
 
 AC_CANONICAL_HOST
 
@@ -267,10 +275,124 @@ AC_ARG_ENABLE([debug],
                              [Enable debug build options.]))
 if test "x$enable_debug" = "xyes"; then
         BUILD_DEBUG=yes
-        CFLAGS="${CFLAGS} -g -O0 -DDEBUG"
+        GF_CFLAGS="${GF_CFLAGS} -g -O0 -DDEBUG"
 else
         BUILD_DEBUG=no
-        CFLAGS="${CFLAGS} -g -O2"
+fi
+
+SANITIZER=none
+
+AC_ARG_ENABLE([asan],
+              AC_HELP_STRING([--enable-asan],
+                             [Enable Address Sanitizer support]))
+if test "x$enable_asan" = "xyes"; then
+        SANITIZER=asan
+        AC_CHECK_LIB([asan], [__asan_init], ,
+                [AC_MSG_ERROR([--enable-asan requires libasan.so, exiting])])
+        GF_CFLAGS="${GF_CFLAGS} -O2 -g -fsanitize=address -fno-omit-frame-pointer"
+        GF_LDFLAGS="${GF_LDFLAGS} -lasan"
+fi
+
+AC_ARG_ENABLE([tsan],
+              AC_HELP_STRING([--enable-tsan],
+                             [Enable Thread Sanitizer support]))
+if test "x$enable_tsan" = "xyes"; then
+        if test "x$SANITIZER" != "xnone"; then
+                AC_MSG_ERROR([only one sanitizer can be enabled at once])
+        fi
+        SANITIZER=tsan
+        AC_CHECK_LIB([tsan], [__tsan_init], ,
+                [AC_MSG_ERROR([--enable-tsan requires libtsan.so, exiting])])
+        if test "x$ac_cv_lib_tsan___tsan_init" = xyes; then
+                AC_MSG_CHECKING([whether tsan API can be used])
+                saved_CFLAGS=${CFLAGS}
+                CFLAGS="${CFLAGS} -fsanitize=thread"
+                AC_COMPILE_IFELSE(
+                [AC_LANG_PROGRAM([
+                   [#include <sanitizer/tsan_interface.h>]],
+                   [[__tsan_create_fiber(0)]])],
+                   [TSAN_API=yes], [TSAN_API=no])
+                AC_MSG_RESULT([$TSAN_API])
+                if test x$TSAN_API = "xyes"; then
+                   AC_DEFINE(HAVE_TSAN_API, 1, [Define if tsan API can be used.])
+                fi
+                CFLAGS=${saved_CFLAGS}
+        fi
+        GF_CFLAGS="${GF_CFLAGS} -O2 -g -fsanitize=thread -fno-omit-frame-pointer"
+        GF_LDFLAGS="${GF_LDFLAGS} -ltsan"
+fi
+
+AC_ARG_ENABLE([ubsan],
+              AC_HELP_STRING([--enable-ubsan],
+                             [Enable Undefined Behavior Sanitizer support]))
+if test "x$enable_ubsan" = "xyes"; then
+        if test "x$SANITIZER" != "xnone"; then
+                AC_MSG_ERROR([only one sanitizer can be enabled at once])
+        fi
+        SANITIZER=ubsan
+        AC_CHECK_LIB([ubsan], [__ubsan_default_options], ,
+                [AC_MSG_ERROR([--enable-ubsan requires libubsan.so, exiting])])
+        GF_CFLAGS="${GF_CFLAGS} -O2 -g -fsanitize=undefined -fno-omit-frame-pointer"
+        GF_LDFLAGS="${GF_LDFLAGS} -lubsan"
+fi
+
+# Initialize CFLAGS before usage
+BUILD_TCMALLOC=no
+AC_ARG_ENABLE([tcmalloc],
+              AC_HELP_STRING([--enable-tcmalloc],
+                             [Enable linking with tcmalloc library.]))
+if test "x$enable_tcmalloc" = "xyes"; then
+    BUILD_TCMALLOC=yes
+    GF_CFLAGS="${GF_CFLAGS} -fno-builtin-malloc -fno-builtin-calloc -fno-builtin-realloc -fno-builtin-free"
+    AC_CHECK_LIB([tcmalloc], [malloc], [],
+                 [AC_MSG_ERROR([when --enable-tcmalloc is used, tcmalloc library needs to be present])])
+    GF_LDFLAGS="-ltcmalloc ${GF_LDFLAGS}"
+fi
+
+
+dnl When possible, prefer libtirpc over glibc rpc.
+dnl
+dnl On newer linux with only libtirpc, use libtirpc. (Specifying
+dnl --without-libtirpc is an error.)
+dnl
+dnl on older linux with glibc rpc and WITH libtirpc, use libtirpc
+dnl by default except when configured with --without-libtirpc.
+dnl
+dnl on old linux with glibc rpc and WITHOUT libtirpc, default to
+dnl use glibc rpc.
+dnl
+AC_ARG_WITH([libtirpc],
+        [AC_HELP_STRING([--without-libtirpc], [Use legacy glibc RPC.])],
+        [with_libtirpc="no"], [with_libtirpc="yes"])
+
+dnl ipv6-default is off by default
+dnl
+dnl ipv6-default requires libtirpc. (glibc rpc does not support IPv6.)
+dnl ipv6-default can only be enabled if libtipc is enabled.
+dnl
+AC_ARG_WITH([ipv6-default],
+        AC_HELP_STRING([--with-ipv6-default], [Set IPv6 as default.]),
+        [with_ipv6_default=${with_libtirpc}], [with_ipv6_default="no"])
+
+AC_CHECK_FILE([/etc/centos-release])
+if test "x$ac_cv_file__etc_centos_release" = "xyes"; then
+        if grep "release 6" /etc/centos-release; then
+                with_ipv6_default="no"
+        fi
+fi
+
+dnl On some distributions '-ldl' isn't automatically added to LIBS
+AC_CHECK_LIB([dl], [dlopen], [LIB_DL=-ldl])
+AC_SUBST(LIB_DL)
+
+AC_ARG_ENABLE([privport_tracking],
+              AC_HELP_STRING([--disable-privport_tracking],
+                             [Disable internal tracking of privileged ports.]))
+TRACK_PRIVPORTS="yes"
+if test x"$enable_privport_tracking" = x"no"; then
+        TRACK_PRIVPORTS="no"
+        AC_DEFINE(GF_DISABLE_PRIVPORT_TRACKING, 1,
+                  [Disable internal tracking of privileged ports.])
 fi
 
 case $host_os in
@@ -280,18 +402,42 @@ case $host_os in
     fi
     # OSX version lesser than 9 has llvm/clang optimization issues which leads to various segfaults
     if test "`/usr/bin/sw_vers | grep ProductVersion: | cut -f 2 | cut -d. -f2`" -lt 9; then
-       CFLAGS="${CFLAGS} -g -O0 -DDEBUG"
+       GF_CFLAGS="${GF_CFLAGS} -g -O0 -DDEBUG"
     fi
     ;;
 esac
 
+# --enable-valgrind prevents calling dlclose(), this leaks memory
+AC_ARG_ENABLE([valgrind],
+              AC_HELP_STRING([--enable-valgrind@<:@=memcheck,drd@:>@],
+                             [Enable valgrind for resource leak (memcheck, which is
+                              the default) or thread synchronization (drd) debugging.]))
+case x$enable_valgrind in
+  xmemcheck|xyes)
+    AC_DEFINE(RUN_WITH_MEMCHECK, 1,
+              [Define if all processes should run under 'valgrind --tool=memcheck'.])
+    VALGRIND_TOOL=memcheck
+    ;;
+  xdrd)
+    AC_DEFINE(RUN_WITH_DRD, 1,
+              [Define if all processes should run under 'valgrind --tool=drd'.])
+    VALGRIND_TOOL=drd
+    ;;
+  x|xno)
+    VALGRIND_TOOL=no
+    ;;
+  *)
+    AC_MSG_ERROR([Please specify --enable-valgrind@<:@=memcheck,drd@:>@])
+    ;;
+esac
+
 AC_ARG_WITH([previous-options],
         [AS_HELP_STRING([--with-previous-options],
                         [read config.status for configure options])
         ],
         [ if test -r ./config.status && \
              args=$(grep 'ac_cs_config=' config.status | \
-                    sed -e 's/.*"\(.*\)".*/\1/'| sed -e "s/'//g") ; then
+                    sed -e 's/.*"\(.*\)".*/\1/' -e "s/'//g" -e "s/--with-previous-options//g") ; then
             echo "###"
             echo "### Rerunning as '$0 $args'"
             echo "###"
@@ -312,10 +458,11 @@ AC_ARG_WITH(mountutildir,
 AC_SUBST(mountutildir)
 
 AC_ARG_WITH(systemddir,
-            [  --with-systemddir=DIR systemd service files in DIR @<:@/usr/lib/systemd/system@:>@],
+            [  --with-systemddir=DIR systemd service files in DIR @<:@PREFIX/lib/systemd/system@:>@],
             [systemddir=$withval],
-            [systemddir='/usr/lib/systemd/system'])
+            [systemddir='${prefix}/lib/systemd/system'])
 AC_SUBST(systemddir)
+AM_CONDITIONAL([USE_SYSTEMD], test [ -d '/usr/lib/systemd/system' ])
 
 AC_ARG_WITH(initdir,
             [  --with-initdir=DIR init.d scripts in DIR @<:@/etc/init.d@:>@],
@@ -343,6 +490,13 @@ AC_ARG_WITH([ocf],
             )
 AC_SUBST(OCF_SUBDIR)
 
+AC_ARG_WITH([server],
+            [AS_HELP_STRING([--without-server], [do not build server components])],
+            [with_server='no'],
+            [with_server='yes'],
+            )
+AM_CONDITIONAL([WITH_SERVER], [test x$with_server = xyes])
+
 # LEX needs a check
 AC_PROG_LEX
 if test  "x${LEX}" != "xflex" -a "x${FLEX}" != "xlex"; then
@@ -393,30 +547,50 @@ AC_CHECK_HEADERS([openssl/dh.h])
 
 AC_CHECK_HEADERS([openssl/ecdh.h])
 
+AC_CHECK_LIB([ssl], [SSL_CTX_get0_param], [AC_DEFINE([HAVE_SSL_CTX_GET0_PARAM], [1], [define if found OpenSSL SSL_CTX_get0_param])])
+
 dnl Math library
 AC_CHECK_LIB([m], [pow], [MATH_LIB='-lm'], [MATH_LIB=''])
 AC_SUBST(MATH_LIB)
 
-dnl use libuuid.so or fall-back to contrib/uuid
+dnl depend on libuuid.so
 PKG_CHECK_MODULES([UUID], [uuid],
-        [HAVE_LIBUUID=yes
+        [have_uuid=yes
          AC_DEFINE(HAVE_LIBUUID, 1, [have libuuid.so])
          PKGCONFIG_UUID=uuid],
-        [HAVE_LIBUUID=no
-         UUID_CFLAGS='-I$(CONTRIBDIR)/uuid'])
-AM_CONDITIONAL([HAVE_LIBUUID], [test x$HAVE_LIBUUID = xyes])
+        [have_uuid=no])
+AM_CONDITIONAL([HAVE_LIBUUID], [test x$have_uuid = xyes])
 
 dnl older version of libuuid (from e2fsprogs) require including uuid/uuid.h
 saved_CFLAGS=${CFLAGS}
 CFLAGS="${CFLAGS} ${UUID_CFLAGS}"
-AC_CHECK_HEADER([uuid.h], [], [AC_CHECK_HEADER([uuid/uuid.h])])
+AC_CHECK_HEADER([uuid.h], [], [AC_CHECK_HEADER([uuid/uuid.h])],
+                [[#if HAVE_UUID_H
+                  #include <uuid.h>
+                  #endif
+                ]])
 CFLAGS=${saved_CFLAGS}
 if test "x$ac_cv_header_uuid_uuid_h" = "xyes"; then
     UUID_CFLAGS="${UUID_CFLAGS} -I$(pkg-config --variable=includedir uuid)/uuid"
+    have_uuid=yes
+fi
+
+if test "x$have_uuid" != "xyes"; then
+    case $host_os in
+        *freebsd*)
+            AC_MSG_ERROR([e2fsprogs-libuuid is required to build glusterfs])
+            ;;
+        linux*)
+            AC_MSG_ERROR([libuuid is required to build glusterfs])
+            ;;
+        *)
+            AC_MSG_ERROR([a Linux compatible libuuid is required to build glusterfs])
+            ;;
+    esac
 fi
 
 dnl libglusterfs needs uuid.h, practically everything depends on it
-GF_CPPFLAGS="${GF_CPPFLAGS} ${UUID_CFLAGS}"
+GF_CFLAGS="${GF_CFLAGS} ${UUID_CFLAGS}"
 dnl PKGCONFIG_UUID is used for the dependency in *.pc.in files
 AC_SUBST(PKGCONFIG_UUID)
 
@@ -462,6 +636,8 @@ AC_SUBST(ZLIB_LIBS)
 
 AC_CHECK_HEADERS([linux/falloc.h])
 
+AC_CHECK_HEADERS([linux/oom.h], AC_DEFINE(HAVE_LINUX_OOM_H, 1, [have linux/oom.h]))
+
 dnl Mac OS X does not have spinlocks
 AC_CHECK_FUNC([pthread_spin_init], [have_spinlock=yes])
 if test "x${have_spinlock}" = "xyes"; then
@@ -491,6 +667,57 @@ if test "x${have_umount2}" = "xyes"; then
    AC_DEFINE(HAVE_UMOUNT2, 1, [define if found umount2])
 fi
 
+dnl Check Python Availability
+have_python=no
+dnl if the user has not specified a python, pick one
+if test -z "${PYTHON}"; then
+  case $host_os in
+    freebsd*)
+      if test -x /usr/local/bin/python3; then
+        PYTHON=/usr/local/bin/python3
+      else
+        PYTHON=/usr/local/bin/python2
+      fi
+      ;;
+    *)
+      if test -x /usr/bin/python3; then
+        PYTHON=/usr/bin/python3
+      else
+        PYTHON=/usr/bin/python2
+      fi
+      ;;
+  esac
+fi
+AM_PATH_PYTHON([2.6],,[:])
+if test -n "${PYTHON}"; then
+   have_python=yes
+fi
+AM_CONDITIONAL(HAVE_PYTHON, test "x$have_python" = "xyes")
+
+dnl Use pkg-config to get runtime search path missing from ${PYTHON}-config
+dnl Just do "true" on failure so that configure does not bail out
+dnl Note: python 2.6's devel pkg (e.g. in CentOS/RHEL 6) does not have
+dnl pkg-config files, so this work-around instead
+if test "x${PYTHON_VERSION}" = "x2.6"; then
+  PYTHON_CFLAGS=$(python-config --includes)
+  PYTHON_LIBS=$(python-config --libs)
+else
+  PKG_CHECK_MODULES([PYTHON], "python-${PYTHON_VERSION}",,true)
+fi
+
+PYTHON_CFLAGS=$(echo ${PYTHON_CFLAGS} | sed -e 's|-I|-isystem |')
+
+BUILD_PYTHON_SITE_PACKAGES=${pythondir}
+AC_SUBST(BUILD_PYTHON_SITE_PACKAGES)
+
+# Eval two times to expand fully. First eval replaces $exec_prefix into $prefix
+# Second eval will expand $prefix
+build_python_site_packages_temp="${pythondir}"
+eval build_python_site_packages_temp=\"${build_python_site_packages_temp}\"
+eval build_python_site_packages_temp=\"${build_python_site_packages_temp}\"
+BUILD_PYTHON_SITE_PACKAGES_EXPANDED=${build_python_site_packages_temp}
+AC_SUBST(BUILD_PYTHON_SITE_PACKAGES_EXPANDED)
+
 # FUSE section
 AC_ARG_ENABLE([fuse-client],
               AC_HELP_STRING([--disable-fuse-client],
@@ -502,70 +729,25 @@ if test "x$enable_fuse_client" != "xno"; then
    BUILD_FUSE_CLIENT="yes"
 fi
 
-AC_ARG_ENABLE([bd-xlator],
-              AC_HELP_STRING([--enable-bd-xlator], [Build BD xlator]))
-
-if test "x$enable_bd_xlator" != "xno"; then
-  AC_CHECK_LIB([lvm2app],
-               [lvm_init,lvm_lv_from_name],
-               [HAVE_BD_LIB="yes"],
-               [HAVE_BD_LIB="no"])
-
-if test "x$HAVE_BD_LIB" = "xyes"; then
-    # lvm_lv_from_name() has been made public with lvm2-2.02.79
-    AC_CHECK_DECLS(
-                  [lvm_lv_from_name],
-                  [NEED_LVM_LV_FROM_NAME_DECL="no"],
-                  [NEED_LVM_LV_FROM_NAME_DECL="yes"],
-                  [[#include <lvm2app.h>]])
-  fi
-fi
-
-if test "x$enable_bd_xlator" = "xyes" -a "x$HAVE_BD_LIB" = "xno"; then
-   echo "BD xlator requested but required lvm2 development library not found."
-   exit 1
-fi
-
-BUILD_BD_XLATOR=no
-if test "x${enable-bd-xlator}" != "xno" -a "x${HAVE_BD_LIB}" = "xyes"; then
-  BUILD_BD_XLATOR=yes
-  AC_DEFINE(HAVE_BD_XLATOR, 1, [define if lvm2app library found and bd xlator
-                            enabled])
-  if test "x$NEED_LVM_LV_FROM_NAME_DECL" = "xyes"; then
-    AC_DEFINE(NEED_LVM_LV_FROM_NAME_DECL, 1, [defined if lvm_lv_from_name()
-                  was not found in the lvm2app.h header, but can be linked])
-  fi
-fi
-
-AM_CONDITIONAL([ENABLE_BD_XLATOR], [test x$BUILD_BD_XLATOR = xyes])
-
-dnl check for old openssl
-AC_CHECK_LIB([crypto], CRYPTO_THREADID_set_callback, [AC_DEFINE([HAVE_CRYPTO_THREADID], [1], [use new OpenSSL functions])])
-
-AC_CHECK_LIB([ssl], TLSv1_2_method, [AC_DEFINE([HAVE_TLSV1_2_METHOD], [1], [use new OpenSSL functions])])
+AC_SUBST(FUSE_CLIENT_SUBDIR)
 
-# start encryption/crypt section
+AC_ARG_ENABLE([fuse-notifications],
+  AS_HELP_STRING([--disable-fuse-notifications], [Disable FUSE notifications]))
 
-AC_CHECK_HEADERS([openssl/cmac.h], [have_cmac_h=yes], [have_cmac_h=no])
+AS_IF([test "x$enable_fuse_notifications" != "xno"], [
+  AC_DEFINE([HAVE_FUSE_NOTIFICATIONS], [1], [Use FUSE notifications])
+])
 
-AC_ARG_ENABLE([crypt-xlator],
-        AC_HELP_STRING([--enable-crypt-xlator], [Build crypt encryption xlator]))
+# end FUSE section
 
-if test "x$enable_crypt_xlator" = "xyes" -a "x$have_cmac_h" = "xno"; then
-   AC_MSG_ERROR([Encryption xlator requires OpenSSL with cmac.h])
-fi
 
-BUILD_CRYPT_XLATOR=no
-if test "x$enable_crypt_xlator" != "xno" -a "x$have_cmac_h" = "xyes"; then
-   BUILD_CRYPT_XLATOR=yes
-   AC_DEFINE(HAVE_CRYPT_XLATOR, 1, [enable building crypt encryption xlator])
+AC_CHECK_LIB([ssl], TLS_method, [HAVE_OPENSSL_1_1="yes"], [HAVE_OPENSSL_1_1="no"])
+if test "x$HAVE_OPENSSL_1_1" = "xyes"; then
+    AC_DEFINE([HAVE_TLS_METHOD], [1], [Using OpenSSL-1.1 TLS_method])
+else
+    AC_CHECK_LIB([ssl], TLSv1_2_method, [AC_DEFINE([HAVE_TLSV1_2_METHOD], [1], [Using OpenSSL-1.0 TLSv1_2_method])])
 fi
 
-AM_CONDITIONAL([ENABLE_CRYPT_XLATOR], [test x$BUILD_CRYPT_XLATOR = xyes])
-
-AC_SUBST(FUSE_CLIENT_SUBDIR)
-# end FUSE section
-
 
 # FUSERMOUNT section
 AC_ARG_ENABLE([fusermount],
@@ -583,33 +765,6 @@ fi
 AC_SUBST(FUSERMOUNT_SUBDIR)
 #end FUSERMOUNT section
 
-# QEMU_BLOCK section
-
-AC_ARG_ENABLE([qemu-block],
-                AC_HELP_STRING([--enable-qemu-block],
-                        [Build QEMU Block formats translator]))
-
-if test "x$enable_qemu_block" != "xno"; then
-  PKG_CHECK_MODULES([GLIB], [glib-2.0],
-                    [HAVE_GLIB_2="yes"],
-                      [HAVE_GLIB_2="no"])
-fi
-
-if test "x$enable_qemu_block" = "xyes" -a "x$HAVE_GLIB_2" = "xno"; then
-   echo "QEMU Block formats translator requires libglib-2.0, but missing."
-   exit 1
-fi
-
-BUILD_QEMU_BLOCK=no
-if test "x${enable_qemu_block}" != "xno" -a "x${HAVE_GLIB_2}" = "xyes"; then
-  BUILD_QEMU_BLOCK=yes
-  AC_DEFINE(HAVE_QEMU_BLOCK, 1, [define if libglib-2.0 library found and QEMU
-                               Block translator enabled])
-fi
-
-
-# end QEMU_BLOCK section
-
 # EPOLL section
 AC_ARG_ENABLE([epoll],
               AC_HELP_STRING([--disable-epoll],
@@ -623,53 +778,6 @@ if test "x$enable_epoll" != "xno"; then
 fi
 # end EPOLL section
 
-
-# IBVERBS section
-AC_ARG_ENABLE([ibverbs],
-              AC_HELP_STRING([--disable-ibverbs],
-                             [Do not build the ibverbs transport]))
-
-if test "x$enable_ibverbs" != "xno"; then
-  AC_CHECK_LIB([ibverbs],
-               [ibv_get_device_list],
-               [HAVE_LIBIBVERBS="yes"],
-               [HAVE_LIBIBVERBS="no"])
-  AC_CHECK_LIB([rdmacm], [rdma_create_id], [HAVE_RDMACM="yes"], [HAVE_RDMACM="no"])
-  if test "x$HAVE_RDMACM" = "xyes" ; then
-    AC_CHECK_DECLS(
-                  [RDMA_OPTION_ID_REUSEADDR],
-                  [],
-                  [AC_ERROR([Need at least version 1.0.15 of librdmacm])],
-                  [[#include <rdma/rdma_cma.h>]])
-  fi
-fi
-
-if test "x$enable_ibverbs" = "xyes"; then
-   if test "x$HAVE_LIBIBVERBS" = "xno"; then
-      echo "ibverbs-transport requested, but libibverbs is not present."
-      exit 1
-   fi
-
-   if test "x$HAVE_RDMACM" = "xno"; then
-      echo "ibverbs-transport requested, but librdmacm is not present."
-      exit 1
-   fi
-fi
-
-BUILD_RDMA=no
-BUILD_IBVERBS=no
-if test "x$enable_ibverbs" != "xno" -a "x$HAVE_LIBIBVERBS" = "xyes" -a "x$HAVE_RDMACM" = "xyes"; then
-  IBVERBS_SUBDIR=ib-verbs
-  BUILD_IBVERBS=yes
-  RDMA_SUBDIR=rdma
-  BUILD_RDMA=yes
-fi
-
-AC_SUBST(IBVERBS_SUBDIR)
-AC_SUBST(RDMA_SUBDIR)
-# end IBVERBS section
-
-
 # SYNCDAEMON section
 AC_ARG_ENABLE([georeplication],
               AC_HELP_STRING([--disable-georeplication],
@@ -677,6 +785,9 @@ AC_ARG_ENABLE([georeplication],
 
 BUILD_SYNCDAEMON=no
 case $host_os in
+     freebsd*)
+#do nothing
+       ;;
      linux*)
 #do nothing
        ;;
@@ -689,25 +800,20 @@ case $host_os in
         ;;
 esac
 SYNCDAEMON_COMPILE=0
-if test "x$enable_georeplication" != "xno"; then
-  SYNCDAEMON_SUBDIR=geo-replication
-  SYNCDAEMON_COMPILE=1
-
-  BUILD_SYNCDAEMON="yes"
-  AM_PATH_PYTHON([2.4])
-  echo -n "checking if python is python 2.x... "
-  if echo $PYTHON_VERSION | grep ^2; then
-    :
+if test "x${with_server}" = "xyes" -a "x${enable_georeplication}" != "xno"; then
+  if test "x${have_python}" = "xno" ; then
+    AC_MSG_ERROR([only python 2 and 3 are supported])
   else
-    echo no
-    AC_MSG_ERROR([only python 2.x is supported])
-  fi
-  echo -n "checking if python has ctypes support... "
-  if "$PYTHON" -c 'import ctypes' 2>/dev/null; then
-    echo yes
-  else
-    echo no
-    AC_MSG_ERROR([python does not have ctypes support])
+    SYNCDAEMON_SUBDIR=geo-replication
+    SYNCDAEMON_COMPILE=1
+
+    BUILD_SYNCDAEMON="yes"
+    AC_MSG_CHECKING([if python has ctypes support...])
+    if "${PYTHON}" -c 'import ctypes' 2>/dev/null; then
+      AC_MSG_RESULT("yes")
+    else
+      AC_MSG_ERROR([python does not have ctypes support])
+    fi
   fi
 fi
 AC_SUBST(SYNCDAEMON_COMPILE)
@@ -715,12 +821,54 @@ AC_SUBST(SYNCDAEMON_SUBDIR)
 # end SYNCDAEMON section
 
 # only install scripts from extras/geo-rep when enabled
-if test "x$enable_georeplication" != "xno"; then
+if test "x${with_server}" = "xyes" -a "x$enable_georeplication" != "xno"; then
   GEOREP_EXTRAS_SUBDIR=geo-rep
 fi
 AC_SUBST(GEOREP_EXTRAS_SUBDIR)
 AM_CONDITIONAL(USE_GEOREP, test "x$enable_georeplication" != "xno")
 
+# METADISP section
+AC_ARG_ENABLE([metadisp],
+              AC_HELP_STRING([--enable-metadisp],
+                             [Enable the metadata dispersal xlator]))
+BUILD_METADISP=no
+if test "x${enable_metadisp}" = "xyes"; then
+  BUILD_METADISP=yes
+fi
+AM_CONDITIONAL([BUILD_METADISP], [test "x$BUILD_METADISP" = "xyes"])
+# end METADISP section
+
+# Events section
+AC_ARG_ENABLE([events],
+              AC_HELP_STRING([--disable-events],
+                             [Do not install Events components]))
+
+BUILD_EVENTS=no
+EVENTS_ENABLED=0
+EVENTS_SUBDIR=
+if test "x$enable_events" != "xno"; then
+  EVENTS_SUBDIR=events
+  EVENTS_ENABLED=1
+
+  BUILD_EVENTS="yes"
+
+  if test "x${have_python}" = "xno"; then
+     if test "x${enable_events}" = "xyes"; then
+        AC_MSG_ERROR([python 2 or 3 required. exiting.])
+     fi
+     AC_MSG_WARN([python not found, disabling events])
+     EVENTS_SUBDIR=
+     EVENTS_ENABLED=0
+     BUILD_EVENTS="no"
+  else
+    AC_DEFINE(USE_EVENTS, 1, [define if events enabled])
+  fi
+fi
+AC_SUBST(EVENTS_ENABLED)
+AC_SUBST(EVENTS_SUBDIR)
+AM_CONDITIONAL([BUILD_EVENTS], [test "x${BUILD_EVENTS}" = "xyes"])
+# end Events section
+
 # CDC xlator - check if libz is present if so enable HAVE_LIB_Z
 BUILD_CDC=yes
 PKG_CHECK_MODULES([ZLIB], [zlib >= 1.2.0],,
@@ -744,8 +892,8 @@ AC_ARG_ENABLE([firewalld],
                              [enable installation configuration for firewalld]),
               [BUILD_FIREWALLD="${enableval}"], [BUILD_FIREWALLD="no"])
 
-if test "x${BUILD_FIREWALLD}" = "xyes"; then
-        if !(which firewalld 1>/dev/null 2>&1) ; then
+if test "x${with_server}" = "xyes" -a "x${BUILD_FIREWALLD}" = "xyes"; then
+        if !(test -d /usr/lib/firewalld/services 1>/dev/null 2>&1) ; then
                 BUILD_FIREWALLD="no (firewalld not installed)"
         fi
 fi
@@ -753,41 +901,13 @@ AM_CONDITIONAL([USE_FIREWALLD],test ["x${BUILD_FIREWALLD}" =  "xyes"])
 
 #endof firewald section
 
-# Data tiering requires sqlite
-AC_ARG_ENABLE([tiering],
-               AC_HELP_STRING([--disable-tiering],
-                             [Disable data classification/tiering]),
-              [BUILD_GFDB="${enableval}"], [BUILD_GFDB="yes"])
-
-case $host_os in
-  darwin*)
-    SQLITE_LIBS="-lsqlite3"
-    AC_CHECK_HEADERS([sqlite3.h], AC_DEFINE(USE_GFDB, 1))
-    ;;
-  *)
-    if test "x${BUILD_GFDB}" = "xyes"; then
-      PKG_CHECK_MODULES([SQLITE], [sqlite3],
-                        AC_DEFINE(USE_GFDB, 1),
-                        AC_MSG_ERROR([pass --disable-tiering to build without sqlite]))
-    else
-      AC_DEFINE(USE_GFDB, 0, [no sqlite, gfdb is disabled])
-    fi
-    ;;
-esac
-
-AC_SUBST(SQLITE_CFLAGS)
-AC_SUBST(SQLITE_LIBS)
-AM_CONDITIONAL(BUILD_GFDB, test "x${BUILD_GFDB}" = "xyes")
-AM_CONDITIONAL(USE_GFDB, test "x${BUILD_GFDB}" = "xyes")
-
 # xml-output
 AC_ARG_ENABLE([xml-output],
               AC_HELP_STRING([--disable-xml-output],
                              [Disable the xml output]))
 BUILD_XML_OUTPUT="yes"
 if test "x$enable_xml_output" != "xno"; then
-    #check if libxml is present if so enable HAVE_LIB_XML
-    m4_ifdef([AM_PATH_XML2],[AM_PATH_XML2([2.6.19])], [no_xml=yes])
+    PKG_CHECK_MODULES([XML], [libxml-2.0], [], [no_xml="yes"])
     if test "x${no_xml}" = "x"; then
         AC_DEFINE([HAVE_LIB_XML], [1], [Define to 1 if using libxml2.])
     else
@@ -807,12 +927,74 @@ else
 fi
 # end of xml-output
 
+dnl cloudsync section
+BUILD_CLOUDSYNC="no"
+AC_CHECK_LIB([curl], [curl_easy_setopt], [LIBCURL="-lcurl"])
+if test -n "$LIBCURL";then
+        HAVE_LIBCURL="yes"
+fi
+AC_CHECK_HEADERS([openssl/hmac.h openssl/evp.h openssl/bio.h openssl/buffer.h], [HAVE_OPENSSL="yes"])
+if test "x$HAVE_LIBCURL" = "xyes" -a "x$HAVE_OPENSSL" = "xyes";then
+   HAVE_AMAZONS3="yes"
+fi
+AM_CONDITIONAL([BUILD_AMAZONS3_PLUGIN], [test "x$HAVE_AMAZONS3" = "xyes"])
+if test "x$HAVE_AMAZONS3" = "xyes";then
+   BUILD_CLOUDSYNC="yes"
+fi
+BUILD_CVLT_PLUGIN="no"
+case $host_os in
+#enable cvlt plugin only for linux platforms
+     linux*)
+       BUILD_CVLT_PLUGIN="yes"
+       BUILD_CLOUDSYNC="yes"
+       ;;
+     *)
+       ;;
+esac
+AM_CONDITIONAL([BUILD_CVLT_PLUGIN], [test "x$BUILD_CVLT_PLUGIN" = "xyes"])
+AM_CONDITIONAL([BUILD_CLOUDSYNC], [test "x$BUILD_CLOUDSYNC" = "xyes"])
+dnl end cloudsync section
+
+dnl SELinux feature enablement
+case $host_os in
+  linux*)
+    AC_ARG_ENABLE([selinux],
+                  AC_HELP_STRING([--disable-selinux],
+                                 [Disable SELinux features]),
+                  [USE_SELINUX="${enableval}"], [USE_SELINUX="yes"])
+    ;;
+  *)
+    USE_SELINUX=no
+    ;;
+esac
+AM_CONDITIONAL(USE_SELINUX, test "x${USE_SELINUX}" = "xyes")
+dnl end of SELinux feature enablement
+
 AC_CHECK_HEADERS([execinfo.h], [have_backtrace=yes])
 if test "x${have_backtrace}" = "xyes"; then
    AC_DEFINE(HAVE_BACKTRACE, 1, [define if found backtrace])
 fi
 AC_SUBST(HAVE_BACKTRACE)
 
+dnl Old (before C11) compiler can compile (but not link) this:
+dnl
+dnl int main () {
+dnl     _Static_assert(1, "True");
+dnl     return 0;
+dnl }
+dnl
+dnl assuming that _Static_assert is an implicitly declared function. So
+dnl we're trying to link just to make sure that this is not the case.
+
+AC_MSG_CHECKING([whether $CC supports C11 _Static_assert])
+AC_TRY_LINK([], [_Static_assert(1, "True");],
+            [STATIC_ASSERT=yes], [STATIC_ASSERT=no])
+
+AC_MSG_RESULT([$STATIC_ASSERT])
+if test x$STATIC_ASSERT = "xyes"; then
+   AC_DEFINE(HAVE_STATIC_ASSERT, 1, [Define if C11 _Static_assert is supported.])
+fi
+
 if test "x${have_backtrace}" != "xyes"; then
 AC_TRY_COMPILE([#include <math.h>], [double x=0.0; x=ceil(0.0);],
    [],
@@ -820,11 +1002,11 @@ AC_TRY_COMPILE([#include <math.h>], [double x=0.0; x=ceil(0.0);],
 fi
 
 dnl glusterfs prints memory usage to stderr by sending it SIGUSR1
-AC_CHECK_FUNC([malloc_stats], [have_malloc_stats=yes])
-if test "x${have_malloc_stats}" = "xyes"; then
-   AC_DEFINE(HAVE_MALLOC_STATS, 1, [define if found malloc_stats])
+AC_CHECK_FUNC([mallinfo], [have_mallinfo=yes])
+if test "x${have_mallinfo}" = "xyes"; then
+   AC_DEFINE(HAVE_MALLINFO, 1, [define if found mallinfo])
 fi
-AC_SUBST(HAVE_MALLOC_STATS)
+AC_SUBST(HAVE_MALLINFO)
 
 dnl Linux, Solaris, Cygwin
 AC_CHECK_MEMBERS([struct stat.st_atim.tv_nsec])
@@ -832,7 +1014,7 @@ dnl FreeBSD, NetBSD
 AC_CHECK_MEMBERS([struct stat.st_atimespec.tv_nsec])
 case $host_os in
         *netbsd*)
-        CFLAGS="${CFLAGS} -D_INCOMPLETE_XOPEN_C063 -DCONFIG_MACHINE_BSWAP_H"
+        GF_CFLAGS="${GF_CFLAGS} -D_INCOMPLETE_XOPEN_C063 -DCONFIG_MACHINE_BSWAP_H"
         ;;
 esac
 AC_CHECK_FUNC([linkat], [have_linkat=yes])
@@ -845,35 +1027,42 @@ dnl check for Monotonic clock
 AC_CHECK_LIB([rt], [clock_gettime], ,
              AC_MSG_WARN([System doesn't have monotonic clock using contrib]))
 
-dnl Check for argp
+dnl check for argp, FreeBSD has the header in /usr/local/include
+case $host_os in
+    *freebsd*)
+    CFLAGS="${CFLAGS} -isystem /usr/local/include"
+    ARGP_LDADD=-largp
+    ;;
+    *netbsd*)
+    ARGP_LDADD=-largp
+    ;;
+esac
+dnl argp-standalone does not provide a pkg-config file
 AC_CHECK_HEADER([argp.h], AC_DEFINE(HAVE_ARGP, 1, [have argp]))
-
-BUILD_ARGP_STANDALONE=no
-if test "x${ac_cv_header_argp_h}" = "xno"; then
-   AC_CONFIG_SUBDIRS(contrib/argp-standalone)
-   BUILD_ARGP_STANDALONE=yes
-   ARGP_STANDALONE_CPPFLAGS='-I${top_srcdir}/contrib/argp-standalone'
-   ARGP_STANDALONE_LDADD='${top_builddir}/contrib/argp-standalone/libargp.a'
-   ARGP_STANDALONE_DIR='${top_builddir}/contrib/argp-standalone'
+if test "x$ac_cv_header_argp_h" != "xyes"; then
+    AC_MSG_ERROR([argp.h not found, install libargp or argp-standalone])
 fi
-
-dnl libglusterfs needs argp.h, practically everything depends on it
-GF_CPPFLAGS="${GF_CPPFLAGS} ${ARGP_STANDALONE_CPPFLAGS}"
-
-AC_SUBST(ARGP_STANDALONE_CPPFLAGS)
-AC_SUBST(ARGP_STANDALONE_LDADD)
-AC_SUBST(ARGP_STANDALONE_DIR)
-
-# Check for atomic operation support
-echo -n "checking for atomic operation support... "
-AC_LINK_IFELSE([AC_LANG_SOURCE([[int main() { long int a = 4; __sync_fetch_and_add_8 (&a, 1); }]])],
-                   [have_sync_fetch_and_add_8=yes], [have_sync_fetch_and_add_8=no])
-if test "x${have_sync_fetch_and_add_8}" = "xyes"; then
-  echo "yes"
-  AC_DEFINE(HAVE_ATOMIC_BUILTINS, 1, [have atomic builtins])
-else
-  echo "no"
+AC_SUBST(ARGP_LDADD)
+
+dnl Check for atomic operation support
+AC_MSG_CHECKING([for gcc __atomic builtins])
+AC_TRY_LINK([], [int v; __atomic_load_n(&v, __ATOMIC_ACQUIRE);],
+            [have_atomic_builtins=yes], [have_atomic_builtins=no])
+if test "x${have_atomic_builtins}" = "xyes"; then
+   AC_DEFINE(HAVE_ATOMIC_BUILTINS, 1, [define if __atomic_*() builtins are available])
+fi
+AC_SUBST(HAVE_ATOMIC_BUILTINS)
+AC_MSG_RESULT([$have_atomic_builtins])
+
+dnl __sync_*() will not be needed if __atomic_*() is available
+AC_MSG_CHECKING([for gcc __sync builtins])
+AC_TRY_LINK([], [__sync_synchronize();],
+            [have_sync_builtins=yes], [have_sync_builtins=no])
+if test "x${have_sync_builtins}" = "xyes"; then
+   AC_DEFINE(HAVE_SYNC_BUILTINS, 1, [define if __sync_*() builtins are available])
 fi
+AC_SUBST(HAVE_SYNC_BUILTINS)
+AC_MSG_RESULT([$have_sync_builtins])
 
 AC_CHECK_HEADER([malloc.h], AC_DEFINE(HAVE_MALLOC_H, 1, [have malloc.h]))
 
@@ -897,6 +1086,63 @@ if test "x${have_posix_fallocate}" = "xyes"; then
    AC_DEFINE(HAVE_POSIX_FALLOCATE, 1, [define if posix_fallocate exists])
 fi
 
+# On fedora-29, copy_file_range syscall and the libc API both are present.
+# Whereas, on some machines such as centos-7, RHEL-7, the API is not there.
+# Only the system call is present. So, this change is to determine whether
+# the API is present or not. If not, then check whether the system call is
+# present or not. Accordingly sys_copy_file_range function will first call
+# the API if it is there. Otherwise it will call syscall(SYS_copy_file_range).
+AC_CHECK_FUNC([copy_file_range], [have_copy_file_range=yes])
+if test "x${have_copy_file_range}" = "xyes"; then
+   AC_DEFINE(HAVE_COPY_FILE_RANGE, 1, [define if copy_file_range exists])
+else
+   OLD_CFLAGS=${CFLAGS}
+   CFLAGS="-D_GNU_SOURCE"
+   AC_CHECK_DECL([SYS_copy_file_range], , , [#include <sys/syscall.h>])
+   if test "x${ac_cv_have_decl_SYS_copy_file_range}" = "xyes"; then
+      AC_DEFINE(HAVE_COPY_FILE_RANGE_SYS, 1, [define if SYS_copy_file_range is available])
+   fi
+   CFLAGS=${OLD_CFLAGS}
+fi
+
+AC_CHECK_FUNC([syncfs], [have_syncfs=yes])
+if test "x${have_syncfs}" = "xyes"; then
+   AC_DEFINE(HAVE_SYNCFS, 1, [define if syncfs exists])
+else
+   OLD_CFLAGS=${CFLAGS}
+   CFLAGS="-D_GNU_SOURCE"
+   AC_CHECK_DECL([SYS_syncfs], , , [#include <sys/syscall.h>])
+   if test "x${ac_cv_have_decl_SYS_syncfs}" = "xyes"; then
+      AC_DEFINE(HAVE_SYNCFS_SYS, 1, [define if SYS_syncfs is available])
+   fi
+   CFLAGS=${OLD_CFLAGS}
+fi
+
+BUILD_NANOSECOND_TIMESTAMPS=no
+AC_CHECK_FUNC([utimensat], [have_utimensat=yes])
+if test "x${have_utimensat}" = "xyes"; then
+   BUILD_NANOSECOND_TIMESTAMPS=yes
+   AC_DEFINE(HAVE_UTIMENSAT, 1, [define if utimensat exists])
+fi
+
+OLD_CFLAGS=${CFLAGS}
+CFLAGS="-D_GNU_SOURCE"
+AC_CHECK_DECL([SEEK_HOLE], , , [#include <unistd.h>])
+if test "x${ac_cv_have_decl_SEEK_HOLE}" = "xyes"; then
+   AC_DEFINE(HAVE_SEEK_HOLE, 1, [define if SEEK_HOLE is available])
+fi
+CFLAGS=${OLD_CFLAGS}
+
+AC_CHECK_FUNC([accept4], [have_accept4=yes])
+if test "x${have_accept4}" = "xyes"; then
+   AC_DEFINE(HAVE_ACCEPT4, 1, [define if accept4 exists])
+fi
+
+AC_CHECK_FUNC([paccept], [have_paccept=yes])
+if test "x${have_paccept}" = "xyes"; then
+AC_DEFINE(HAVE_PACCEPT, 1, [define if paccept exists])
+fi
+
 # Check the distribution where you are compiling glusterfs on
 
 GF_DISTRIBUTION=
@@ -917,7 +1163,34 @@ fi
 AC_SUBST(GF_DISTRIBUTION)
 
 GF_HOST_OS=""
-GF_LDFLAGS="-rdynamic"
+GF_LDFLAGS="${GF_LDFLAGS} -rdynamic"
+
+dnl see --with-libtirpc option check above, libtirpc(-devel) is required for
+dnl ipv6-default
+if test "x${with_libtirpc}" = "xyes" || test "x${with_ipv6_default}" = "xyes" ; then
+    PKG_CHECK_MODULES([TIRPC], [libtirpc],
+       [with_libtirpc="yes"; GF_CFLAGS="$GF_CFLAGS $TIRPC_CFLAGS"; GF_LDFLAGS="$GF_LDFLAGS $TIRPC_LIBS";],
+       [with_libtirpc="missing"; with_ipv6_default="no"])
+fi
+
+if test "x${with_libtirpc}" = "xmissing" ; then
+    AC_CHECK_HEADERS([rpc/rpc.h],[
+        AC_MSG_WARN([
+            ---------------------------------------------------------------------------------
+            libtirpc (and/or ipv6-default) were enabled but libtirpc-devel is not installed.
+            Disabling libtirpc and ipv6-default and falling back to legacy glibc rpc headers.
+            This is a transitional warning message. Eventually it will be an error message.
+            ---------------------------------------------------------------------------------])],[
+        AC_MSG_ERROR([
+            ---------------------------------------------------------------------------------
+            libtirpc (and/or ipv6-default) were enabled but libtirpc-devel is not installed
+            and there were no legacy glibc rpc headers and library to fall back to.
+            ---------------------------------------------------------------------------------])])
+fi
+
+if test "x$with_ipv6_default" = "xyes" ; then
+   GF_CFLAGS="$GF_CFLAGS -DIPV6_DEFAULT"
+fi
 
 dnl check for gcc -Werror=format-security
 saved_CFLAGS=$CFLAGS
@@ -925,28 +1198,21 @@ CFLAGS="-Wformat -Werror=format-security"
 AC_MSG_CHECKING([whether $CC accepts -Werror=format-security])
 AC_COMPILE_IFELSE([AC_LANG_PROGRAM()], [cc_werror_format_security=yes], [cc_werror_format_security=no])
 echo $cc_werror_format_security
-if test "x$cc_werror_format_security" = "xno"; then
-    CFLAGS="$saved_CFLAGS"
-else
-    CFLAGS="$saved_CFLAGS $CFLAGS"
-    GF_CFLAGS="$GF_CFLAGS $CFLAGS"
+if test "x$cc_werror_format_security" = "xyes"; then
+    GF_CFLAGS="$GF_CFLAGS ${CFLAGS}"
 fi
+CFLAGS="$saved_CFLAGS"
 
 dnl check for gcc -Werror=implicit-function-declaration
 saved_CFLAGS=$CFLAGS
-saved_GF_CFLAGS=$GF_CFLAGS
 CFLAGS="-Werror=implicit-function-declaration"
-GF_CFLAGS="-Werror=implicit-function-declaration"
 AC_MSG_CHECKING([whether $CC accepts -Werror=implicit-function-declaration])
 AC_COMPILE_IFELSE([AC_LANG_PROGRAM()], [cc_werror_implicit=yes], [cc_werror_implicit=no])
 echo $cc_werror_implicit
-if test "x$cc_werror_implicit" = "xno"; then
-    CFLAGS="$saved_CFLAGS"
-    GF_CFLAGS="$saved_GF_CFLAGS"
-else
-    CFLAGS="$saved_CFLAGS $CFLAGS"
-    GF_CFLAGS="$saved_GF_CFLAGS $GF_CFLAGS"
+if test "x$cc_werror_implicit" = "xyes"; then
+    GF_CFLAGS="${GF_CFLAGS} ${CFLAGS}"
 fi
+CFLAGS="$saved_CFLAGS"
 
 dnl clang is mostly GCC-compatible, but its version is much lower,
 dnl so we have to check for it.
@@ -983,20 +1249,25 @@ fi
 
 old_prefix=$prefix
 if test "x$prefix" = xNONE; then
-	prefix=$ac_default_prefix
+        prefix=$ac_default_prefix
 fi
-GLUSTERFS_LIBEXECDIR="$(eval echo $prefix)/libexec/glusterfs"
-GLUSTERFSD_MISCDIR="$(eval echo $prefix)/var/lib/misc/glusterfsd"
+old_exec_prefix=$exec_prefix
+if test "x$exec_prefix" = xNONE; then
+        exec_prefix="$(eval echo $prefix)"
+fi
+GLUSTERFS_LIBEXECDIR="$(eval echo $libexecdir)/glusterfs"
 prefix=$old_prefix
+exec_prefix=$old_exec_prefix
 
 ### Dirty hacky stuff to make LOCALSTATEDIR work
 if test "x$prefix" = xNONE; then
    test $localstatedir = '${prefix}/var' && localstatedir=$ac_default_prefix/var
    localstatedir=/var
-   LOCALSTATEDIR=$(eval echo ${localstatedir})
-else
-   LOCALSTATEDIR=$(eval echo ${localstatedir})
 fi
+localstatedir="$(eval echo ${localstatedir})"
+LOCALSTATEDIR=$localstatedir
+
+GLUSTERFSD_MISCDIR="$(eval echo ${localstatedir})/lib/misc/glusterfsd"
 
 old_prefix=$prefix
 if test "x$prefix" = xNONE; then
@@ -1006,6 +1277,7 @@ GLUSTERD_VOLFILE="$(eval echo ${sysconfdir})/glusterfs/glusterd.vol"
 prefix=$old_prefix
 
 
+GFAPI_EXTRA_LDFLAGS='-Wl,--version-script=$(top_srcdir)/api/src/gfapi.map'
 case $host_os in
      linux*)
         GF_HOST_OS="GF_LINUX_HOST_OS"
@@ -1025,9 +1297,9 @@ case $host_os in
         GF_CFLAGS="${GF_CFLAGS} -DTHREAD_UNSAFE_BASENAME"
         GF_CFLAGS="${GF_CFLAGS} -DTHREAD_UNSAFE_DIRNAME"
         GF_FUSE_CFLAGS="-DFUSERMOUNT_DIR=\\\"\$(sbindir)\\\""
-        GF_LDADD="${ARGP_STANDALONE_LDADD}"
+        GF_LDADD="${ARGP_LDADD}"
         if test "x$ac_cv_header_execinfo_h" = "xyes"; then
-           GF_LDFLAGS="-lexecinfo"
+           GF_LDFLAGS="${GF_LDFLAGS} -lexecinfo"
         fi
         GF_FUSE_LDADD="-lperfuse"
         BUILD_FUSE_CLIENT=yes
@@ -1038,17 +1310,13 @@ case $host_os in
         ;;
      *freebsd*)
         GF_HOST_OS="GF_BSD_HOST_OS"
-        GF_CFLAGS="${GF_CFLAGS} ${ARGP_STANDALONE_CPPFLAGS} -O0"
-        GF_CFLAGS="${GF_CFLAGS} -DTHREAD_UNSAFE_BASENAME"
-        GF_CFLAGS="${GF_CFLAGS} -DTHREAD_UNSAFE_DIRNAME"
-        GF_CFLAGS="${GF_CFLAGS} -D_LIBGEN_H_"
         GF_CFLAGS="${GF_CFLAGS} -DO_DSYNC=0"
         GF_CFLAGS="${GF_CFLAGS} -Dxdr_quad_t=xdr_longlong_t"
         GF_CFLAGS="${GF_CFLAGS} -Dxdr_u_quad_t=xdr_u_longlong_t"
         GF_FUSE_CFLAGS="-DFUSERMOUNT_DIR=\\\"\$(sbindir)\\\""
-        GF_LDADD="${ARGP_STANDALONE_LDADD}"
+        GF_LDADD="${ARGP_LDADD}"
         if test "x$ac_cv_header_execinfo_h" = "xyes"; then
-           GF_LDFLAGS="-lexecinfo"
+           GF_LDFLAGS="${GF_LDFLAGS} -lexecinfo"
         fi
         BUILD_FUSE_CLIENT=yes
         BUILD_FUSERMOUNT=no
@@ -1058,29 +1326,41 @@ case $host_os in
      darwin*)
         GF_HOST_OS="GF_DARWIN_HOST_OS"
         LIBTOOL=glibtool
-        GF_CFLAGS="${GF_CFLAGS} ${ARGP_STANDALONE_CPPFLAGS} "
         GF_CFLAGS="${GF_CFLAGS} -D_REENTRANT -D_XOPEN_SOURCE "
         GF_CFLAGS="${GF_CFLAGS} -D_DARWIN_USE_64_BIT_INODE "
         GF_CFLAGS="${GF_CFLAGS} -DTHREAD_UNSAFE_BASENAME"
         GF_CFLAGS="${GF_CFLAGS} -DTHREAD_UNSAFE_DIRNAME"
-        GF_LDADD="${ARGP_STANDALONE_LDADD}"
-        GF_LDFLAGS=""
+        GF_LDADD="${ARGP_LDADD}"
+        GF_LDFLAGS="${GF_LDFLAGS}"
         GF_FUSE_CFLAGS="-I\$(CONTRIBDIR)/macfuse"
         BUILD_FUSERMOUNT="no"
         FUSERMOUNT_SUBDIR=""
         GLUSTERD_WORKDIR="${LOCALSTATEDIR}/db/glusterd"
-        ;;
-esac
-
-case $host_os in
-     darwin*)
         GFAPI_EXTRA_LDFLAGS='-Wl,-alias_list,$(top_srcdir)/api/src/gfapi.aliases'
         ;;
-     *)
-        GFAPI_EXTRA_LDFLAGS='-Wl,--version-script=$(top_srcdir)/api/src/gfapi.map'
-        ;;
 esac
 
+# Default value for sbindir
+prefix_temp=$prefix
+exec_prefix_temp=$exec_prefix
+
+test "${prefix}" = "NONE" && prefix="${ac_default_prefix}"
+test "${exec_prefix}" = "NONE" && exec_prefix='${prefix}'
+sbintemp="${sbindir}"
+eval sbintemp=\"${sbintemp}\"
+eval sbintemp=\"${sbintemp}\"
+SBIN_DIR=${sbintemp}
+
+sysconfdirtemp="${sysconfdir}"
+eval sysconfdirtemp=\"${sysconfdirtemp}\"
+SYSCONF_DIR=${sysconfdirtemp}
+
+prefix=$prefix_temp
+exec_prefix=$exec_prefix_temp
+
+AC_SUBST(SBIN_DIR)
+AC_SUBST(SYSCONF_DIR)
+
 # lazy umount emulation
 UMOUNTD_SUBDIR=""
 if test "x${GF_HOST_OS}" != "xGF_LINUX_HOST_OS" ; then
@@ -1088,15 +1368,21 @@ if test "x${GF_HOST_OS}" != "xGF_LINUX_HOST_OS" ; then
 fi
 AC_SUBST(UMOUNTD_SUBDIR)
 
-# enable/disable QEMU
-AM_CONDITIONAL([ENABLE_QEMU_BLOCK], [test x$BUILD_QEMU_BLOCK = xyes])
-
 
 # enable debug section
 AC_ARG_ENABLE([debug],
               AC_HELP_STRING([--enable-debug],
                              [Enable debug build options.]))
 
+AC_ARG_ENABLE([mempool],
+              AC_HELP_STRING([--disable-mempool],
+                             [Disable the Gluster memory pooler.]))
+
+USE_MEMPOOL="yes"
+if test "x$enable_mempool" = "xno"; then
+        USE_MEMPOOL="no"
+        AC_DEFINE(GF_DISABLE_MEMPOOL, 1, [Disable the Gluster memory pooler.])
+fi
 
 # syslog section
 AC_ARG_ENABLE([syslog],
@@ -1117,7 +1403,7 @@ AC_CHECK_LIB([readline -lcurses],[readline],[RLLIBS="-lreadline -lcurses"])
 AC_CHECK_LIB([readline -ltermcap],[readline],[RLLIBS="-lreadline -ltermcap"])
 AC_CHECK_LIB([readline -lncurses],[readline],[RLLIBS="-lreadline -lncurses"])
 
-if test "x$RLLIBS" != "x"; then
+if test -n "$RLLIBS"; then
    if test "x$RL_UNDO" = "xyes"; then
       AC_DEFINE(HAVE_READLINE, 1, [readline enabled CLI])
       BUILD_READLINE=yes
@@ -1130,104 +1416,41 @@ fi
 BUILD_LIBAIO=no
 AC_CHECK_LIB([aio],[io_setup],[LIBAIO="-laio"])
 
-if test "x$LIBAIO" != "x"; then
+if test -n "$LIBAIO"; then
    AC_DEFINE(HAVE_LIBAIO, 1, [libaio based POSIX enabled])
    BUILD_LIBAIO=yes
 fi
 
-# glupy section
-BUILD_GLUPY=no
-have_python2=no
-have_Python_h=no
-
-AM_PATH_PYTHON()
-if echo $PYTHON_VERSION | grep ^2; then
-  have_python2=yes
+dnl gnfs section
+BUILD_GNFS="no"
+RPCBIND_SERVICE=""
+AC_ARG_ENABLE([gnfs],
+              AC_HELP_STRING([--enable-gnfs],
+                             [Enable legacy gnfs server xlator.]))
+if test "x${with_server}" = "xyes" -a "x$enable_gnfs" = "xyes"; then
+    BUILD_GNFS="yes"
+    GF_CFLAGS="$GF_CFLAGS -DBUILD_GNFS"
+    RPCBIND_SERVICE="rpcbind.service"
 fi
-
-# Save flags before testing python
-saved_CFLAGS=$CFLAGS
-saved_CPPFLAGS=$CPPFLAGS
-saved_LDFLAGS=$LDFLAGS
-
-# Use pkg-config to get runtime search patch missing from ${PYTHON}-config
-# Just do "true" on failure so that configure does not bail out
-PKG_CHECK_MODULES([PYTHON], "python-$PYTHON_VERSION",,true)
-CFLAGS="`${PYTHON}-config --cflags`"
-CPPFLAGS=$CFLAGS
-LDFLAGS="${PYTHON_LIBS} -L`${PYTHON}-config --prefix`/lib -L`${PYTHON}-config --prefix`/$libdir `${PYTHON}-config --ldflags`"
-
-AC_CHECK_HEADERS([python$PYTHON_VERSION/Python.h],[have_Python_h=yes],[])
-AC_ARG_ENABLE([glupy],
-              AS_HELP_STRING([--enable-glupy],
-                             [build glupy]))
-case x$enable_glupy in
-   xyes)
-      if test "x$have_python2" = "xyes" -a "x$have_Python_h" = "xyes"; then
-        BUILD_GLUPY=yes
-        PYTHONDEV_CFLAGS="$CFLAGS"
-        PYTHONDEV_CPPFLAGS="$CPPFLAGS"
-        PYTHONDEV_LDFLAGS="$LDFLAGS"
-        AC_SUBST(PYTHONDEV_CFLAGS)
-        AC_SUBST(PYTHONDEV_CPPFLAGS)
-        AC_SUBST(PYTHONDEV_LDFLAGS)
-      else
-        AC_MSG_ERROR([glupy requires python-devel/python-dev package and python2.x])
-      fi
-      ;;
-   xno)
-      ;;
-   *)
-      if test "x$have_python2" = "xyes" -a "x$have_Python_h" = "xyes"; then
-        BUILD_GLUPY=yes
-        PYTHONDEV_CFLAGS="$CFLAGS"
-        PYTHONDEV_CPPFLAGS="$CPPFLAGS"
-        PYTHONDEV_LDFLAGS="$LDFLAGS"
-        AC_SUBST(PYTHONDEV_CFLAGS)
-        AC_SUBST(PYTHONDEV_CPPFLAGS)
-        AC_SUBST(PYTHONDEV_LDFLAGS)
-      else
-        AC_MSG_WARN([
-        ---------------------------------------------------------------------------------
-         cannot build glupy. python 2.x and python-devel/python-dev package are required.
-        ---------------------------------------------------------------------------------])
-      fi
-      ;;
-esac
-# Restore flags
-CFLAGS=$saved_CFLAGS
-CPPFLAGS=$saved_CPPFLAGS
-LDFLAGS=$saved_LDFLAGS
-
-case $host_os in
-     darwin*)
-        BUILD_GLUPY=no
-        ;;
-esac
-
-if test "x$BUILD_GLUPY" = "xyes"; then
-   BUILD_PYTHON_SITE_PACKAGES=`$PYTHON -c 'from distutils.sysconfig import get_python_lib; print(get_python_lib())'`
-   BUILD_PYTHON_INC=`$PYTHON -c "from distutils import sysconfig; print sysconfig.get_python_inc()"`
-   BUILD_PYTHON_LIB=python$PYTHON_VERSION
-   GLUPY_SUBDIR=glupy
-   GLUPY_SUBDIR_MAKEFILE=xlators/features/glupy/Makefile
-   GLUPY_SUBDIR_SRC_MAKEFILE=xlators/features/glupy/src/Makefile
-   echo "building glupy with -isystem $BUILD_PYTHON_INC -l $BUILD_PYTHON_LIB"
-   AC_SUBST(BUILD_PYTHON_SITE_PACKAGES)
-   AC_SUBST(BUILD_PYTHON_INC)
-   AC_SUBST(BUILD_PYTHON_LIB)
-   AC_SUBST(GLUPY_SUBDIR)
-   AC_SUBST(GLUPY_SUBDIR_MAKEFILE)
-   AC_SUBST(GLUPY_SUBDIR_SRC_MAKEFILE)
-fi
-# end glupy section
+AM_CONDITIONAL([BUILD_GNFS], [test x$BUILD_GNFS = xyes])
+AC_SUBST(BUILD_GNFS)
+AC_SUBST(RPCBIND_SERVICE)
+dnl end gnfs section
 
 dnl Check for userspace-rcu
-PKG_CHECK_MODULES([URCU], [liburcu-bp])
+PKG_CHECK_MODULES([URCU], [liburcu-bp], [],
+  [AC_CHECK_HEADERS([urcu-bp.h],
+     [URCU_LIBS='-lurcu-bp'],
+     AC_MSG_ERROR([liburcu-bp not found]))])
 PKG_CHECK_MODULES([URCU_CDS], [liburcu-cds >= 0.8], [],
   [PKG_CHECK_MODULES([URCU_CDS], [liburcu-cds >= 0.7],
-    [AC_DEFINE(URCU_0_7, 1, [Define if liburcu 0.7 is found])],
-    [AC_MSG_ERROR([liburcu >= 0.7 required])])])
+    [AC_DEFINE(URCU_OLD, 1, [Define if liburcu 0.6 or 0.7 is found])
+     USE_CONTRIB_URCU='yes'],
+    [AC_CHECK_HEADERS([urcu/cds.h],
+      [AC_DEFINE(URCU_OLD, 1, [Define if liburcu 0.6 or 0.7 is found])
+       URCU_CDS_LIBS='-lurcu-cds'
+       USE_CONTRIB_URCU='yes'],
+      [AC_MSG_ERROR([liburcu-cds not found])])])])
 
 BUILD_UNITTEST="no"
 AC_ARG_ENABLE([cmocka],
@@ -1263,22 +1486,113 @@ AC_SUBST(UNITTEST_LDFLAGS)
 AC_SUBST(CFLAGS)
 # end enable debug section
 
-# experimental section
-AC_ARG_ENABLE([experimental],
-              AC_HELP_STRING([--disable-experimental],
-                             [Disable building experimental xlators]))
+# EC dynamic code generation section
+
+EC_DYNAMIC_SUPPORT="none"
+EC_DYNAMIC_ARCH="none"
 
-BUILD_EXPERIMENTAL="yes"
-if test "x$enable_experimental" == "xno"; then
-  BUILD_EXPERIMENTAL="no"
+AC_ARG_ENABLE([ec-dynamic],
+              AC_HELP_STRING([--disable-ec-dynamic],
+                             [Disable all dynamic code generation extensions for EC module]))
+
+AC_ARG_ENABLE([ec-dynamic-intel],
+              AC_HELP_STRING([--disable-ec-dynamic-intel],
+                             [Disable all INTEL dynamic code generation extensions for EC module]))
+
+AC_ARG_ENABLE([ec-dynamic-arm],
+              AC_HELP_STRING([--disable-ec-dynamic-arm],
+                             [Disable all ARM dynamic code generation extensions for EC module]))
+
+AC_ARG_ENABLE([ec-dynamic-x64],
+              AC_HELP_STRING([--disable-ec-dynamic-x64],
+                             [Disable dynamic INTEL x64 code generation for EC module]))
+
+AC_ARG_ENABLE([ec-dynamic-sse],
+              AC_HELP_STRING([--disable-ec-dynamic-sse],
+                             [Disable dynamic INTEL SSE code generation for EC module]))
+
+AC_ARG_ENABLE([ec-dynamic-avx],
+              AC_HELP_STRING([--disable-ec-dynamic-avx],
+                             [Disable dynamic INTEL AVX code generation for EC module]))
+
+AC_ARG_ENABLE([ec-dynamic-neon],
+              AC_HELP_STRING([--disable-ec-dynamic-neon],
+                             [Disable dynamic ARM NEON code generation for EC module]))
+
+if test "x$enable_ec_dynamic" != "xno"; then
+  case $host in
+    x86_64*)
+      if test "x$enable_ec_dynamic_intel" != "xno"; then
+        if test "x$enable_ec_dynamic_x64" != "xno"; then
+          EC_DYNAMIC_SUPPORT="$EC_DYNAMIC_SUPPORT x64"
+          AC_DEFINE(USE_EC_DYNAMIC_X64, 1, [Defined if using dynamic INTEL x64 code])
+        fi
+        if test "x$enable_ec_dynamic_sse" != "xno"; then
+          EC_DYNAMIC_SUPPORT="$EC_DYNAMIC_SUPPORT sse"
+          AC_DEFINE(USE_EC_DYNAMIC_SSE, 1, [Defined if using dynamic INTEL SSE code])
+        fi
+        if test "x$enable_ec_dynamic_avx" != "xno"; then
+          EC_DYNAMIC_SUPPORT="$EC_DYNAMIC_SUPPORT avx"
+          AC_DEFINE(USE_EC_DYNAMIC_AVX, 1, [Defined if using dynamic INTEL AVX code])
+        fi
+
+        if test "x$EC_DYNAMIC_SUPPORT" != "xnone"; then
+          EC_DYNAMIC_ARCH="intel"
+        fi
+      fi
+      ;;
+    arm*)
+      if test "x$enable_ec_dynamic_arm" != "xno"; then
+        if test "x$enable_ec_dynamic_neon" != "xno"; then
+          EC_DYNAMIC_SUPPORT="$EC_DYNAMIC_SUPPORT neon"
+          AC_DEFINE(USE_EC_DYNAMIC_NEON, 1, [Defined if using dynamic ARM NEON code])
+        fi
+
+        if test "x$EC_DYNAMIC_SUPPORT" != "xnone"; then
+          EC_DYNAMIC_ARCH="arm"
+        fi
+      fi
+      ;;
+  esac
+
+  EC_DYNAMIC_SUPPORT="${EC_DYNAMIC_SUPPORT#none }"
 fi
-AM_CONDITIONAL([ENABLE_EXPERIMENTAL], [test x$BUILD_EXPERIMENTAL = xyes])
-#end experimental section
+
+AM_CONDITIONAL([ENABLE_EC_DYNAMIC_INTEL], [test "x$EC_DYNAMIC_ARCH" = "xintel"])
+AM_CONDITIONAL([ENABLE_EC_DYNAMIC_ARM], [test "x$EC_DYNAMIC_ARCH" = "xarm"])
+
+AM_CONDITIONAL([ENABLE_EC_DYNAMIC_X64], [test "x${EC_DYNAMIC_SUPPORT##*x64*}" = "x"])
+AM_CONDITIONAL([ENABLE_EC_DYNAMIC_SSE], [test "x${EC_DYNAMIC_SUPPORT##*sse*}" = "x"])
+AM_CONDITIONAL([ENABLE_EC_DYNAMIC_AVX], [test "x${EC_DYNAMIC_SUPPORT##*avx*}" = "x"])
+AM_CONDITIONAL([ENABLE_EC_DYNAMIC_NEON], [test "x${EC_DYNAMIC_SUPPORT##*neon*}" = "x"])
+
+AC_SUBST(USE_EC_DYNAMIC_X64)
+AC_SUBST(USE_EC_DYNAMIC_SSE)
+AC_SUBST(USE_EC_DYNAMIC_AVX)
+AC_SUBST(USE_EC_DYNAMIC_NEON)
+
+# end EC dynamic code generation section
 
 dnl libglusterfs.so uses math functions
 GF_LDADD="${GF_LDADD} ${MATH_LIB}"
 
-GF_XLATOR_DEFAULT_LDFLAGS='-avoid-version -export-symbols $(top_srcdir)/xlators/xlator.sym'
+case $host_os in
+  dnl Can't use libtool's portable "-no-undefined" as it seems to be ignored on Linux
+  linux*)
+    GF_NO_UNDEFINED='-Wl,--no-undefined'
+    ;;
+  darwin*)
+    GF_NO_UNDEFINED='-Wl,-undefined'
+    ;;
+  *)
+  dnl There's an issue on FreeBSD with reference to __progname used in some parts of code
+    GF_NO_UNDEFINED=''
+    ;;
+esac
+dnl GF_XLATOR_DEFAULT_LDFLAGS is for most xlators that expose a common set of symbols
+GF_XLATOR_DEFAULT_LDFLAGS='-avoid-version -export-symbols $(top_srcdir)/xlators/xlator.sym $(UUID_LIBS) $(GF_NO_UNDEFINED) $(TIRPC_LIBS)'
+dnl GF_XLATOR_LDFLAGS is for xlators that expose extra symbols, e.g. dht
+GF_XLATOR_LDFLAGS='-avoid-version $(UUID_LIBS) $(GF_NO_UNDEFINED) $(TIRPC_LIBS)'
 
 AC_SUBST(GF_HOST_OS)
 AC_SUBST(GF_CFLAGS)
@@ -1290,19 +1604,34 @@ AC_SUBST(RLLIBS)
 AC_SUBST(LIBAIO)
 AC_SUBST(AM_MAKEFLAGS)
 AC_SUBST(AM_LIBTOOLFLAGS)
+AC_SUBST(GF_NO_UNDEFINED)
 AC_SUBST(GF_XLATOR_DEFAULT_LDFLAGS)
+AC_SUBST(GF_XLATOR_LDFLAGS)
+AC_SUBST(GF_XLATOR_MGNT_LIBADD)
+
+case $host_os in
+     *freebsd*)
+        GF_XLATOR_MGNT_LIBADD="-lutil -lprocstat"
+        ;;
+esac
 
 CONTRIBDIR='$(top_srcdir)/contrib'
 AC_SUBST(CONTRIBDIR)
 
 GF_CPPDEFINES='-D_FILE_OFFSET_BITS=64 -D_GNU_SOURCE -D$(GF_HOST_OS)'
-GF_CPPINCLUDES='-include $(top_builddir)/config.h -I$(top_srcdir)/libglusterfs/src'
+GF_CPPINCLUDES='-include $(top_builddir)/config.h -include $(top_builddir)/site.h -I$(top_srcdir)/libglusterfs/src -I$(top_builddir)/libglusterfs/src'
+if test "x${USE_CONTRIB_URCU}" = "xyes"; then
+    GF_CPPINCLUDES="${GF_CPPINCLUDES} -I\$(CONTRIBDIR)/userspace-rcu"
+fi
 GF_CPPFLAGS="$GF_CPPFLAGS $GF_CPPDEFINES $GF_CPPINCLUDES"
 AC_SUBST([GF_CPPFLAGS])
 
 AM_CONDITIONAL([GF_LINUX_HOST_OS], test "${GF_HOST_OS}" = "GF_LINUX_HOST_OS")
 AM_CONDITIONAL([GF_DARWIN_HOST_OS], test "${GF_HOST_OS}" = "GF_DARWIN_HOST_OS")
 AM_CONDITIONAL([GF_BSD_HOST_OS], test "${GF_HOST_OS}" = "GF_BSD_HOST_OS")
+if  test "${GF_HOST_OS}" = "GF_BSD_HOST_OS"; then
+    AC_DEFINE(GF_BSD_HOST_OS, 1, [This is a BSD compatible OS.])
+fi
 
 AC_SUBST(GLUSTERD_WORKDIR)
 AM_CONDITIONAL([GF_INSTALL_GLUSTERD_WORKDIR], test ! -d ${GLUSTERD_WORKDIR} && test -d ${sysconfdir}/glusterd )
@@ -1324,8 +1653,6 @@ GFAPI_VERSION="7."${PACKAGE_VERSION}
 LIBGFCHANGELOG_VERSION="0.0.1"
 AC_SUBST(GFAPI_VERSION)
 AC_SUBST(LIBGFCHANGELOG_VERSION)
-LIBGFDB_VERSION="0.0.1"
-AC_SUBST(LIBGFDB_VERSION)
 
 dnl libtool versioning
 LIBGFXDR_LT_VERSION="0:1:0"
@@ -1352,23 +1679,39 @@ echo
 echo "GlusterFS configure summary"
 echo "==========================="
 echo "FUSE client          : $BUILD_FUSE_CLIENT"
-echo "Infiniband verbs     : $BUILD_IBVERBS"
 echo "epoll IO multiplex   : $BUILD_EPOLL"
-echo "argp-standalone      : $BUILD_ARGP_STANDALONE"
 echo "fusermount           : $BUILD_FUSERMOUNT"
 echo "readline             : $BUILD_READLINE"
 echo "georeplication       : $BUILD_SYNCDAEMON"
 echo "Linux-AIO            : $BUILD_LIBAIO"
 echo "Enable Debug         : $BUILD_DEBUG"
-echo "Block Device xlator  : $BUILD_BD_XLATOR"
-echo "glupy                : $BUILD_GLUPY"
+echo "Run with Valgrind    : $VALGRIND_TOOL"
+echo "Sanitizer enabled    : $SANITIZER"
 echo "Use syslog           : $USE_SYSLOG"
 echo "XML output           : $BUILD_XML_OUTPUT"
-echo "QEMU Block formats   : $BUILD_QEMU_BLOCK"
-echo "Encryption xlator    : $BUILD_CRYPT_XLATOR"
 echo "Unit Tests           : $BUILD_UNITTEST"
+echo "Track priv ports     : $TRACK_PRIVPORTS"
 echo "POSIX ACLs           : $BUILD_POSIX_ACLS"
-echo "Data Classification  : $BUILD_GFDB"
+echo "SELinux features     : $USE_SELINUX"
 echo "firewalld-config     : $BUILD_FIREWALLD"
-echo "Experimental xlators : $BUILD_EXPERIMENTAL"
+echo "Events               : $BUILD_EVENTS"
+echo "EC dynamic support   : $EC_DYNAMIC_SUPPORT"
+echo "Use memory pools     : $USE_MEMPOOL"
+echo "Nanosecond m/atimes  : $BUILD_NANOSECOND_TIMESTAMPS"
+echo "Server components    : $with_server"
+echo "Legacy gNFS server   : $BUILD_GNFS"
+echo "IPV6 default         : $with_ipv6_default"
+echo "Use TIRPC            : $with_libtirpc"
+echo "With Python          : ${PYTHON_VERSION}"
+echo "Cloudsync            : $BUILD_CLOUDSYNC"
+echo "Metadata dispersal   : $BUILD_METADISP"
+echo "Link with TCMALLOC   : $BUILD_TCMALLOC"
 echo
+
+# dnl Note: ${X^^} capitalization assumes bash >= 4.x
+if test "x$SANITIZER" != "xnone"; then
+        echo "Note: since glusterfs processes are daemon processes, use"
+        echo "'export ${SANITIZER^^}_OPTIONS=log_path=/path/to/xxx.log' to collect"
+        echo "sanitizer output. Further details and more options can be"
+        echo "found at https://github.com/google/sanitizers."
+fi
diff --git a/contrib/argp-standalone/Makefile.am b/contrib/argp-standalone/Makefile.am
deleted file mode 100644
index 4775d4876aa..00000000000
--- a/contrib/argp-standalone/Makefile.am
+++ /dev/null
@@ -1,38 +0,0 @@
-# From glibc
-
-# Copyright (C) 1997, 2003, 2004 Free Software Foundation, Inc.
-# This file is part of the GNU C Library.
-
-# The GNU C Library is free software; you can redistribute it and/or
-# modify it under the terms of the GNU Library General Public License as
-# published by the Free Software Foundation; either version 2 of the
-# License, or (at your option) any later version.
-
-# The GNU C Library is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-# Library General Public License for more details.
-
-# You should have received a copy of the GNU Library General Public
-# License along with the GNU C Library; see the file COPYING.LIB.  If
-# not, write to the Free Software Foundation, Inc.,
-# 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
-AUTOMAKE_OPTIONS = foreign
-SUBDIRS = .
-
-LIBOBJS = @LIBOBJS@
-
-noinst_LIBRARIES = libargp.a
-
-noinst_HEADERS = argp.h argp-fmtstream.h argp-namefrob.h
-
-EXTRA_DIST = mempcpy.c strchrnul.c strndup.c strcasecmp.c vsnprintf.c autogen.sh
-
-# Leaves out argp-fs-xinl.c and argp-xinl.c 
-libargp_a_SOURCES = argp-ba.c argp-eexst.c argp-fmtstream.c \
-		    argp-help.c argp-parse.c argp-pv.c \
-		    argp-pvh.c
-
-libargp_a_LIBADD = $(LIBOBJS)
-
-
diff --git a/contrib/argp-standalone/acinclude.m4 b/contrib/argp-standalone/acinclude.m4
deleted file mode 100644
index fb61e957dfa..00000000000
--- a/contrib/argp-standalone/acinclude.m4
+++ /dev/null
@@ -1,1084 +0,0 @@
-dnl Try to detect the type of the third arg to getsockname() et al
-AC_DEFUN([LSH_TYPE_SOCKLEN_T],
-[AH_TEMPLATE([socklen_t], [Length type used by getsockopt])
-AC_CACHE_CHECK([for socklen_t in sys/socket.h], ac_cv_type_socklen_t,
-[AC_EGREP_HEADER(socklen_t, sys/socket.h,
-  [ac_cv_type_socklen_t=yes], [ac_cv_type_socklen_t=no])])
-if test $ac_cv_type_socklen_t = no; then
-        AC_MSG_CHECKING(for AIX)
-        AC_EGREP_CPP(yes, [
-#ifdef _AIX
- yes
-#endif
-],[
-AC_MSG_RESULT(yes)
-AC_DEFINE(socklen_t, size_t)
-],[
-AC_MSG_RESULT(no)
-AC_DEFINE(socklen_t, int)
-])
-fi
-])
-
-dnl Choose cc flags for compiling position independent code
-AC_DEFUN([LSH_CCPIC],
-[AC_MSG_CHECKING(CCPIC)
-AC_CACHE_VAL(lsh_cv_sys_ccpic,[
-  if test -z "$CCPIC" ; then
-    if test "$GCC" = yes ; then
-      case `uname -sr` in
-	BSD/OS*)
-         case `uname -r` in
-           4.*) CCPIC="-fPIC";;
-           *) CCPIC="";;
-         esac
-	;;
-	Darwin*)
-	  CCPIC="-fPIC"
-	;;
-	SunOS\ 5.*)
-	  # Could also use -fPIC, if there are a large number of symbol reference
-	  CCPIC="-fPIC"
-	;;
-	CYGWIN*)
-	  CCPIC=""
-	;;
-	*)
-	  CCPIC="-fpic"
-	;;
-      esac
-    else
-      case `uname -sr` in
-	Darwin*)
-	  CCPIC="-fPIC"
-	;;
-        IRIX*)
-          CCPIC="-share"
-        ;;
-	hp*|HP*) CCPIC="+z"; ;;
-	FreeBSD*) CCPIC="-fpic";;
-	SCO_SV*) CCPIC="-KPIC -dy -Bdynamic";;
-        UnixWare*|OpenUNIX*) CCPIC="-KPIC -dy -Bdynamic";;
-	Solaris*) CCPIC="-KPIC -Bdynamic";;
-	Windows_NT*) CCPIC="-shared" ;;
-      esac
-    fi
-  fi
-  OLD_CFLAGS="$CFLAGS"
-  CFLAGS="$CFLAGS $CCPIC"
-  AC_TRY_COMPILE([], [exit(0);],
-    lsh_cv_sys_ccpic="$CCPIC", lsh_cv_sys_ccpic='')
-  CFLAGS="$OLD_CFLAGS"
-])
-CCPIC="$lsh_cv_sys_ccpic"
-AC_MSG_RESULT($CCPIC)
-AC_SUBST([CCPIC])])
-
-dnl LSH_PATH_ADD(path-id, directory)
-AC_DEFUN([LSH_PATH_ADD],
-[AC_MSG_CHECKING($2)
-ac_exists=no
-if test -d "$2/." ; then
-  ac_real_dir=`cd $2 && pwd`
-  if test -n "$ac_real_dir" ; then
-    ac_exists=yes
-    for old in $1_REAL_DIRS ; do
-      ac_found=no
-      if test x$ac_real_dir = x$old ; then
-        ac_found=yes;
-	break;
-      fi
-    done
-    if test $ac_found = yes ; then
-      AC_MSG_RESULT(already added)
-    else
-      AC_MSG_RESULT(added)
-      # LDFLAGS="$LDFLAGS -L $2"
-      $1_REAL_DIRS="$ac_real_dir [$]$1_REAL_DIRS"
-      $1_DIRS="$2 [$]$1_DIRS"
-    fi
-  fi
-fi
-if test $ac_exists = no ; then
-  AC_MSG_RESULT(not found)
-fi
-])
-
-dnl LSH_RPATH_ADD(dir)
-AC_DEFUN([LSH_RPATH_ADD], [LSH_PATH_ADD(RPATH_CANDIDATE, $1)])
-
-dnl LSH_RPATH_INIT(candidates)
-AC_DEFUN([LSH_RPATH_INIT],
-[AC_MSG_CHECKING([for -R flag])
-RPATHFLAG=''
-case `uname -sr` in
-  OSF1\ V4.*)
-    RPATHFLAG="-rpath "
-    ;;
-  IRIX\ 6.*)
-    RPATHFLAG="-rpath "
-    ;;
-  IRIX\ 5.*)
-    RPATHFLAG="-rpath "
-    ;;
-  SunOS\ 5.*)
-    if test "$TCC" = "yes"; then
-      # tcc doesn't know about -R
-      RPATHFLAG="-Wl,-R,"
-    else
-      RPATHFLAG=-R
-    fi
-    ;;
-  Linux\ 2.*)
-    RPATHFLAG="-Wl,-rpath,"
-    ;;
-  *)
-    :
-    ;;
-esac
-
-if test x$RPATHFLAG = x ; then
-  AC_MSG_RESULT(none)
-else
-  AC_MSG_RESULT([using $RPATHFLAG])
-fi
-
-RPATH_CANDIDATE_REAL_DIRS=''
-RPATH_CANDIDATE_DIRS=''
-
-AC_MSG_RESULT([Searching for libraries])
-
-for d in $1 ; do
-  LSH_RPATH_ADD($d)
-done
-])    
-
-dnl Try to execute a main program, and if it fails, try adding some
-dnl -R flag.
-dnl LSH_RPATH_FIX
-AC_DEFUN([LSH_RPATH_FIX],
-[if test $cross_compiling = no -a "x$RPATHFLAG" != x ; then
-  ac_success=no
-  AC_TRY_RUN([int main(int argc, char **argv) { return 0; }],
-    ac_success=yes, ac_success=no, :)
-  
-  if test $ac_success = no ; then
-    AC_MSG_CHECKING([Running simple test program failed. Trying -R flags])
-dnl echo RPATH_CANDIDATE_DIRS = $RPATH_CANDIDATE_DIRS
-    ac_remaining_dirs=''
-    ac_rpath_save_LDFLAGS="$LDFLAGS"
-    for d in $RPATH_CANDIDATE_DIRS ; do
-      if test $ac_success = yes ; then
-  	ac_remaining_dirs="$ac_remaining_dirs $d"
-      else
-  	LDFLAGS="$RPATHFLAG$d $LDFLAGS"
-dnl echo LDFLAGS = $LDFLAGS
-  	AC_TRY_RUN([int main(int argc, char **argv) { return 0; }],
-  	  [ac_success=yes
-  	  ac_rpath_save_LDFLAGS="$LDFLAGS"
-  	  AC_MSG_RESULT([adding $RPATHFLAG$d])
-  	  ],
-  	  [ac_remaining_dirs="$ac_remaining_dirs $d"], :)
-  	LDFLAGS="$ac_rpath_save_LDFLAGS"
-      fi
-    done
-    RPATH_CANDIDATE_DIRS=$ac_remaining_dirs
-  fi
-  if test $ac_success = no ; then
-    AC_MSG_RESULT(failed)
-  fi
-fi
-])
-
-dnl Like AC_CHECK_LIB, but uses $KRB_LIBS rather than $LIBS.
-dnl LSH_CHECK_KRB_LIB(LIBRARY, FUNCTION, [, ACTION-IF-FOUND [,
-dnl                  ACTION-IF-NOT-FOUND [, OTHER-LIBRARIES]]])
-
-AC_DEFUN([LSH_CHECK_KRB_LIB],
-[AC_CHECK_LIB([$1], [$2],
-  ifelse([$3], ,
-      [[ac_tr_lib=HAVE_LIB`echo $1 | sed -e 's/[^a-zA-Z0-9_]/_/g' \
-     	    -e 'y/abcdefghijklmnopqrstuvwxyz/ABCDEFGHIJKLMNOPQRSTUVWXYZ/'`
-        AC_DEFINE_UNQUOTED($ac_tr_lib)
-        KRB_LIBS="-l$1 $KRB_LIBS"
-      ]], [$3]),
-  ifelse([$4], , , [$4
-])dnl
-, [$5 $KRB_LIBS])
-])
-
-dnl LSH_LIB_ARGP(ACTION-IF-OK, ACTION-IF-BAD)
-AC_DEFUN([LSH_LIB_ARGP],
-[ ac_argp_save_LIBS="$LIBS"
-  ac_argp_save_LDFLAGS="$LDFLAGS"
-  ac_argp_ok=no
-  # First check if we can link with argp.
-  AC_SEARCH_LIBS(argp_parse, argp,
-  [ LSH_RPATH_FIX
-    AC_CACHE_CHECK([for working argp],
-      lsh_cv_lib_argp_works,
-      [ AC_TRY_RUN(
-[#include <argp.h>
-#include <stdlib.h>
-
-static const struct argp_option
-options[] =
-{
-  { NULL, 0, NULL, 0, NULL, 0 }
-};
-
-struct child_state
-{
-  int n;
-};
-
-static error_t
-child_parser(int key, char *arg, struct argp_state *state)
-{
-  struct child_state *input = (struct child_state *) state->input;
-  
-  switch(key)
-    {
-    default:
-      return ARGP_ERR_UNKNOWN;
-    case ARGP_KEY_END:
-      if (!input->n)
-	input->n = 1;
-      break;
-    }
-  return 0;
-}
-
-const struct argp child_argp =
-{
-  options,
-  child_parser,
-  NULL, NULL, NULL, NULL, NULL
-};
-
-struct main_state
-{
-  struct child_state child;
-  int m;
-};
-
-static error_t
-main_parser(int key, char *arg, struct argp_state *state)
-{
-  struct main_state *input = (struct main_state *) state->input;
-
-  switch(key)
-    {
-    default:
-      return ARGP_ERR_UNKNOWN;
-    case ARGP_KEY_INIT:
-      state->child_inputs[0] = &input->child;
-      break;
-    case ARGP_KEY_END:
-      if (!input->m)
-	input->m = input->child.n;
-      
-      break;
-    }
-  return 0;
-}
-
-static const struct argp_child
-main_children[] =
-{
-  { &child_argp, 0, "", 0 },
-  { NULL, 0, NULL, 0}
-};
-
-static const struct argp
-main_argp =
-{ options, main_parser, 
-  NULL,
-  NULL,
-  main_children,
-  NULL, NULL
-};
-
-int main(int argc, char **argv)
-{
-  struct main_state input = { { 0 }, 0 };
-  char *v[2] = { "foo", NULL };
-
-  argp_parse(&main_argp, 1, v, 0, NULL, &input);
-
-  if ( (input.m == 1) && (input.child.n == 1) )
-    return 0;
-  else
-    return 1;
-}
-], lsh_cv_lib_argp_works=yes,
-   lsh_cv_lib_argp_works=no,
-   lsh_cv_lib_argp_works=no)])
-
-  if test x$lsh_cv_lib_argp_works = xyes ; then
-    ac_argp_ok=yes
-  else
-    # Reset link flags
-    LIBS="$ac_argp_save_LIBS"
-    LDFLAGS="$ac_argp_save_LDFLAGS"
-  fi])
-
-  if test x$ac_argp_ok = xyes ; then
-    ifelse([$1],, true, [$1])
-  else
-    ifelse([$2],, true, [$2])
-  fi   
-])
-
-dnl LSH_GCC_ATTRIBUTES
-dnl Check for gcc's __attribute__ construction
-
-AC_DEFUN([LSH_GCC_ATTRIBUTES],
-[AC_CACHE_CHECK(for __attribute__,
-	       lsh_cv_c_attribute,
-[ AC_TRY_COMPILE([
-#include <stdlib.h>
-],
-[
-static void foo(void) __attribute__ ((noreturn));
-
-static void __attribute__ ((noreturn))
-foo(void)
-{
-  exit(1);
-}
-],
-lsh_cv_c_attribute=yes,
-lsh_cv_c_attribute=no)])
-
-AH_TEMPLATE([HAVE_GCC_ATTRIBUTE], [Define if the compiler understands __attribute__])
-if test "x$lsh_cv_c_attribute" = "xyes"; then
-  AC_DEFINE(HAVE_GCC_ATTRIBUTE)
-fi
-
-AH_BOTTOM(
-[#if __GNUC__ || HAVE_GCC_ATTRIBUTE
-# define NORETURN __attribute__ ((__noreturn__))
-# define PRINTF_STYLE(f, a) __attribute__ ((__format__ (__printf__, f, a)))
-# define UNUSED __attribute__ ((__unused__))
-#else
-# define NORETURN
-# define PRINTF_STYLE(f, a)
-# define UNUSED
-#endif
-])])
-
-AC_DEFUN([LSH_GCC_FUNCTION_NAME],
-[# Check for gcc's __FUNCTION__ variable
-AH_TEMPLATE([HAVE_GCC_FUNCTION],
-	    [Define if the compiler understands __FUNCTION__])
-AH_BOTTOM(
-[#if HAVE_GCC_FUNCTION
-# define FUNCTION_NAME __FUNCTION__
-#else
-# define FUNCTION_NAME "Unknown"
-#endif
-])
-
-AC_CACHE_CHECK(for __FUNCTION__,
-	       lsh_cv_c_FUNCTION,
-  [ AC_TRY_COMPILE(,
-      [ #if __GNUC__ == 3
-	#  error __FUNCTION__ is broken in gcc-3
-	#endif
-        void foo(void) { char c = __FUNCTION__[0]; } ],
-      lsh_cv_c_FUNCTION=yes,
-      lsh_cv_c_FUNCTION=no)])
-
-if test "x$lsh_cv_c_FUNCTION" = "xyes"; then
-  AC_DEFINE(HAVE_GCC_FUNCTION)
-fi
-])
-
-# Check for alloca, and include the standard blurb in config.h
-AC_DEFUN([LSH_FUNC_ALLOCA],
-[AC_FUNC_ALLOCA
-AC_CHECK_HEADERS([malloc.h])
-AH_BOTTOM(
-[/* AIX requires this to be the first thing in the file.  */
-#ifndef __GNUC__
-# if HAVE_ALLOCA_H
-#  include <alloca.h>
-# else
-#  ifdef _AIX
- #pragma alloca
-#  else
-#   ifndef alloca /* predefined by HP cc +Olibcalls */
-char *alloca ();
-#   endif
-#  endif
-# endif
-#else /* defined __GNUC__ */
-# if HAVE_ALLOCA_H
-#  include <alloca.h>
-# endif
-#endif
-/* Needed for alloca on windows */
-#if HAVE_MALLOC_H
-# include <malloc.h>
-#endif
-])])
-
-AC_DEFUN([LSH_FUNC_STRERROR],
-[AC_CHECK_FUNCS(strerror)
-AH_BOTTOM(
-[#if HAVE_STRERROR
-#define STRERROR strerror
-#else
-#define STRERROR(x) (sys_errlist[x])
-#endif
-])])
-
-AC_DEFUN([LSH_FUNC_STRSIGNAL],
-[AC_CHECK_FUNCS(strsignal)
-AC_CHECK_DECLS([sys_siglist, _sys_siglist])
-AH_BOTTOM(
-[#if HAVE_STRSIGNAL
-# define STRSIGNAL strsignal
-#else /* !HAVE_STRSIGNAL */
-# if HAVE_DECL_SYS_SIGLIST
-#  define STRSIGNAL(x) (sys_siglist[x])
-# else
-#  if HAVE_DECL__SYS_SIGLIST
-#   define STRSIGNAL(x) (_sys_siglist[x])
-#  else
-#   define STRSIGNAL(x) "Unknown signal"
-#   if __GNUC__
-#    warning Using dummy STRSIGNAL
-#   endif
-#  endif
-# endif
-#endif /* !HAVE_STRSIGNAL */
-])])
-
-dnl LSH_MAKE_CONDITIONAL(symbol, test)
-AC_DEFUN([LSH_MAKE_CONDITIONAL],
-[if $2 ; then
-  IF_$1=''
-  UNLESS_$1='# '
-else
-  IF_$1='# '
-  UNLESS_$1=''
-fi 
-AC_SUBST(IF_$1)
-AC_SUBST(UNLESS_$1)])
-
-dnl LSH_DEPENDENCY_TRACKING
-
-dnl Defines compiler flags DEP_FLAGS to generate dependency
-dnl information, and DEP_PROCESS that is any shell commands needed for
-dnl massaging the dependency information further. Dependencies are
-dnl generated as a side effect of compilation. Dependency files
-dnl themselves are not treated as targets.
-
-AC_DEFUN([LSH_DEPENDENCY_TRACKING],
-[AC_ARG_ENABLE(dependency_tracking,
-  AC_HELP_STRING([--disable-dependency-tracking],
-    [Disable dependency tracking. Dependency tracking doesn't work with BSD make]),,
-  [enable_dependency_tracking=yes])
-
-DEP_FLAGS=''
-DEP_PROCESS='true'
-if test x$enable_dependency_tracking = xyes ; then
-  if test x$GCC = xyes ; then
-    gcc_version=`gcc --version | head -1`
-    case "$gcc_version" in
-      2.*|*[[!0-9.]]2.*)
-        enable_dependency_tracking=no
-        AC_MSG_WARN([Dependency tracking disabled, gcc-3.x is needed])
-      ;;
-      *)
-        DEP_FLAGS='-MT $[]@ -MD -MP -MF $[]@.d'
-        DEP_PROCESS='true'
-      ;;
-    esac
-  else
-    enable_dependency_tracking=no
-    AC_MSG_WARN([Dependency tracking disabled])
-  fi
-fi
-
-if test x$enable_dependency_tracking = xyes ; then
-  DEP_INCLUDE='include '
-else
-  DEP_INCLUDE='# '
-fi
-
-AC_SUBST([DEP_INCLUDE])
-AC_SUBST([DEP_FLAGS])
-AC_SUBST([DEP_PROCESS])])
-
-dnl @synopsis AX_CREATE_STDINT_H [( HEADER-TO-GENERATE [, HEADERS-TO-CHECK])]
-dnl
-dnl the "ISO C9X: 7.18 Integer types <stdint.h>" section requires the
-dnl existence of an include file <stdint.h> that defines a set of 
-dnl typedefs, especially uint8_t,int32_t,uintptr_t.
-dnl Many older installations will not provide this file, but some will
-dnl have the very same definitions in <inttypes.h>. In other enviroments
-dnl we can use the inet-types in <sys/types.h> which would define the
-dnl typedefs int8_t and u_int8_t respectivly.
-dnl
-dnl This macros will create a local "_stdint.h" or the headerfile given as 
-dnl an argument. In many cases that file will just "#include <stdint.h>" 
-dnl or "#include <inttypes.h>", while in other environments it will provide 
-dnl the set of basic 'stdint's definitions/typedefs: 
-dnl   int8_t,uint8_t,int16_t,uint16_t,int32_t,uint32_t,intptr_t,uintptr_t
-dnl   int_least32_t.. int_fast32_t.. intmax_t
-dnl which may or may not rely on the definitions of other files,
-dnl or using the AC_CHECK_SIZEOF macro to determine the actual
-dnl sizeof each type.
-dnl
-dnl if your header files require the stdint-types you will want to create an
-dnl installable file mylib-int.h that all your other installable header
-dnl may include. So if you have a library package named "mylib", just use
-dnl      AX_CREATE_STDINT_H(mylib-int.h) 
-dnl in configure.ac and go to install that very header file in Makefile.am
-dnl along with the other headers (mylib.h) - and the mylib-specific headers
-dnl can simply use "#include <mylib-int.h>" to obtain the stdint-types.
-dnl
-dnl Remember, if the system already had a valid <stdint.h>, the generated
-dnl file will include it directly. No need for fuzzy HAVE_STDINT_H things...
-dnl
-dnl @, (status: used on new platforms) (see http://ac-archive.sf.net/gstdint/)
-dnl @version $Id: acinclude.m4,v 1.27 2004/11/23 21:27:35 nisse Exp $
-dnl @author  Guido Draheim <guidod@gmx.de> 
-
-AC_DEFUN([AX_CREATE_STDINT_H],
-[# ------ AX CREATE STDINT H -------------------------------------
-AC_MSG_CHECKING([for stdint types])
-ac_stdint_h=`echo ifelse($1, , _stdint.h, $1)`
-# try to shortcircuit - if the default include path of the compiler
-# can find a "stdint.h" header then we assume that all compilers can.
-AC_CACHE_VAL([ac_cv_header_stdint_t],[
-old_CXXFLAGS="$CXXFLAGS" ; CXXFLAGS=""
-old_CPPFLAGS="$CPPFLAGS" ; CPPFLAGS=""
-old_CFLAGS="$CFLAGS"     ; CFLAGS=""
-AC_TRY_COMPILE([#include <stdint.h>],[int_least32_t v = 0;],
-[ac_cv_stdint_result="(assuming C99 compatible system)"
- ac_cv_header_stdint_t="stdint.h"; ],
-[ac_cv_header_stdint_t=""])
-CXXFLAGS="$old_CXXFLAGS"
-CPPFLAGS="$old_CPPFLAGS"
-CFLAGS="$old_CFLAGS" ])
-
-v="... $ac_cv_header_stdint_h"
-if test "$ac_stdint_h" = "stdint.h" ; then
- AC_MSG_RESULT([(are you sure you want them in ./stdint.h?)])
-elif test "$ac_stdint_h" = "inttypes.h" ; then
- AC_MSG_RESULT([(are you sure you want them in ./inttypes.h?)])
-elif test "_$ac_cv_header_stdint_t" = "_" ; then
- AC_MSG_RESULT([(putting them into $ac_stdint_h)$v])
-else
- ac_cv_header_stdint="$ac_cv_header_stdint_t"
- AC_MSG_RESULT([$ac_cv_header_stdint (shortcircuit)])
-fi
-
-if test "_$ac_cv_header_stdint_t" = "_" ; then # can not shortcircuit..
-
-dnl .....intro message done, now do a few system checks.....
-dnl btw, all CHECK_TYPE macros do automatically "DEFINE" a type, therefore
-dnl we use the autoconf implementation detail _AC CHECK_TYPE_NEW instead
-
-inttype_headers=`echo $2 | sed -e 's/,/ /g'`
-
-ac_cv_stdint_result="(no helpful system typedefs seen)"
-AC_CACHE_CHECK([for stdint uintptr_t], [ac_cv_header_stdint_x],[
- ac_cv_header_stdint_x="" # the 1997 typedefs (inttypes.h)
-  AC_MSG_RESULT([(..)])
-  for i in stdint.h inttypes.h sys/inttypes.h $inttype_headers ; do
-   unset ac_cv_type_uintptr_t 
-   unset ac_cv_type_uint64_t
-   _AC_CHECK_TYPE_NEW(uintptr_t,[ac_cv_header_stdint_x=$i],dnl
-     continue,[#include <$i>])
-   AC_CHECK_TYPE(uint64_t,[and64="/uint64_t"],[and64=""],[#include<$i>])
-   ac_cv_stdint_result="(seen uintptr_t$and64 in $i)"
-   break;
-  done
-  AC_MSG_CHECKING([for stdint uintptr_t])
- ])
-
-if test "_$ac_cv_header_stdint_x" = "_" ; then
-AC_CACHE_CHECK([for stdint uint32_t], [ac_cv_header_stdint_o],[
- ac_cv_header_stdint_o="" # the 1995 typedefs (sys/inttypes.h)
-  AC_MSG_RESULT([(..)])
-  for i in inttypes.h sys/inttypes.h stdint.h $inttype_headers ; do
-   unset ac_cv_type_uint32_t
-   unset ac_cv_type_uint64_t
-   AC_CHECK_TYPE(uint32_t,[ac_cv_header_stdint_o=$i],dnl
-     continue,[#include <$i>])
-   AC_CHECK_TYPE(uint64_t,[and64="/uint64_t"],[and64=""],[#include<$i>])
-   ac_cv_stdint_result="(seen uint32_t$and64 in $i)"
-   break;
-  done
-  AC_MSG_CHECKING([for stdint uint32_t])
- ])
-fi
-
-if test "_$ac_cv_header_stdint_x" = "_" ; then
-if test "_$ac_cv_header_stdint_o" = "_" ; then
-AC_CACHE_CHECK([for stdint u_int32_t], [ac_cv_header_stdint_u],[
- ac_cv_header_stdint_u="" # the BSD typedefs (sys/types.h)
-  AC_MSG_RESULT([(..)])
-  for i in sys/types.h inttypes.h sys/inttypes.h $inttype_headers ; do
-   unset ac_cv_type_u_int32_t
-   unset ac_cv_type_u_int64_t
-   AC_CHECK_TYPE(u_int32_t,[ac_cv_header_stdint_u=$i],dnl
-     continue,[#include <$i>])
-   AC_CHECK_TYPE(u_int64_t,[and64="/u_int64_t"],[and64=""],[#include<$i>])
-   ac_cv_stdint_result="(seen u_int32_t$and64 in $i)"
-   break;
-  done
-  AC_MSG_CHECKING([for stdint u_int32_t])
- ])
-fi fi
-
-dnl if there was no good C99 header file, do some typedef checks...
-if test "_$ac_cv_header_stdint_x" = "_" ; then
-   AC_MSG_CHECKING([for stdint datatype model])
-   AC_MSG_RESULT([(..)])
-   AC_CHECK_SIZEOF(char)
-   AC_CHECK_SIZEOF(short)
-   AC_CHECK_SIZEOF(int)
-   AC_CHECK_SIZEOF(long)
-   AC_CHECK_SIZEOF(void*)
-   ac_cv_stdint_char_model=""
-   ac_cv_stdint_char_model="$ac_cv_stdint_char_model$ac_cv_sizeof_char"
-   ac_cv_stdint_char_model="$ac_cv_stdint_char_model$ac_cv_sizeof_short"
-   ac_cv_stdint_char_model="$ac_cv_stdint_char_model$ac_cv_sizeof_int"
-   ac_cv_stdint_long_model=""
-   ac_cv_stdint_long_model="$ac_cv_stdint_long_model$ac_cv_sizeof_int"
-   ac_cv_stdint_long_model="$ac_cv_stdint_long_model$ac_cv_sizeof_long"
-   ac_cv_stdint_long_model="$ac_cv_stdint_long_model$ac_cv_sizeof_voidp"
-   name="$ac_cv_stdint_long_model"
-   case "$ac_cv_stdint_char_model/$ac_cv_stdint_long_model" in
-    122/242)     name="$name,  IP16 (standard 16bit machine)" ;;
-    122/244)     name="$name,  LP32 (standard 32bit mac/win)" ;;
-    122/*)       name="$name        (unusual int16 model)" ;; 
-    124/444)     name="$name, ILP32 (standard 32bit unixish)" ;;
-    124/488)     name="$name,  LP64 (standard 64bit unixish)" ;;
-    124/448)     name="$name, LLP64 (unusual  64bit unixish)" ;;
-    124/*)       name="$name        (unusual int32 model)" ;; 
-    128/888)     name="$name, ILP64 (unusual  64bit numeric)" ;;
-    128/*)       name="$name        (unusual int64 model)" ;; 
-    222/*|444/*) name="$name        (unusual dsptype)" ;;
-     *)          name="$name        (very unusal model)" ;;
-   esac
-   AC_MSG_RESULT([combined for stdint datatype model...  $name])
-fi
-
-if test "_$ac_cv_header_stdint_x" != "_" ; then
-   ac_cv_header_stdint="$ac_cv_header_stdint_x"
-elif  test "_$ac_cv_header_stdint_o" != "_" ; then
-   ac_cv_header_stdint="$ac_cv_header_stdint_o"
-elif  test "_$ac_cv_header_stdint_u" != "_" ; then
-   ac_cv_header_stdint="$ac_cv_header_stdint_u"
-else
-   ac_cv_header_stdint="stddef.h"
-fi
-
-AC_MSG_CHECKING([for extra inttypes in chosen header])
-AC_MSG_RESULT([($ac_cv_header_stdint)])
-dnl see if int_least and int_fast types are present in _this_ header.
-unset ac_cv_type_int_least32_t
-unset ac_cv_type_int_fast32_t
-AC_CHECK_TYPE(int_least32_t,,,[#include <$ac_cv_header_stdint>])
-AC_CHECK_TYPE(int_fast32_t,,,[#include<$ac_cv_header_stdint>])
-AC_CHECK_TYPE(intmax_t,,,[#include <$ac_cv_header_stdint>])
-
-fi # shortcircut to system "stdint.h"
-# ------------------ PREPARE VARIABLES ------------------------------
-if test "$GCC" = "yes" ; then
-ac_cv_stdint_message="using gnu compiler "`$CC --version | head -1` 
-else
-ac_cv_stdint_message="using $CC"
-fi
-
-AC_MSG_RESULT([make use of $ac_cv_header_stdint in $ac_stdint_h dnl
-$ac_cv_stdint_result])
-
-# ----------------- DONE inttypes.h checks START header -------------
-AC_CONFIG_COMMANDS([$ac_stdint_h],[
-AC_MSG_NOTICE(creating $ac_stdint_h : $_ac_stdint_h)
-ac_stdint=$tmp/_stdint.h
-
-echo "#ifndef" $_ac_stdint_h >$ac_stdint
-echo "#define" $_ac_stdint_h "1" >>$ac_stdint
-echo "#ifndef" _GENERATED_STDINT_H >>$ac_stdint
-echo "#define" _GENERATED_STDINT_H '"'$PACKAGE $VERSION'"' >>$ac_stdint
-echo "/* generated $ac_cv_stdint_message */" >>$ac_stdint
-if test "_$ac_cv_header_stdint_t" != "_" ; then 
-echo "#define _STDINT_HAVE_STDINT_H" "1" >>$ac_stdint
-fi
-
-cat >>$ac_stdint <<STDINT_EOF
-
-/* ................... shortcircuit part ........................... */
-
-#if defined HAVE_STDINT_H || defined _STDINT_HAVE_STDINT_H
-#include <stdint.h>
-#else
-#include <stddef.h>
-
-/* .................... configured part ............................ */
-
-STDINT_EOF
-
-echo "/* whether we have a C99 compatible stdint header file */" >>$ac_stdint
-if test "_$ac_cv_header_stdint_x" != "_" ; then
-  ac_header="$ac_cv_header_stdint_x"
-  echo "#define _STDINT_HEADER_INTPTR" '"'"$ac_header"'"' >>$ac_stdint
-else
-  echo "/* #undef _STDINT_HEADER_INTPTR */" >>$ac_stdint
-fi
-
-echo "/* whether we have a C96 compatible inttypes header file */" >>$ac_stdint
-if  test "_$ac_cv_header_stdint_o" != "_" ; then
-  ac_header="$ac_cv_header_stdint_o"
-  echo "#define _STDINT_HEADER_UINT32" '"'"$ac_header"'"' >>$ac_stdint
-else
-  echo "/* #undef _STDINT_HEADER_UINT32 */" >>$ac_stdint
-fi
-
-echo "/* whether we have a BSD compatible inet types header */" >>$ac_stdint
-if  test "_$ac_cv_header_stdint_u" != "_" ; then
-  ac_header="$ac_cv_header_stdint_u"
-  echo "#define _STDINT_HEADER_U_INT32" '"'"$ac_header"'"' >>$ac_stdint
-else
-  echo "/* #undef _STDINT_HEADER_U_INT32 */" >>$ac_stdint
-fi
-
-echo "" >>$ac_stdint
-
-if test "_$ac_header" != "_" ; then if test "$ac_header" != "stddef.h" ; then
-  echo "#include <$ac_header>" >>$ac_stdint
-  echo "" >>$ac_stdint
-fi fi
-
-echo "/* which 64bit typedef has been found */" >>$ac_stdint
-if test "$ac_cv_type_uint64_t" = "yes" ; then
-echo "#define   _STDINT_HAVE_UINT64_T" "1"  >>$ac_stdint
-else
-echo "/* #undef _STDINT_HAVE_UINT64_T */" >>$ac_stdint
-fi
-if test "$ac_cv_type_u_int64_t" = "yes" ; then
-echo "#define   _STDINT_HAVE_U_INT64_T" "1"  >>$ac_stdint
-else
-echo "/* #undef _STDINT_HAVE_U_INT64_T */" >>$ac_stdint
-fi
-echo "" >>$ac_stdint
-
-echo "/* which type model has been detected */" >>$ac_stdint
-if test "_$ac_cv_stdint_char_model" != "_" ; then
-echo "#define   _STDINT_CHAR_MODEL" "$ac_cv_stdint_char_model" >>$ac_stdint
-echo "#define   _STDINT_LONG_MODEL" "$ac_cv_stdint_long_model" >>$ac_stdint
-else
-echo "/* #undef _STDINT_CHAR_MODEL // skipped */" >>$ac_stdint
-echo "/* #undef _STDINT_LONG_MODEL // skipped */" >>$ac_stdint
-fi
-echo "" >>$ac_stdint
-
-echo "/* whether int_least types were detected */" >>$ac_stdint
-if test "$ac_cv_type_int_least32_t" = "yes"; then
-echo "#define   _STDINT_HAVE_INT_LEAST32_T" "1"  >>$ac_stdint
-else
-echo "/* #undef _STDINT_HAVE_INT_LEAST32_T */" >>$ac_stdint
-fi
-echo "/* whether int_fast types were detected */" >>$ac_stdint
-if test "$ac_cv_type_int_fast32_t" = "yes"; then
-echo "#define   _STDINT_HAVE_INT_FAST32_T" "1" >>$ac_stdint
-else
-echo "/* #undef _STDINT_HAVE_INT_FAST32_T */" >>$ac_stdint
-fi
-echo "/* whether intmax_t type was detected */" >>$ac_stdint
-if test "$ac_cv_type_intmax_t" = "yes"; then
-echo "#define   _STDINT_HAVE_INTMAX_T" "1" >>$ac_stdint
-else
-echo "/* #undef _STDINT_HAVE_INTMAX_T */" >>$ac_stdint
-fi
-echo "" >>$ac_stdint
-
-  cat >>$ac_stdint <<STDINT_EOF
-/* .................... detections part ............................ */
-
-/* whether we need to define bitspecific types from compiler base types */
-#ifndef _STDINT_HEADER_INTPTR
-#ifndef _STDINT_HEADER_UINT32
-#ifndef _STDINT_HEADER_U_INT32
-#define _STDINT_NEED_INT_MODEL_T
-#else
-#define _STDINT_HAVE_U_INT_TYPES
-#endif
-#endif
-#endif
-
-#ifdef _STDINT_HAVE_U_INT_TYPES
-#undef _STDINT_NEED_INT_MODEL_T
-#endif
-
-#ifdef  _STDINT_CHAR_MODEL
-#if     _STDINT_CHAR_MODEL+0 == 122 || _STDINT_CHAR_MODEL+0 == 124
-#ifndef _STDINT_BYTE_MODEL
-#define _STDINT_BYTE_MODEL 12
-#endif
-#endif
-#endif
-
-#ifndef _STDINT_HAVE_INT_LEAST32_T
-#define _STDINT_NEED_INT_LEAST_T
-#endif
-
-#ifndef _STDINT_HAVE_INT_FAST32_T
-#define _STDINT_NEED_INT_FAST_T
-#endif
-
-#ifndef _STDINT_HEADER_INTPTR
-#define _STDINT_NEED_INTPTR_T
-#ifndef _STDINT_HAVE_INTMAX_T
-#define _STDINT_NEED_INTMAX_T
-#endif
-#endif
-
-
-/* .................... definition part ............................ */
-
-/* some system headers have good uint64_t */
-#ifndef _HAVE_UINT64_T
-#if     defined _STDINT_HAVE_UINT64_T  || defined HAVE_UINT64_T
-#define _HAVE_UINT64_T
-#elif   defined _STDINT_HAVE_U_INT64_T || defined HAVE_U_INT64_T
-#define _HAVE_UINT64_T
-typedef u_int64_t uint64_t;
-#endif
-#endif
-
-#ifndef _HAVE_UINT64_T
-/* .. here are some common heuristics using compiler runtime specifics */
-#if defined __STDC_VERSION__ && defined __STDC_VERSION__ >= 199901L
-#define _HAVE_UINT64_T
-typedef long long int64_t;
-typedef unsigned long long uint64_t;
-
-#elif !defined __STRICT_ANSI__
-#if defined _MSC_VER || defined __WATCOMC__ || defined __BORLANDC__
-#define _HAVE_UINT64_T
-typedef __int64 int64_t;
-typedef unsigned __int64 uint64_t;
-
-#elif defined __GNUC__ || defined __MWERKS__ || defined __ELF__
-/* note: all ELF-systems seem to have loff-support which needs 64-bit */
-#if !defined _NO_LONGLONG
-#define _HAVE_UINT64_T
-typedef long long int64_t;
-typedef unsigned long long uint64_t;
-#endif
-
-#elif defined __alpha || (defined __mips && defined _ABIN32)
-#if !defined _NO_LONGLONG
-typedef long int64_t;
-typedef unsigned long uint64_t;
-#endif
-  /* compiler/cpu type to define int64_t */
-#endif
-#endif
-#endif
-
-#if defined _STDINT_HAVE_U_INT_TYPES
-/* int8_t int16_t int32_t defined by inet code, redeclare the u_intXX types */
-typedef u_int8_t uint8_t;
-typedef u_int16_t uint16_t;
-typedef u_int32_t uint32_t;
-
-/* glibc compatibility */
-#ifndef __int8_t_defined
-#define __int8_t_defined
-#endif
-#endif
-
-#ifdef _STDINT_NEED_INT_MODEL_T
-/* we must guess all the basic types. Apart from byte-adressable system, */
-/* there a few 32-bit-only dsp-systems that we guard with BYTE_MODEL 8-} */
-/* (btw, those nibble-addressable systems are way off, or so we assume) */
-
-dnl   /* have a look at "64bit and data size neutrality" at */
-dnl   /* http://unix.org/version2/whatsnew/login_64bit.html */
-dnl   /* (the shorthand "ILP" types always have a "P" part) */
-
-#if defined _STDINT_BYTE_MODEL
-#if _STDINT_LONG_MODEL+0 == 242
-/* 2:4:2 =  IP16 = a normal 16-bit system                */
-typedef unsigned char   uint8_t;
-typedef unsigned short  uint16_t;
-typedef unsigned long   uint32_t;
-#ifndef __int8_t_defined
-#define __int8_t_defined
-typedef          char    int8_t;
-typedef          short   int16_t;
-typedef          long    int32_t;
-#endif
-#elif _STDINT_LONG_MODEL+0 == 244 || _STDINT_LONG_MODEL == 444
-/* 2:4:4 =  LP32 = a 32-bit system derived from a 16-bit */
-/* 4:4:4 = ILP32 = a normal 32-bit system                */
-typedef unsigned char   uint8_t;
-typedef unsigned short  uint16_t;
-typedef unsigned int    uint32_t;
-#ifndef __int8_t_defined
-#define __int8_t_defined
-typedef          char    int8_t;
-typedef          short   int16_t;
-typedef          int     int32_t;
-#endif
-#elif _STDINT_LONG_MODEL+0 == 484 || _STDINT_LONG_MODEL+0 == 488
-/* 4:8:4 =  IP32 = a 32-bit system prepared for 64-bit    */
-/* 4:8:8 =  LP64 = a normal 64-bit system                 */
-typedef unsigned char   uint8_t;
-typedef unsigned short  uint16_t;
-typedef unsigned int    uint32_t;
-#ifndef __int8_t_defined
-#define __int8_t_defined
-typedef          char    int8_t;
-typedef          short   int16_t;
-typedef          int     int32_t;
-#endif
-/* this system has a "long" of 64bit */
-#ifndef _HAVE_UINT64_T
-#define _HAVE_UINT64_T
-typedef unsigned long   uint64_t;
-typedef          long    int64_t;
-#endif
-#elif _STDINT_LONG_MODEL+0 == 448
-/*      LLP64   a 64-bit system derived from a 32-bit system */
-typedef unsigned char   uint8_t;
-typedef unsigned short  uint16_t;
-typedef unsigned int    uint32_t;
-#ifndef __int8_t_defined
-#define __int8_t_defined
-typedef          char    int8_t;
-typedef          short   int16_t;
-typedef          int     int32_t;
-#endif
-/* assuming the system has a "long long" */
-#ifndef _HAVE_UINT64_T
-#define _HAVE_UINT64_T
-typedef unsigned long long uint64_t;
-typedef          long long  int64_t;
-#endif
-#else
-#define _STDINT_NO_INT32_T
-#endif
-#else
-#define _STDINT_NO_INT8_T
-#define _STDINT_NO_INT32_T
-#endif
-#endif
-
-/*
- * quote from SunOS-5.8 sys/inttypes.h:
- * Use at your own risk.  As of February 1996, the committee is squarely
- * behind the fixed sized types; the "least" and "fast" types are still being
- * discussed.  The probability that the "fast" types may be removed before
- * the standard is finalized is high enough that they are not currently
- * implemented.
- */
-
-#if defined _STDINT_NEED_INT_LEAST_T
-typedef  int8_t    int_least8_t;
-typedef  int16_t   int_least16_t;
-typedef  int32_t   int_least32_t;
-#ifdef _HAVE_UINT64_T
-typedef  int64_t   int_least64_t;
-#endif
-
-typedef uint8_t   uint_least8_t;
-typedef uint16_t  uint_least16_t;
-typedef uint32_t  uint_least32_t;
-#ifdef _HAVE_UINT64_T
-typedef uint64_t  uint_least64_t;
-#endif
-  /* least types */
-#endif
-
-#if defined _STDINT_NEED_INT_FAST_T
-typedef  int8_t    int_fast8_t; 
-typedef  int       int_fast16_t;
-typedef  int32_t   int_fast32_t;
-#ifdef _HAVE_UINT64_T
-typedef  int64_t   int_fast64_t;
-#endif
-
-typedef uint8_t   uint_fast8_t; 
-typedef unsigned  uint_fast16_t;
-typedef uint32_t  uint_fast32_t;
-#ifdef _HAVE_UINT64_T
-typedef uint64_t  uint_fast64_t;
-#endif
-  /* fast types */
-#endif
-
-#ifdef _STDINT_NEED_INTMAX_T
-#ifdef _HAVE_UINT64_T
-typedef  int64_t       intmax_t;
-typedef uint64_t      uintmax_t;
-#else
-typedef          long  intmax_t;
-typedef unsigned long uintmax_t;
-#endif
-#endif
-
-#ifdef _STDINT_NEED_INTPTR_T
-#ifndef __intptr_t_defined
-#define __intptr_t_defined
-/* we encourage using "long" to store pointer values, never use "int" ! */
-#if   _STDINT_LONG_MODEL+0 == 242 || _STDINT_LONG_MODEL+0 == 484
-typedef  unsinged int   uintptr_t;
-typedef           int    intptr_t;
-#elif _STDINT_LONG_MODEL+0 == 244 || _STDINT_LONG_MODEL+0 == 444
-typedef  unsigned long  uintptr_t;
-typedef           long   intptr_t;
-#elif _STDINT_LONG_MODEL+0 == 448 && defined _HAVE_UINT64_T
-typedef        uint64_t uintptr_t;
-typedef         int64_t  intptr_t;
-#else /* matches typical system types ILP32 and LP64 - but not IP16 or LLP64 */
-typedef  unsigned long  uintptr_t;
-typedef           long   intptr_t;
-#endif
-#endif
-#endif
-
-  /* shortcircuit*/
-#endif
-  /* once */
-#endif
-#endif
-STDINT_EOF
-    if cmp -s $ac_stdint_h $ac_stdint 2>/dev/null; then
-      AC_MSG_NOTICE([$ac_stdint_h is unchanged])
-    else
-      ac_dir=`AS_DIRNAME(["$ac_stdint_h"])`
-      AS_MKDIR_P(["$ac_dir"])
-      rm -f $ac_stdint_h
-      mv $ac_stdint $ac_stdint_h
-    fi
-],[# variables for create stdint.h replacement
-PACKAGE="$PACKAGE"
-VERSION="$VERSION"
-ac_stdint_h="$ac_stdint_h"
-_ac_stdint_h=AS_TR_CPP(_$PACKAGE-$ac_stdint_h)
-ac_cv_stdint_message="$ac_cv_stdint_message"
-ac_cv_header_stdint_t="$ac_cv_header_stdint_t"
-ac_cv_header_stdint_x="$ac_cv_header_stdint_x"
-ac_cv_header_stdint_o="$ac_cv_header_stdint_o"
-ac_cv_header_stdint_u="$ac_cv_header_stdint_u"
-ac_cv_type_uint64_t="$ac_cv_type_uint64_t"
-ac_cv_type_u_int64_t="$ac_cv_type_u_int64_t"
-ac_cv_stdint_char_model="$ac_cv_stdint_char_model"
-ac_cv_stdint_long_model="$ac_cv_stdint_long_model"
-ac_cv_type_int_least32_t="$ac_cv_type_int_least32_t"
-ac_cv_type_int_fast32_t="$ac_cv_type_int_fast32_t"
-ac_cv_type_intmax_t="$ac_cv_type_intmax_t"
-])
-])
diff --git a/contrib/argp-standalone/argp-ba.c b/contrib/argp-standalone/argp-ba.c
deleted file mode 100644
index 0d3958c1151..00000000000
--- a/contrib/argp-standalone/argp-ba.c
+++ /dev/null
@@ -1,26 +0,0 @@
-/* Default definition for ARGP_PROGRAM_BUG_ADDRESS.
-   Copyright (C) 1996, 1997, 1999, 2004 Free Software Foundation, Inc.
-   This file is part of the GNU C Library.
-   Written by Miles Bader <miles@gnu.ai.mit.edu>.
-
-   The GNU C Library is free software; you can redistribute it and/or
-   modify it under the terms of the GNU Library General Public License as
-   published by the Free Software Foundation; either version 2 of the
-   License, or (at your option) any later version.
-
-   The GNU C Library is distributed in the hope that it will be useful,
-   but WITHOUT ANY WARRANTY; without even the implied warranty of
-   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-   Library General Public License for more details.
-
-   You should have received a copy of the GNU Library General Public
-   License along with the GNU C Library; see the file COPYING.LIB.  If not,
-   write to the Free Software Foundation, Inc., 59 Temple Place - Suite 330,
-   Boston, MA 02111-1307, USA.  */
-
-/* If set by the user program, it should point to string that is the
-   bug-reporting address for the program.  It will be printed by argp_help if
-   the ARGP_HELP_BUG_ADDR flag is set (as it is by various standard help
-   messages), embedded in a sentence that says something like `Report bugs to
-   ADDR.'.  */
-const char *argp_program_bug_address = 0;
diff --git a/contrib/argp-standalone/argp-eexst.c b/contrib/argp-standalone/argp-eexst.c
deleted file mode 100644
index 46b27847ad4..00000000000
--- a/contrib/argp-standalone/argp-eexst.c
+++ /dev/null
@@ -1,36 +0,0 @@
-/* Default definition for ARGP_ERR_EXIT_STATUS
-   Copyright (C) 1997 Free Software Foundation, Inc.
-   This file is part of the GNU C Library.
-   Written by Miles Bader <miles@gnu.ai.mit.edu>.
-
-   The GNU C Library is free software; you can redistribute it and/or
-   modify it under the terms of the GNU Library General Public License as
-   published by the Free Software Foundation; either version 2 of the
-   License, or (at your option) any later version.
-
-   The GNU C Library is distributed in the hope that it will be useful,
-   but WITHOUT ANY WARRANTY; without even the implied warranty of
-   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-   Library General Public License for more details.
-
-   You should have received a copy of the GNU Library General Public
-   License along with the GNU C Library; see the file COPYING.LIB.  If not,
-   write to the Free Software Foundation, Inc., 59 Temple Place - Suite 330,
-   Boston, MA 02111-1307, USA.  */
-
-#ifdef HAVE_CONFIG_H
-#include <config.h>
-#endif
-
-#if HAVE_SYSEXITS_H
-# include <sysexits.h>
-#else
-# define EX_USAGE 64
-#endif
-
-#include "argp.h"
-
-/* The exit status that argp will use when exiting due to a parsing error.
-   If not defined or set by the user program, this defaults to EX_USAGE from
-   <sysexits.h>.  */
-error_t argp_err_exit_status = EX_USAGE;
diff --git a/contrib/argp-standalone/argp-fmtstream.c b/contrib/argp-standalone/argp-fmtstream.c
deleted file mode 100644
index 494b6b31d12..00000000000
--- a/contrib/argp-standalone/argp-fmtstream.c
+++ /dev/null
@@ -1,477 +0,0 @@
-/* Word-wrapping and line-truncating streams
-   Copyright (C) 1997, 1998, 1999, 2001 Free Software Foundation, Inc.
-   This file is part of the GNU C Library.
-   Written by Miles Bader <miles@gnu.ai.mit.edu>.
-
-   The GNU C Library is free software; you can redistribute it and/or
-   modify it under the terms of the GNU Library General Public License as
-   published by the Free Software Foundation; either version 2 of the
-   License, or (at your option) any later version.
-
-   The GNU C Library is distributed in the hope that it will be useful,
-   but WITHOUT ANY WARRANTY; without even the implied warranty of
-   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-   Library General Public License for more details.
-
-   You should have received a copy of the GNU Library General Public
-   License along with the GNU C Library; see the file COPYING.LIB.  If not,
-   write to the Free Software Foundation, Inc., 59 Temple Place - Suite 330,
-   Boston, MA 02111-1307, USA.  */
-
-/* This package emulates glibc `line_wrap_stream' semantics for systems that
-   don't have that.  */
-
-#ifdef HAVE_CONFIG_H
-#include <config.h>
-#endif
-
-#include <stdlib.h>
-#include <string.h>
-#include <errno.h>
-#include <stdarg.h>
-#include <ctype.h>
-
-#include "argp-fmtstream.h"
-#include "argp-namefrob.h"
-
-#ifndef ARGP_FMTSTREAM_USE_LINEWRAP
-
-#ifndef isblank
-#define isblank(ch) ((ch)==' ' || (ch)=='\t')
-#endif
-
-#if defined _LIBC && defined USE_IN_LIBIO
-# include <libio/libioP.h>
-# define __vsnprintf(s, l, f, a) _IO_vsnprintf (s, l, f, a)
-#endif
-
-#define INIT_BUF_SIZE 200
-#define PRINTF_SIZE_GUESS 150
-
-/* Return an argp_fmtstream that outputs to STREAM, and which prefixes lines
-   written on it with LMARGIN spaces and limits them to RMARGIN columns
-   total.  If WMARGIN >= 0, words that extend past RMARGIN are wrapped by
-   replacing the whitespace before them with a newline and WMARGIN spaces.
-   Otherwise, chars beyond RMARGIN are simply dropped until a newline.
-   Returns NULL if there was an error.  */
-argp_fmtstream_t
-__argp_make_fmtstream (FILE *stream,
-		       size_t lmargin, size_t rmargin, ssize_t wmargin)
-{
-  argp_fmtstream_t fs = malloc (sizeof (struct argp_fmtstream));
-  if (fs)
-    {
-      fs->stream = stream;
-
-      fs->lmargin = lmargin;
-      fs->rmargin = rmargin;
-      fs->wmargin = wmargin;
-      fs->point_col = 0;
-      fs->point_offs = 0;
-
-      fs->buf = malloc (INIT_BUF_SIZE);
-      if (! fs->buf)
-	{
-	  free (fs);
-	  fs = 0;
-	}
-      else
-	{
-	  fs->p = fs->buf;
-	  fs->end = fs->buf + INIT_BUF_SIZE;
-	}
-    }
-
-  return fs;
-}
-#ifdef weak_alias
-weak_alias (__argp_make_fmtstream, argp_make_fmtstream)
-#endif
-
-/* Flush FS to its stream, and free it (but don't close the stream).  */
-void
-__argp_fmtstream_free (argp_fmtstream_t fs)
-{
-  __argp_fmtstream_update (fs);
-  if (fs->p > fs->buf)
-    FWRITE_UNLOCKED (fs->buf, 1, fs->p - fs->buf, fs->stream);
-  free (fs->buf);
-  free (fs);
-}
-#ifdef weak_alias
-weak_alias (__argp_fmtstream_free, argp_fmtstream_free)
-#endif
-
-/* Process FS's buffer so that line wrapping is done from POINT_OFFS to the
-   end of its buffer.  This code is mostly from glibc stdio/linewrap.c.  */
-void
-__argp_fmtstream_update (argp_fmtstream_t fs)
-{
-  char *buf, *nl;
-  size_t len;
-
-  /* Scan the buffer for newlines.  */
-  buf = fs->buf + fs->point_offs;
-  while (buf < fs->p)
-    {
-      size_t r;
-
-      if (fs->point_col == 0 && fs->lmargin != 0)
-	{
-	  /* We are starting a new line.  Print spaces to the left margin.  */
-	  const size_t pad = fs->lmargin;
-	  if (fs->p + pad < fs->end)
-	    {
-	      /* We can fit in them in the buffer by moving the
-		 buffer text up and filling in the beginning.  */
-	      memmove (buf + pad, buf, fs->p - buf);
-	      fs->p += pad; /* Compensate for bigger buffer. */
-	      memset (buf, ' ', pad); /* Fill in the spaces.  */
-	      buf += pad; /* Don't bother searching them.  */
-	    }
-	  else
-	    {
-	      /* No buffer space for spaces.  Must flush.  */
-	      size_t i;
-	      for (i = 0; i < pad; i++)
-		PUTC_UNLOCKED (' ', fs->stream);
-	    }
-	  fs->point_col = pad;
-	}
-
-      len = fs->p - buf;
-      nl = memchr (buf, '\n', len);
-
-      if (fs->point_col < 0)
-	fs->point_col = 0;
-
-      if (!nl)
-	{
-	  /* The buffer ends in a partial line.  */
-
-	  if (fs->point_col + len < fs->rmargin)
-	    {
-	      /* The remaining buffer text is a partial line and fits
-		 within the maximum line width.  Advance point for the
-		 characters to be written and stop scanning.  */
-	      fs->point_col += len;
-	      break;
-	    }
-	  else
-	    /* Set the end-of-line pointer for the code below to
-	       the end of the buffer.  */
-	    nl = fs->p;
-	}
-      else if (fs->point_col + (nl - buf) < (ssize_t) fs->rmargin)
-	{
-	  /* The buffer contains a full line that fits within the maximum
-	     line width.  Reset point and scan the next line.  */
-	  fs->point_col = 0;
-	  buf = nl + 1;
-	  continue;
-	}
-
-      /* This line is too long.  */
-      r = fs->rmargin - 1;
-
-      if (fs->wmargin < 0)
-	{
-	  /* Truncate the line by overwriting the excess with the
-	     newline and anything after it in the buffer.  */
-	  if (nl < fs->p)
-	    {
-	      memmove (buf + (r - fs->point_col), nl, fs->p - nl);
-	      fs->p -= buf + (r - fs->point_col) - nl;
-	      /* Reset point for the next line and start scanning it.  */
-	      fs->point_col = 0;
-	      buf += r + 1; /* Skip full line plus \n. */
-	    }
-	  else
-	    {
-	      /* The buffer ends with a partial line that is beyond the
-		 maximum line width.  Advance point for the characters
-		 written, and discard those past the max from the buffer.  */
-	      fs->point_col += len;
-	      fs->p -= fs->point_col - r;
-	      break;
-	    }
-	}
-      else
-	{
-	  /* Do word wrap.  Go to the column just past the maximum line
-	     width and scan back for the beginning of the word there.
-	     Then insert a line break.  */
-
-	  char *p, *nextline;
-	  int i;
-
-	  p = buf + (r + 1 - fs->point_col);
-	  while (p >= buf && !isblank (*p))
-	    --p;
-	  nextline = p + 1;	/* This will begin the next line.  */
-
-	  if (nextline > buf)
-	    {
-	      /* Swallow separating blanks.  */
-	      if (p >= buf)
-		do
-		  --p;
-		while (p >= buf && isblank (*p));
-	      nl = p + 1;	/* The newline will replace the first blank. */
-	    }
-	  else
-	    {
-	      /* A single word that is greater than the maximum line width.
-		 Oh well.  Put it on an overlong line by itself.  */
-	      p = buf + (r + 1 - fs->point_col);
-	      /* Find the end of the long word.  */
-	      do
-		++p;
-	      while (p < nl && !isblank (*p));
-	      if (p == nl)
-		{
-		  /* It already ends a line.  No fussing required.  */
-		  fs->point_col = 0;
-		  buf = nl + 1;
-		  continue;
-		}
-	      /* We will move the newline to replace the first blank.  */
-	      nl = p;
-	      /* Swallow separating blanks.  */
-	      do
-		++p;
-	      while (isblank (*p));
-	      /* The next line will start here.  */
-	      nextline = p;
-	    }
-
-	  /* Note: There are a bunch of tests below for
-	     NEXTLINE == BUF + LEN + 1; this case is where NL happens to fall
-	     at the end of the buffer, and NEXTLINE is in fact empty (and so
-	     we need not be careful to maintain its contents).  */
-
-	  if (nextline == buf + len + 1
-	      ? fs->end - nl < fs->wmargin + 1
-	      : nextline - (nl + 1) < fs->wmargin)
-	    {
-	      /* The margin needs more blanks than we removed.  */
-	      if (fs->end - fs->p > fs->wmargin + 1)
-		/* Make some space for them.  */
-		{
-		  size_t mv = fs->p - nextline;
-		  memmove (nl + 1 + fs->wmargin, nextline, mv);
-		  nextline = nl + 1 + fs->wmargin;
-		  len = nextline + mv - buf;
-		  *nl++ = '\n';
-		}
-	      else
-		/* Output the first line so we can use the space.  */
-		{
-		  if (nl > fs->buf)
-		    FWRITE_UNLOCKED (fs->buf, 1, nl - fs->buf, fs->stream);
-		  PUTC_UNLOCKED ('\n', fs->stream);
-		  len += buf - fs->buf;
-		  nl = buf = fs->buf;
-		}
-	    }
-	  else
-	    /* We can fit the newline and blanks in before
-	       the next word.  */
-	    *nl++ = '\n';
-
-	  if (nextline - nl >= fs->wmargin
-	      || (nextline == buf + len + 1 && fs->end - nextline >= fs->wmargin))
-	    /* Add blanks up to the wrap margin column.  */
-	    for (i = 0; i < fs->wmargin; ++i)
-	      *nl++ = ' ';
-	  else
-	    for (i = 0; i < fs->wmargin; ++i)
-	      PUTC_UNLOCKED (' ', fs->stream);
-
-	  /* Copy the tail of the original buffer into the current buffer
-	     position.  */
-	  if (nl < nextline)
-	    memmove (nl, nextline, buf + len - nextline);
-	  len -= nextline - buf;
-
-	  /* Continue the scan on the remaining lines in the buffer.  */
-	  buf = nl;
-
-	  /* Restore bufp to include all the remaining text.  */
-	  fs->p = nl + len;
-
-	  /* Reset the counter of what has been output this line.  If wmargin
-	     is 0, we want to avoid the lmargin getting added, so we set
-	     point_col to a magic value of -1 in that case.  */
-	  fs->point_col = fs->wmargin ? fs->wmargin : -1;
-	}
-    }
-
-  /* Remember that we've scanned as far as the end of the buffer.  */
-  fs->point_offs = fs->p - fs->buf;
-}
-
-/* Ensure that FS has space for AMOUNT more bytes in its buffer, either by
-   growing the buffer, or by flushing it.  True is returned iff we succeed. */
-int
-__argp_fmtstream_ensure (struct argp_fmtstream *fs, size_t amount)
-{
-  if ((size_t) (fs->end - fs->p) < amount)
-    {
-      ssize_t wrote;
-
-      /* Flush FS's buffer.  */
-      __argp_fmtstream_update (fs);
-
-      wrote = FWRITE_UNLOCKED (fs->buf, 1, fs->p - fs->buf, fs->stream);
-      if (wrote == fs->p - fs->buf)
-	{
-	  fs->p = fs->buf;
-	  fs->point_offs = 0;
-	}
-      else
-	{
-	  fs->p -= wrote;
-	  fs->point_offs -= wrote;
-	  memmove (fs->buf, fs->buf + wrote, fs->p - fs->buf);
-	  return 0;
-	}
-
-      if ((size_t) (fs->end - fs->buf) < amount)
-	/* Gotta grow the buffer.  */
-	{
-	  size_t new_size = fs->end - fs->buf + amount;
-	  char *new_buf = realloc (fs->buf, new_size);
-
-	  if (! new_buf)
-	    {
-	      __set_errno (ENOMEM);
-	      return 0;
-	    }
-
-	  fs->buf = new_buf;
-	  fs->end = new_buf + new_size;
-	  fs->p = fs->buf;
-	}
-    }
-
-  return 1;
-}
-
-ssize_t
-__argp_fmtstream_printf (struct argp_fmtstream *fs, const char *fmt, ...)
-{
-  size_t out;
-  size_t avail;
-  size_t size_guess = PRINTF_SIZE_GUESS; /* How much space to reserve. */
-
-  do
-    {
-      va_list args;
-
-      if (! __argp_fmtstream_ensure (fs, size_guess))
-	return -1;
-
-      va_start (args, fmt);
-      avail = fs->end - fs->p;
-      out = __vsnprintf (fs->p, avail, fmt, args);
-      va_end (args);
-      if (out >= avail)
-	size_guess = out + 1;
-    }
-  while (out >= avail);
-
-  fs->p += out;
-
-  return out;
-}
-#ifdef weak_alias
-weak_alias (__argp_fmtstream_printf, argp_fmtstream_printf)
-#endif
-
-#if __STDC_VERSION__ - 199900L < 1
-/* Duplicate the inline definitions in argp-fmtstream.h, for compilers
- * that don't do inlining. */
-size_t
-__argp_fmtstream_write (argp_fmtstream_t __fs,
-			__const char *__str, size_t __len)
-{
-  if (__fs->p + __len <= __fs->end || __argp_fmtstream_ensure (__fs, __len))
-    {
-      memcpy (__fs->p, __str, __len);
-      __fs->p += __len;
-      return __len;
-    }
-  else
-    return 0;
-}
-
-int
-__argp_fmtstream_puts (argp_fmtstream_t __fs, __const char *__str)
-{
-  size_t __len = strlen (__str);
-  if (__len)
-    {
-      size_t __wrote = __argp_fmtstream_write (__fs, __str, __len);
-      return __wrote == __len ? 0 : -1;
-    }
-  else
-    return 0;
-}
-
-int
-__argp_fmtstream_putc (argp_fmtstream_t __fs, int __ch)
-{
-  if (__fs->p < __fs->end || __argp_fmtstream_ensure (__fs, 1))
-    return *__fs->p++ = __ch;
-  else
-    return EOF;
-}
-
-/* Set __FS's left margin to __LMARGIN and return the old value.  */
-size_t
-__argp_fmtstream_set_lmargin (argp_fmtstream_t __fs, size_t __lmargin)
-{
-  size_t __old;
-  if ((size_t) (__fs->p - __fs->buf) > __fs->point_offs)
-    __argp_fmtstream_update (__fs);
-  __old = __fs->lmargin;
-  __fs->lmargin = __lmargin;
-  return __old;
-}
-
-/* Set __FS's right margin to __RMARGIN and return the old value.  */
-size_t
-__argp_fmtstream_set_rmargin (argp_fmtstream_t __fs, size_t __rmargin)
-{
-  size_t __old;
-  if ((size_t) (__fs->p - __fs->buf) > __fs->point_offs)
-    __argp_fmtstream_update (__fs);
-  __old = __fs->rmargin;
-  __fs->rmargin = __rmargin;
-  return __old;
-}
-
-/* Set FS's wrap margin to __WMARGIN and return the old value.  */
-size_t
-__argp_fmtstream_set_wmargin (argp_fmtstream_t __fs, size_t __wmargin)
-{
-  size_t __old;
-  if ((size_t) (__fs->p - __fs->buf) > __fs->point_offs)
-    __argp_fmtstream_update (__fs);
-  __old = __fs->wmargin;
-  __fs->wmargin = __wmargin;
-  return __old;
-}
-
-/* Return the column number of the current output point in __FS.  */
-size_t
-__argp_fmtstream_point (argp_fmtstream_t __fs)
-{
-  if ((size_t) (__fs->p - __fs->buf) > __fs->point_offs)
-    __argp_fmtstream_update (__fs);
-  return __fs->point_col >= 0 ? __fs->point_col : 0;
-}
-#endif /* __STDC_VERSION__ - 199900L < 1 */
-
-#endif /* !ARGP_FMTSTREAM_USE_LINEWRAP */
diff --git a/contrib/argp-standalone/argp-fmtstream.h b/contrib/argp-standalone/argp-fmtstream.h
deleted file mode 100644
index 828f4357d56..00000000000
--- a/contrib/argp-standalone/argp-fmtstream.h
+++ /dev/null
@@ -1,327 +0,0 @@
-/* Word-wrapping and line-truncating streams.
-   Copyright (C) 1997, 2003 Free Software Foundation, Inc.
-   This file is part of the GNU C Library.
-   Written by Miles Bader <miles@gnu.ai.mit.edu>.
-
-   The GNU C Library is free software; you can redistribute it and/or
-   modify it under the terms of the GNU Library General Public License as
-   published by the Free Software Foundation; either version 2 of the
-   License, or (at your option) any later version.
-
-   The GNU C Library is distributed in the hope that it will be useful,
-   but WITHOUT ANY WARRANTY; without even the implied warranty of
-   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-   Library General Public License for more details.
-
-   You should have received a copy of the GNU Library General Public
-   License along with the GNU C Library; see the file COPYING.LIB.  If not,
-   write to the Free Software Foundation, Inc., 59 Temple Place - Suite 330,
-   Boston, MA 02111-1307, USA.  */
-
-/* This package emulates glibc `line_wrap_stream' semantics for systems that
-   don't have that.  If the system does have it, it is just a wrapper for
-   that.  This header file is only used internally while compiling argp, and
-   shouldn't be installed.  */
-
-#ifndef _ARGP_FMTSTREAM_H
-#define _ARGP_FMTSTREAM_H
-
-#include <stdio.h>
-#include <string.h>
-
-#if HAVE_UNISTD_H
-# include <unistd.h>
-#else
-/* This is a kludge to make the code compile on windows. Perhaps it
-   would be better to just replace ssize_t with int through out the
-   code. */
-# define ssize_t int
-#endif
-
-#if _LIBC || (defined (HAVE_FLOCKFILE) && defined(HAVE_PUTC_UNLOCKED) \
-     && defined (HAVE_FPUTS_UNLOCKED) && defined (HAVE_FWRITE_UNLOCKED) )
-/* Use locking funxtions */
-# define FLOCKFILE(f) flockfile(f)
-# define FUNLOCKFILE(f) funlockfile(f)
-# define PUTC_UNLOCKED(c, f) putc_unlocked((c), (f))
-# define FPUTS_UNLOCKED(s, f) fputs_unlocked((s), (f))
-# define FWRITE_UNLOCKED(b, s, n, f) fwrite_unlocked((b), (s), (n), (f))
-#else
-/* Disable stdio locking */
-# define FLOCKFILE(f)
-# define FUNLOCKFILE(f)
-# define PUTC_UNLOCKED(c, f) putc((c), (f))
-# define FPUTS_UNLOCKED(s, f) fputs((s), (f))
-# define FWRITE_UNLOCKED(b, s, n, f) fwrite((b), (s), (n), (f))
-#endif /* No thread safe i/o */
-
-#if    (_LIBC - 0 && !defined (USE_IN_LIBIO)) \
-    || (defined (__GNU_LIBRARY__) && defined (HAVE_LINEWRAP_H))
-/* line_wrap_stream is available, so use that.  */
-#define ARGP_FMTSTREAM_USE_LINEWRAP
-#endif
-
-#ifdef ARGP_FMTSTREAM_USE_LINEWRAP
-/* Just be a simple wrapper for line_wrap_stream; the semantics are
-   *slightly* different, as line_wrap_stream doesn't actually make a new
-   object, it just modifies the given stream (reversibly) to do
-   line-wrapping.  Since we control who uses this code, it doesn't matter.  */
-
-#include <linewrap.h>
-
-typedef FILE *argp_fmtstream_t;
-
-#define argp_make_fmtstream line_wrap_stream
-#define __argp_make_fmtstream line_wrap_stream
-#define argp_fmtstream_free line_unwrap_stream
-#define __argp_fmtstream_free line_unwrap_stream
-
-#define __argp_fmtstream_putc(fs,ch) putc(ch,fs)
-#define argp_fmtstream_putc(fs,ch) putc(ch,fs)
-#define __argp_fmtstream_puts(fs,str) fputs(str,fs)
-#define argp_fmtstream_puts(fs,str) fputs(str,fs)
-#define __argp_fmtstream_write(fs,str,len) fwrite(str,1,len,fs)
-#define argp_fmtstream_write(fs,str,len) fwrite(str,1,len,fs)
-#define __argp_fmtstream_printf fprintf
-#define argp_fmtstream_printf fprintf
-
-#define __argp_fmtstream_lmargin line_wrap_lmargin
-#define argp_fmtstream_lmargin line_wrap_lmargin
-#define __argp_fmtstream_set_lmargin line_wrap_set_lmargin
-#define argp_fmtstream_set_lmargin line_wrap_set_lmargin
-#define __argp_fmtstream_rmargin line_wrap_rmargin
-#define argp_fmtstream_rmargin line_wrap_rmargin
-#define __argp_fmtstream_set_rmargin line_wrap_set_rmargin
-#define argp_fmtstream_set_rmargin line_wrap_set_rmargin
-#define __argp_fmtstream_wmargin line_wrap_wmargin
-#define argp_fmtstream_wmargin line_wrap_wmargin
-#define __argp_fmtstream_set_wmargin line_wrap_set_wmargin
-#define argp_fmtstream_set_wmargin line_wrap_set_wmargin
-#define __argp_fmtstream_point line_wrap_point
-#define argp_fmtstream_point line_wrap_point
-
-#else /* !ARGP_FMTSTREAM_USE_LINEWRAP */
-/* Guess we have to define our own version.  */
-
-#ifndef __const
-#define __const const
-#endif
-
-
-struct argp_fmtstream
-{
-  FILE *stream;			/* The stream we're outputting to.  */
-
-  size_t lmargin, rmargin;	/* Left and right margins.  */
-  ssize_t wmargin;		/* Margin to wrap to, or -1 to truncate.  */
-
-  /* Point in buffer to which we've processed for wrapping, but not output.  */
-  size_t point_offs;
-  /* Output column at POINT_OFFS, or -1 meaning 0 but don't add lmargin.  */
-  ssize_t point_col;
-
-  char *buf;			/* Output buffer.  */
-  char *p;			/* Current end of text in BUF. */
-  char *end;			/* Absolute end of BUF.  */
-};
-
-typedef struct argp_fmtstream *argp_fmtstream_t;
-
-/* Return an argp_fmtstream that outputs to STREAM, and which prefixes lines
-   written on it with LMARGIN spaces and limits them to RMARGIN columns
-   total.  If WMARGIN >= 0, words that extend past RMARGIN are wrapped by
-   replacing the whitespace before them with a newline and WMARGIN spaces.
-   Otherwise, chars beyond RMARGIN are simply dropped until a newline.
-   Returns NULL if there was an error.  */
-extern argp_fmtstream_t __argp_make_fmtstream (FILE *__stream,
-					       size_t __lmargin,
-					       size_t __rmargin,
-					       ssize_t __wmargin);
-extern argp_fmtstream_t argp_make_fmtstream (FILE *__stream,
-					     size_t __lmargin,
-					     size_t __rmargin,
-					     ssize_t __wmargin);
-
-/* Flush __FS to its stream, and free it (but don't close the stream).  */
-extern void __argp_fmtstream_free (argp_fmtstream_t __fs);
-extern void argp_fmtstream_free (argp_fmtstream_t __fs);
-
-extern ssize_t __argp_fmtstream_printf (argp_fmtstream_t __fs,
-				       __const char *__fmt, ...)
-     PRINTF_STYLE(2,3);
-extern ssize_t argp_fmtstream_printf (argp_fmtstream_t __fs,
-				      __const char *__fmt, ...)
-     PRINTF_STYLE(2,3);
-
-#if __STDC_VERSION__ - 199900L < 1
-extern int __argp_fmtstream_putc (argp_fmtstream_t __fs, int __ch);
-extern int argp_fmtstream_putc (argp_fmtstream_t __fs, int __ch);
-
-extern int __argp_fmtstream_puts (argp_fmtstream_t __fs, __const char *__str);
-extern int argp_fmtstream_puts (argp_fmtstream_t __fs, __const char *__str);
-
-extern size_t __argp_fmtstream_write (argp_fmtstream_t __fs,
-				      __const char *__str, size_t __len);
-extern size_t argp_fmtstream_write (argp_fmtstream_t __fs,
-				    __const char *__str, size_t __len);
-#endif /* __STDC_VERSION__ - 199900L < 1 */
-
-/* Access macros for various bits of state.  */
-#define argp_fmtstream_lmargin(__fs) ((__fs)->lmargin)
-#define argp_fmtstream_rmargin(__fs) ((__fs)->rmargin)
-#define argp_fmtstream_wmargin(__fs) ((__fs)->wmargin)
-#define __argp_fmtstream_lmargin argp_fmtstream_lmargin
-#define __argp_fmtstream_rmargin argp_fmtstream_rmargin
-#define __argp_fmtstream_wmargin argp_fmtstream_wmargin
-
-#if __STDC_VERSION__ - 199900L < 1
-/* Set __FS's left margin to LMARGIN and return the old value.  */
-extern size_t argp_fmtstream_set_lmargin (argp_fmtstream_t __fs,
-					  size_t __lmargin);
-extern size_t __argp_fmtstream_set_lmargin (argp_fmtstream_t __fs,
-					    size_t __lmargin);
-
-/* Set __FS's right margin to __RMARGIN and return the old value.  */
-extern size_t argp_fmtstream_set_rmargin (argp_fmtstream_t __fs,
-					  size_t __rmargin);
-extern size_t __argp_fmtstream_set_rmargin (argp_fmtstream_t __fs,
-					    size_t __rmargin);
-
-/* Set __FS's wrap margin to __WMARGIN and return the old value.  */
-extern size_t argp_fmtstream_set_wmargin (argp_fmtstream_t __fs,
-					  size_t __wmargin);
-extern size_t __argp_fmtstream_set_wmargin (argp_fmtstream_t __fs,
-					    size_t __wmargin);
-
-/* Return the column number of the current output point in __FS.  */
-extern size_t argp_fmtstream_point (argp_fmtstream_t __fs);
-extern size_t __argp_fmtstream_point (argp_fmtstream_t __fs);
-#endif /* __STDC_VERSION__ - 199900L < 1 */
-
-/* Internal routines.  */
-extern void _argp_fmtstream_update (argp_fmtstream_t __fs);
-extern void __argp_fmtstream_update (argp_fmtstream_t __fs);
-extern int _argp_fmtstream_ensure (argp_fmtstream_t __fs, size_t __amount);
-extern int __argp_fmtstream_ensure (argp_fmtstream_t __fs, size_t __amount);
-
-#ifdef __OPTIMIZE__
-/* Inline versions of above routines.  */
-
-#if !_LIBC
-#define __argp_fmtstream_putc argp_fmtstream_putc
-#define __argp_fmtstream_puts argp_fmtstream_puts
-#define __argp_fmtstream_write argp_fmtstream_write
-#define __argp_fmtstream_set_lmargin argp_fmtstream_set_lmargin
-#define __argp_fmtstream_set_rmargin argp_fmtstream_set_rmargin
-#define __argp_fmtstream_set_wmargin argp_fmtstream_set_wmargin
-#define __argp_fmtstream_point argp_fmtstream_point
-#define __argp_fmtstream_update _argp_fmtstream_update
-#define __argp_fmtstream_ensure _argp_fmtstream_ensure
-#endif
-
-#ifndef ARGP_FS_EI
-#if defined(__GNUC__) && !defined(__GNUC_STDC_INLINE__)
-#define ARGP_FS_EI extern inline
-#else
-#define ARGP_FS_EI inline
-#endif
-#endif
-
-ARGP_FS_EI size_t
-__argp_fmtstream_write (argp_fmtstream_t __fs,
-			__const char *__str, size_t __len)
-{
-  if (__fs->p + __len <= __fs->end || __argp_fmtstream_ensure (__fs, __len))
-    {
-      memcpy (__fs->p, __str, __len);
-      __fs->p += __len;
-      return __len;
-    }
-  else
-    return 0;
-}
-
-ARGP_FS_EI int
-__argp_fmtstream_puts (argp_fmtstream_t __fs, __const char *__str)
-{
-  size_t __len = strlen (__str);
-  if (__len)
-    {
-      size_t __wrote = __argp_fmtstream_write (__fs, __str, __len);
-      return __wrote == __len ? 0 : -1;
-    }
-  else
-    return 0;
-}
-
-ARGP_FS_EI int
-__argp_fmtstream_putc (argp_fmtstream_t __fs, int __ch)
-{
-  if (__fs->p < __fs->end || __argp_fmtstream_ensure (__fs, 1))
-    return *__fs->p++ = __ch;
-  else
-    return EOF;
-}
-
-/* Set __FS's left margin to __LMARGIN and return the old value.  */
-ARGP_FS_EI size_t
-__argp_fmtstream_set_lmargin (argp_fmtstream_t __fs, size_t __lmargin)
-{
-  size_t __old;
-  if ((size_t) (__fs->p - __fs->buf) > __fs->point_offs)
-    __argp_fmtstream_update (__fs);
-  __old = __fs->lmargin;
-  __fs->lmargin = __lmargin;
-  return __old;
-}
-
-/* Set __FS's right margin to __RMARGIN and return the old value.  */
-ARGP_FS_EI size_t
-__argp_fmtstream_set_rmargin (argp_fmtstream_t __fs, size_t __rmargin)
-{
-  size_t __old;
-  if ((size_t) (__fs->p - __fs->buf) > __fs->point_offs)
-    __argp_fmtstream_update (__fs);
-  __old = __fs->rmargin;
-  __fs->rmargin = __rmargin;
-  return __old;
-}
-
-/* Set FS's wrap margin to __WMARGIN and return the old value.  */
-ARGP_FS_EI size_t
-__argp_fmtstream_set_wmargin (argp_fmtstream_t __fs, size_t __wmargin)
-{
-  size_t __old;
-  if ((size_t) (__fs->p - __fs->buf) > __fs->point_offs)
-    __argp_fmtstream_update (__fs);
-  __old = __fs->wmargin;
-  __fs->wmargin = __wmargin;
-  return __old;
-}
-
-/* Return the column number of the current output point in __FS.  */
-ARGP_FS_EI size_t
-__argp_fmtstream_point (argp_fmtstream_t __fs)
-{
-  if ((size_t) (__fs->p - __fs->buf) > __fs->point_offs)
-    __argp_fmtstream_update (__fs);
-  return __fs->point_col >= 0 ? __fs->point_col : 0;
-}
-
-#if !_LIBC
-#undef __argp_fmtstream_putc
-#undef __argp_fmtstream_puts
-#undef __argp_fmtstream_write
-#undef __argp_fmtstream_set_lmargin
-#undef __argp_fmtstream_set_rmargin
-#undef __argp_fmtstream_set_wmargin
-#undef __argp_fmtstream_point
-#undef __argp_fmtstream_update
-#undef __argp_fmtstream_ensure
-#endif
-
-#endif /* __OPTIMIZE__ */
-
-#endif /* ARGP_FMTSTREAM_USE_LINEWRAP */
-
-#endif /* argp-fmtstream.h */
diff --git a/contrib/argp-standalone/argp-help.c b/contrib/argp-standalone/argp-help.c
deleted file mode 100644
index ced78c4cb26..00000000000
--- a/contrib/argp-standalone/argp-help.c
+++ /dev/null
@@ -1,1849 +0,0 @@
-/* Hierarchial argument parsing help output
-   Copyright (C) 1995,96,97,98,99,2000, 2003 Free Software Foundation, Inc.
-   This file is part of the GNU C Library.
-   Written by Miles Bader <miles@gnu.ai.mit.edu>.
-
-   The GNU C Library is free software; you can redistribute it and/or
-   modify it under the terms of the GNU Library General Public License as
-   published by the Free Software Foundation; either version 2 of the
-   License, or (at your option) any later version.
-
-   The GNU C Library is distributed in the hope that it will be useful,
-   but WITHOUT ANY WARRANTY; without even the implied warranty of
-   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-   Library General Public License for more details.
-
-   You should have received a copy of the GNU Library General Public
-   License along with the GNU C Library; see the file COPYING.LIB.  If not,
-   write to the Free Software Foundation, Inc., 59 Temple Place - Suite 330,
-   Boston, MA 02111-1307, USA.  */
-
-#ifndef _GNU_SOURCE
-# define _GNU_SOURCE	1
-#endif
-
-#ifdef HAVE_CONFIG_H
-#include <config.h>
-#endif
-
-#if HAVE_ALLOCA_H
-#include <alloca.h>
-#endif
-
-#include <stddef.h>
-#include <stdlib.h>
-#include <string.h>
-#include <assert.h>
-#include <stdarg.h>
-#include <ctype.h>
-#if HAVE_MALLOC_H
-/* Needed, for alloca on windows */
-# include <malloc.h>
-#endif
-
-#ifndef _
-/* This is for other GNU distributions with internationalized messages.  */
-# if defined HAVE_LIBINTL_H || defined _LIBC
-#  include <libintl.h>
-#  ifdef _LIBC
-#   undef dgettext
-#   define dgettext(domain, msgid) __dcgettext (domain, msgid, LC_MESSAGES)
-#  endif
-# else
-#  define dgettext(domain, msgid) (msgid)
-# endif
-#endif
-
-#include "argp.h"
-#include "argp-fmtstream.h"
-#include "argp-namefrob.h"
-
-
-#ifndef _LIBC
-# ifndef __strchrnul
-#  define __strchrnul strchrnul
-# endif
-# ifndef __mempcpy
-#  define __mempcpy mempcpy
-# endif
-/* We need to use a different name, as __strndup is likely a macro. */
-# define STRNDUP strndup
-# if HAVE_STRERROR
-#  define STRERROR strerror
-# else
-#  define STRERROR(x) (sys_errlist[x])
-# endif
-#else /* _LIBC */
-# define FLOCKFILE __flockfile
-# define FUNLOCKFILE __funlockfile
-# define STRNDUP __strndup
-# define STRERROR strerror
-#endif
-
-#if !_LIBC
-# if !HAVE_STRNDUP
-char *strndup (const char *s, size_t size);
-# endif /* !HAVE_STRNDUP */
-
-# if !HAVE_MEMPCPY
-void *mempcpy (void *to, const void *from, size_t size);
-# endif /* !HAVE_MEMPCPY */
-
-# if !HAVE_STRCHRNUL
-char *strchrnul(const char *s, int c);
-# endif /* !HAVE_STRCHRNUL */
-
-# if !HAVE_STRCASECMP
-int strcasecmp(const char *s1, const char *s2);
-#endif
-
-#endif /* !_LIBC */
-
-
-/* User-selectable (using an environment variable) formatting parameters.
-
-   These may be specified in an environment variable called `ARGP_HELP_FMT',
-   with a contents like:  VAR1=VAL1,VAR2=VAL2,BOOLVAR2,no-BOOLVAR2
-   Where VALn must be a positive integer.  The list of variables is in the
-   UPARAM_NAMES vector, below.  */
-
-/* Default parameters.  */
-#define DUP_ARGS      0		/* True if option argument can be duplicated. */
-#define DUP_ARGS_NOTE 1		/* True to print a note about duplicate args. */
-#define SHORT_OPT_COL 2		/* column in which short options start */
-#define LONG_OPT_COL  6		/* column in which long options start */
-#define DOC_OPT_COL   2		/* column in which doc options start */
-#define OPT_DOC_COL  29		/* column in which option text starts */
-#define HEADER_COL    1		/* column in which group headers are printed */
-#define USAGE_INDENT 12		/* indentation of wrapped usage lines */
-#define RMARGIN      79		/* right margin used for wrapping */
-
-/* User-selectable (using an environment variable) formatting parameters.
-   They must all be of type `int' for the parsing code to work.  */
-struct uparams
-{
-  /* If true, arguments for an option are shown with both short and long
-     options, even when a given option has both, e.g. `-x ARG, --longx=ARG'.
-     If false, then if an option has both, the argument is only shown with
-     the long one, e.g., `-x, --longx=ARG', and a message indicating that
-     this really means both is printed below the options.  */
-  int dup_args;
-
-  /* This is true if when DUP_ARGS is false, and some duplicate arguments have
-     been suppressed, an explanatory message should be printed.  */
-  int dup_args_note;
-
-  /* Various output columns.  */
-  int short_opt_col;
-  int long_opt_col;
-  int doc_opt_col;
-  int opt_doc_col;
-  int header_col;
-  int usage_indent;
-  int rmargin;
-
-  int valid;			/* True when the values in here are valid.  */
-};
-
-/* This is a global variable, as user options are only ever read once.  */
-static struct uparams uparams = {
-  DUP_ARGS, DUP_ARGS_NOTE,
-  SHORT_OPT_COL, LONG_OPT_COL, DOC_OPT_COL, OPT_DOC_COL, HEADER_COL,
-  USAGE_INDENT, RMARGIN,
-  0
-};
-
-/* A particular uparam, and what the user name is.  */
-struct uparam_name
-{
-  const char *name;		/* User name.  */
-  int is_bool;			/* Whether it's `boolean'.  */
-  size_t uparams_offs;		/* Location of the (int) field in UPARAMS.  */
-};
-
-/* The name-field mappings we know about.  */
-static const struct uparam_name uparam_names[] =
-{
-  { "dup-args",       1, offsetof (struct uparams, dup_args) },
-  { "dup-args-note",  1, offsetof (struct uparams, dup_args_note) },
-  { "short-opt-col",  0, offsetof (struct uparams, short_opt_col) },
-  { "long-opt-col",   0, offsetof (struct uparams, long_opt_col) },
-  { "doc-opt-col",    0, offsetof (struct uparams, doc_opt_col) },
-  { "opt-doc-col",    0, offsetof (struct uparams, opt_doc_col) },
-  { "header-col",     0, offsetof (struct uparams, header_col) },
-  { "usage-indent",   0, offsetof (struct uparams, usage_indent) },
-  { "rmargin",        0, offsetof (struct uparams, rmargin) },
-  { 0, 0, 0 }
-};
-
-/* Read user options from the environment, and fill in UPARAMS appropiately.  */
-static void
-fill_in_uparams (const struct argp_state *state)
-{
-
-  const char *var = getenv ("ARGP_HELP_FMT");
-
-#define SKIPWS(p) do { while (isspace (*p)) p++; } while (0);
-
-  if (var)
-    /* Parse var. */
-    while (*var)
-      {
-	SKIPWS (var);
-
-	if (isalpha (*var))
-	  {
-	    size_t var_len;
-	    const struct uparam_name *un;
-	    int unspec = 0, val = 0;
-	    const char *arg = var;
-
-	    while (isalnum (*arg) || *arg == '-' || *arg == '_')
-	      arg++;
-	    var_len = arg - var;
-
-	    SKIPWS (arg);
-
-	    if (*arg == '\0' || *arg == ',')
-	      unspec = 1;
-	    else if (*arg == '=')
-	      {
-		arg++;
-		SKIPWS (arg);
-	      }
-
-	    if (unspec)
-	      {
-		if (var[0] == 'n' && var[1] == 'o' && var[2] == '-')
-		  {
-		    val = 0;
-		    var += 3;
-		    var_len -= 3;
-		  }
-		else
-		  val = 1;
-	      }
-	    else if (isdigit (*arg))
-	      {
-		val = atoi (arg);
-		while (isdigit (*arg))
-		  arg++;
-		SKIPWS (arg);
-	      }
-
-	    for (un = uparam_names; un->name; un++)
-	      if (strlen (un->name) == var_len
-		  && strncmp (var, un->name, var_len) == 0)
-		{
-		  if (unspec && !un->is_bool)
-		    __argp_failure (state, 0, 0,
-				    dgettext (state->root_argp->argp_domain, "\
-%.*s: ARGP_HELP_FMT parameter requires a value"),
-				    (int) var_len, var);
-		  else
-		    *(int *)((char *)&uparams + un->uparams_offs) = val;
-		  break;
-		}
-	    if (! un->name)
-	      __argp_failure (state, 0, 0,
-			      dgettext (state->root_argp->argp_domain, "\
-%.*s: Unknown ARGP_HELP_FMT parameter"),
-			      (int) var_len, var);
-
-	    var = arg;
-	    if (*var == ',')
-	      var++;
-	  }
-	else if (*var)
-	  {
-	    __argp_failure (state, 0, 0,
-			    dgettext (state->root_argp->argp_domain,
-				      "Garbage in ARGP_HELP_FMT: %s"), var);
-	    break;
-	  }
-      }
-}
-
-/* Returns true if OPT hasn't been marked invisible.  Visibility only affects
-   whether OPT is displayed or used in sorting, not option shadowing.  */
-#define ovisible(opt) (! ((opt)->flags & OPTION_HIDDEN))
-
-/* Returns true if OPT is an alias for an earlier option.  */
-#define oalias(opt) ((opt)->flags & OPTION_ALIAS)
-
-/* Returns true if OPT is an documentation-only entry.  */
-#define odoc(opt) ((opt)->flags & OPTION_DOC)
-
-/* Returns true if OPT is the end-of-list marker for a list of options.  */
-#define oend(opt) __option_is_end (opt)
-
-/* Returns true if OPT has a short option.  */
-#define oshort(opt) __option_is_short (opt)
-
-/*
-   The help format for a particular option is like:
-
-     -xARG, -yARG, --long1=ARG, --long2=ARG        Documentation...
-
-   Where ARG will be omitted if there's no argument, for this option, or
-   will be surrounded by "[" and "]" appropiately if the argument is
-   optional.  The documentation string is word-wrapped appropiately, and if
-   the list of options is long enough, it will be started on a separate line.
-   If there are no short options for a given option, the first long option is
-   indented slighly in a way that's supposed to make most long options appear
-   to be in a separate column.
-
-   For example, the following output (from ps):
-
-     -p PID, --pid=PID          List the process PID
-	 --pgrp=PGRP            List processes in the process group PGRP
-     -P, -x, --no-parent        Include processes without parents
-     -Q, --all-fields           Don't elide unusable fields (normally if there's
-				some reason ps can't print a field for any
-				process, it's removed from the output entirely)
-     -r, --reverse, --gratuitously-long-reverse-option
-				Reverse the order of any sort
-	 --session[=SID]        Add the processes from the session SID (which
-				defaults to the sid of the current process)
-
-    Here are some more options:
-     -f ZOT, --foonly=ZOT       Glork a foonly
-     -z, --zaza                 Snit a zar
-
-     -?, --help                 Give this help list
-	 --usage                Give a short usage message
-     -V, --version              Print program version
-
-   The struct argp_option array for the above could look like:
-
-   {
-     {"pid",       'p',      "PID",  0, "List the process PID"},
-     {"pgrp",      OPT_PGRP, "PGRP", 0, "List processes in the process group PGRP"},
-     {"no-parent", 'P',	      0,     0, "Include processes without parents"},
-     {0,           'x',       0,     OPTION_ALIAS},
-     {"all-fields",'Q',       0,     0, "Don't elide unusable fields (normally"
-                                        " if there's some reason ps can't"
-					" print a field for any process, it's"
-                                        " removed from the output entirely)" },
-     {"reverse",   'r',       0,     0, "Reverse the order of any sort"},
-     {"gratuitously-long-reverse-option", 0, 0, OPTION_ALIAS},
-     {"session",   OPT_SESS,  "SID", OPTION_ARG_OPTIONAL,
-                                        "Add the processes from the session"
-					" SID (which defaults to the sid of"
-					" the current process)" },
-
-     {0,0,0,0, "Here are some more options:"},
-     {"foonly", 'f', "ZOT", 0, "Glork a foonly"},
-     {"zaza", 'z', 0, 0, "Snit a zar"},
-
-     {0}
-   }
-
-   Note that the last three options are automatically supplied by argp_parse,
-   unless you tell it not to with ARGP_NO_HELP.
-
-*/
-
-/* Returns true if CH occurs between BEG and END.  */
-static int
-find_char (char ch, char *beg, char *end)
-{
-  while (beg < end)
-    if (*beg == ch)
-      return 1;
-    else
-      beg++;
-  return 0;
-}
-
-struct hol_cluster;		/* fwd decl */
-
-struct hol_entry
-{
-  /* First option.  */
-  const struct argp_option *opt;
-  /* Number of options (including aliases).  */
-  unsigned num;
-
-  /* A pointers into the HOL's short_options field, to the first short option
-     letter for this entry.  The order of the characters following this point
-     corresponds to the order of options pointed to by OPT, and there are at
-     most NUM.  A short option recorded in a option following OPT is only
-     valid if it occurs in the right place in SHORT_OPTIONS (otherwise it's
-     probably been shadowed by some other entry).  */
-  char *short_options;
-
-  /* Entries are sorted by their group first, in the order:
-       1, 2, ..., n, 0, -m, ..., -2, -1
-     and then alphabetically within each group.  The default is 0.  */
-  int group;
-
-  /* The cluster of options this entry belongs to, or 0 if none.  */
-  struct hol_cluster *cluster;
-
-  /* The argp from which this option came.  */
-  const struct argp *argp;
-};
-
-/* A cluster of entries to reflect the argp tree structure.  */
-struct hol_cluster
-{
-  /* A descriptive header printed before options in this cluster.  */
-  const char *header;
-
-  /* Used to order clusters within the same group with the same parent,
-     according to the order in which they occurred in the parent argp's child
-     list.  */
-  int index;
-
-  /* How to sort this cluster with respect to options and other clusters at the
-     same depth (clusters always follow options in the same group).  */
-  int group;
-
-  /* The cluster to which this cluster belongs, or 0 if it's at the base
-     level.  */
-  struct hol_cluster *parent;
-
-  /* The argp from which this cluster is (eventually) derived.  */
-  const struct argp *argp;
-
-  /* The distance this cluster is from the root.  */
-  int depth;
-
-  /* Clusters in a given hol are kept in a linked list, to make freeing them
-     possible.  */
-  struct hol_cluster *next;
-};
-
-/* A list of options for help.  */
-struct hol
-{
-  /* An array of hol_entry's.  */
-  struct hol_entry *entries;
-  /* The number of entries in this hol.  If this field is zero, the others
-     are undefined.  */
-  unsigned num_entries;
-
-  /* A string containing all short options in this HOL.  Each entry contains
-     pointers into this string, so the order can't be messed with blindly.  */
-  char *short_options;
-
-  /* Clusters of entries in this hol.  */
-  struct hol_cluster *clusters;
-};
-
-/* Create a struct hol from the options in ARGP.  CLUSTER is the
-   hol_cluster in which these entries occur, or 0, if at the root.  */
-static struct hol *
-make_hol (const struct argp *argp, struct hol_cluster *cluster)
-{
-  char *so;
-  const struct argp_option *o;
-  const struct argp_option *opts = argp->options;
-  struct hol_entry *entry;
-  unsigned num_short_options = 0;
-  struct hol *hol = malloc (sizeof (struct hol));
-
-  assert (hol);
-
-  hol->num_entries = 0;
-  hol->clusters = 0;
-
-  if (opts)
-    {
-      int cur_group = 0;
-
-      /* The first option must not be an alias.  */
-      assert (! oalias (opts));
-
-      /* Calculate the space needed.  */
-      for (o = opts; ! oend (o); o++)
-	{
-	  if (! oalias (o))
-	    hol->num_entries++;
-	  if (oshort (o))
-	    num_short_options++;	/* This is an upper bound.  */
-	}
-
-      hol->entries = malloc (sizeof (struct hol_entry) * hol->num_entries);
-      hol->short_options = malloc (num_short_options + 1);
-
-      assert (hol->entries && hol->short_options);
-
-      /* Fill in the entries.  */
-      so = hol->short_options;
-      for (o = opts, entry = hol->entries; ! oend (o); entry++)
-	{
-	  entry->opt = o;
-	  entry->num = 0;
-	  entry->short_options = so;
-	  entry->group = cur_group =
-	    o->group
-	    ? o->group
-	    : ((!o->name && !o->key)
-	       ? cur_group + 1
-	       : cur_group);
-	  entry->cluster = cluster;
-	  entry->argp = argp;
-
-	  do
-	    {
-	      entry->num++;
-	      if (oshort (o) && ! find_char (o->key, hol->short_options, so))
-		/* O has a valid short option which hasn't already been used.*/
-		*so++ = o->key;
-	      o++;
-	    }
-	  while (! oend (o) && oalias (o));
-	}
-      *so = '\0';		/* null terminated so we can find the length */
-    }
-
-  return hol;
-}
-
-/* Add a new cluster to HOL, with the given GROUP and HEADER (taken from the
-   associated argp child list entry), INDEX, and PARENT, and return a pointer
-   to it.  ARGP is the argp that this cluster results from.  */
-static struct hol_cluster *
-hol_add_cluster (struct hol *hol, int group, const char *header, int index,
-		 struct hol_cluster *parent, const struct argp *argp)
-{
-  struct hol_cluster *cl = malloc (sizeof (struct hol_cluster));
-  if (cl)
-    {
-      cl->group = group;
-      cl->header = header;
-
-      cl->index = index;
-      cl->parent = parent;
-      cl->argp = argp;
-      cl->depth = parent ? parent->depth + 1 : 0;
-
-      cl->next = hol->clusters;
-      hol->clusters = cl;
-    }
-  return cl;
-}
-
-/* Free HOL and any resources it uses.  */
-static void
-hol_free (struct hol *hol)
-{
-  struct hol_cluster *cl = hol->clusters;
-
-  while (cl)
-    {
-      struct hol_cluster *next = cl->next;
-      free (cl);
-      cl = next;
-    }
-
-  if (hol->num_entries > 0)
-    {
-      free (hol->entries);
-      free (hol->short_options);
-    }
-
-  free (hol);
-}
-
-static inline int
-hol_entry_short_iterate (const struct hol_entry *entry,
-			 int (*func)(const struct argp_option *opt,
-				     const struct argp_option *real,
-				     const char *domain, void *cookie),
-			 const char *domain, void *cookie)
-{
-  unsigned nopts;
-  int val = 0;
-  const struct argp_option *opt, *real = entry->opt;
-  char *so = entry->short_options;
-
-  for (opt = real, nopts = entry->num; nopts > 0 && !val; opt++, nopts--)
-    if (oshort (opt) && *so == opt->key)
-      {
-	if (!oalias (opt))
-	  real = opt;
-	if (ovisible (opt))
-	  val = (*func)(opt, real, domain, cookie);
-	so++;
-      }
-
-  return val;
-}
-
-static inline int
-hol_entry_long_iterate (const struct hol_entry *entry,
-			int (*func)(const struct argp_option *opt,
-				    const struct argp_option *real,
-				    const char *domain, void *cookie),
-			const char *domain, void *cookie)
-{
-  unsigned nopts;
-  int val = 0;
-  const struct argp_option *opt, *real = entry->opt;
-
-  for (opt = real, nopts = entry->num; nopts > 0 && !val; opt++, nopts--)
-    if (opt->name)
-      {
-	if (!oalias (opt))
-	  real = opt;
-	if (ovisible (opt))
-	  val = (*func)(opt, real, domain, cookie);
-      }
-
-  return val;
-}
-
-/* Iterator that returns true for the first short option.  */
-static inline int
-until_short (const struct argp_option *opt, const struct argp_option *real UNUSED,
-	     const char *domain UNUSED, void *cookie UNUSED)
-{
-  return oshort (opt) ? opt->key : 0;
-}
-
-/* Returns the first valid short option in ENTRY, or 0 if there is none.  */
-static char
-hol_entry_first_short (const struct hol_entry *entry)
-{
-  return hol_entry_short_iterate (entry, until_short,
-				  entry->argp->argp_domain, 0);
-}
-
-/* Returns the first valid long option in ENTRY, or 0 if there is none.  */
-static const char *
-hol_entry_first_long (const struct hol_entry *entry)
-{
-  const struct argp_option *opt;
-  unsigned num;
-  for (opt = entry->opt, num = entry->num; num > 0; opt++, num--)
-    if (opt->name && ovisible (opt))
-      return opt->name;
-  return 0;
-}
-
-/* Returns the entry in HOL with the long option name NAME, or 0 if there is
-   none.  */
-static struct hol_entry *
-hol_find_entry (struct hol *hol, const char *name)
-{
-  struct hol_entry *entry = hol->entries;
-  unsigned num_entries = hol->num_entries;
-
-  while (num_entries-- > 0)
-    {
-      const struct argp_option *opt = entry->opt;
-      unsigned num_opts = entry->num;
-
-      while (num_opts-- > 0)
-	if (opt->name && ovisible (opt) && strcmp (opt->name, name) == 0)
-	  return entry;
-	else
-	  opt++;
-
-      entry++;
-    }
-
-  return 0;
-}
-
-/* If an entry with the long option NAME occurs in HOL, set it's special
-   sort position to GROUP.  */
-static void
-hol_set_group (struct hol *hol, const char *name, int group)
-{
-  struct hol_entry *entry = hol_find_entry (hol, name);
-  if (entry)
-    entry->group = group;
-}
-
-/* Order by group:  0, 1, 2, ..., n, -m, ..., -2, -1.
-   EQ is what to return if GROUP1 and GROUP2 are the same.  */
-static int
-group_cmp (int group1, int group2, int eq)
-{
-  if (group1 == group2)
-    return eq;
-  else if ((group1 < 0 && group2 < 0) || (group1 >= 0 && group2 >= 0))
-    return group1 - group2;
-  else
-    return group2 - group1;
-}
-
-/* Compare clusters CL1 & CL2 by the order that they should appear in
-   output.  */
-static int
-hol_cluster_cmp (const struct hol_cluster *cl1, const struct hol_cluster *cl2)
-{
-  /* If one cluster is deeper than the other, use its ancestor at the same
-     level, so that finding the common ancestor is straightforward.  */
-  while (cl1->depth < cl2->depth)
-    cl1 = cl1->parent;
-  while (cl2->depth < cl1->depth)
-    cl2 = cl2->parent;
-
-  /* Now reduce both clusters to their ancestors at the point where both have
-     a common parent; these can be directly compared.  */
-  while (cl1->parent != cl2->parent)
-    cl1 = cl1->parent, cl2 = cl2->parent;
-
-  return group_cmp (cl1->group, cl2->group, cl2->index - cl1->index);
-}
-
-/* Return the ancestor of CL that's just below the root (i.e., has a parent
-   of 0).  */
-static struct hol_cluster *
-hol_cluster_base (struct hol_cluster *cl)
-{
-  while (cl->parent)
-    cl = cl->parent;
-  return cl;
-}
-
-/* Return true if CL1 is a child of CL2.  */
-static int
-hol_cluster_is_child (const struct hol_cluster *cl1,
-		      const struct hol_cluster *cl2)
-{
-  while (cl1 && cl1 != cl2)
-    cl1 = cl1->parent;
-  return cl1 == cl2;
-}
-
-/* Given the name of a OPTION_DOC option, modifies NAME to start at the tail
-   that should be used for comparisons, and returns true iff it should be
-   treated as a non-option.  */
-
-/* FIXME: Can we use unsigned char * for the argument? */
-static int
-canon_doc_option (const char **name)
-{
-  int non_opt;
-  /* Skip initial whitespace.  */
-  while (isspace ( (unsigned char) **name))
-    (*name)++;
-  /* Decide whether this looks like an option (leading `-') or not.  */
-  non_opt = (**name != '-');
-  /* Skip until part of name used for sorting.  */
-  while (**name && !isalnum ( (unsigned char) **name))
-    (*name)++;
-  return non_opt;
-}
-
-/* Order ENTRY1 & ENTRY2 by the order which they should appear in a help
-   listing.  */
-static int
-hol_entry_cmp (const struct hol_entry *entry1,
-	       const struct hol_entry *entry2)
-{
-  /* The group numbers by which the entries should be ordered; if either is
-     in a cluster, then this is just the group within the cluster.  */
-  int group1 = entry1->group, group2 = entry2->group;
-
-  if (entry1->cluster != entry2->cluster)
-    {
-      /* The entries are not within the same cluster, so we can't compare them
-	 directly, we have to use the appropiate clustering level too.  */
-      if (! entry1->cluster)
-	/* ENTRY1 is at the `base level', not in a cluster, so we have to
-	   compare it's group number with that of the base cluster in which
-	   ENTRY2 resides.  Note that if they're in the same group, the
-	   clustered option always comes laster.  */
-	return group_cmp (group1, hol_cluster_base (entry2->cluster)->group, -1);
-      else if (! entry2->cluster)
-	/* Likewise, but ENTRY2's not in a cluster.  */
-	return group_cmp (hol_cluster_base (entry1->cluster)->group, group2, 1);
-      else
-	/* Both entries are in clusters, we can just compare the clusters.  */
-	return hol_cluster_cmp (entry1->cluster, entry2->cluster);
-    }
-  else if (group1 == group2)
-    /* The entries are both in the same cluster and group, so compare them
-       alphabetically.  */
-    {
-      int short1 = hol_entry_first_short (entry1);
-      int short2 = hol_entry_first_short (entry2);
-      int doc1 = odoc (entry1->opt);
-      int doc2 = odoc (entry2->opt);
-      /* FIXME: Can we use unsigned char * instead? */
-      const char *long1 = hol_entry_first_long (entry1);
-      const char *long2 = hol_entry_first_long (entry2);
-
-      if (doc1)
-	doc1 = canon_doc_option (&long1);
-      if (doc2)
-	doc2 = canon_doc_option (&long2);
-
-      if (doc1 != doc2)
-	/* `documentation' options always follow normal options (or
-	   documentation options that *look* like normal options).  */
-	return doc1 - doc2;
-      else if (!short1 && !short2 && long1 && long2)
-	/* Only long options.  */
-	return __strcasecmp (long1, long2);
-      else
-	/* Compare short/short, long/short, short/long, using the first
-	   character of long options.  Entries without *any* valid
-	   options (such as options with OPTION_HIDDEN set) will be put
-	   first, but as they're not displayed, it doesn't matter where
-	   they are.  */
-	{
-	  unsigned char first1 = short1 ? short1 : long1 ? *long1 : 0;
-	  unsigned char first2 = short2 ? short2 : long2 ? *long2 : 0;
-#ifdef _tolower
-	  int lower_cmp = _tolower (first1) - _tolower (first2);
-#else
-	  int lower_cmp = tolower (first1) - tolower (first2);
-#endif
-	  /* Compare ignoring case, except when the options are both the
-	     same letter, in which case lower-case always comes first.  */
-	  /* NOTE: The subtraction below does the right thing
-	     even with eight-bit chars: first1 and first2 are
-	     converted to int *before* the subtraction. */
-	  return lower_cmp ? lower_cmp : first2 - first1;
-	}
-    }
-  else
-    /* Within the same cluster, but not the same group, so just compare
-       groups.  */
-    return group_cmp (group1, group2, 0);
-}
-
-/* Version of hol_entry_cmp with correct signature for qsort.  */
-static int
-hol_entry_qcmp (const void *entry1_v, const void *entry2_v)
-{
-  return hol_entry_cmp (entry1_v, entry2_v);
-}
-
-/* Sort HOL by group and alphabetically by option name (with short options
-   taking precedence over long).  Since the sorting is for display purposes
-   only, the shadowing of options isn't effected.  */
-static void
-hol_sort (struct hol *hol)
-{
-  if (hol->num_entries > 0)
-    qsort (hol->entries, hol->num_entries, sizeof (struct hol_entry),
-	   hol_entry_qcmp);
-}
-
-/* Append MORE to HOL, destroying MORE in the process.  Options in HOL shadow
-   any in MORE with the same name.  */
-static void
-hol_append (struct hol *hol, struct hol *more)
-{
-  struct hol_cluster **cl_end = &hol->clusters;
-
-  /* Steal MORE's cluster list, and add it to the end of HOL's.  */
-  while (*cl_end)
-    cl_end = &(*cl_end)->next;
-  *cl_end = more->clusters;
-  more->clusters = 0;
-
-  /* Merge entries.  */
-  if (more->num_entries > 0)
-    {
-      if (hol->num_entries == 0)
-	{
-	  hol->num_entries = more->num_entries;
-	  hol->entries = more->entries;
-	  hol->short_options = more->short_options;
-	  more->num_entries = 0;	/* Mark MORE's fields as invalid.  */
-	}
-      else
-	/* Append the entries in MORE to those in HOL, taking care to only add
-	   non-shadowed SHORT_OPTIONS values.  */
-	{
-	  unsigned left;
-	  char *so, *more_so;
-	  struct hol_entry *e;
-	  unsigned num_entries = hol->num_entries + more->num_entries;
-	  struct hol_entry *entries =
-	    malloc (num_entries * sizeof (struct hol_entry));
-	  unsigned hol_so_len = strlen (hol->short_options);
-	  char *short_options =
-	    malloc (hol_so_len + strlen (more->short_options) + 1);
-
-	  __mempcpy (__mempcpy (entries, hol->entries,
-				hol->num_entries * sizeof (struct hol_entry)),
-		     more->entries,
-		     more->num_entries * sizeof (struct hol_entry));
-
-	  __mempcpy (short_options, hol->short_options, hol_so_len);
-
-	  /* Fix up the short options pointers from HOL.  */
-	  for (e = entries, left = hol->num_entries; left > 0; e++, left--)
-	    e->short_options += (short_options - hol->short_options);
-
-	  /* Now add the short options from MORE, fixing up its entries
-	     too.  */
-	  so = short_options + hol_so_len;
-	  more_so = more->short_options;
-	  for (left = more->num_entries; left > 0; e++, left--)
-	    {
-	      int opts_left;
-	      const struct argp_option *opt;
-
-	      e->short_options = so;
-
-	      for (opts_left = e->num, opt = e->opt; opts_left; opt++, opts_left--)
-		{
-		  int ch = *more_so;
-		  if (oshort (opt) && ch == opt->key)
-		    /* The next short option in MORE_SO, CH, is from OPT.  */
-		    {
-		      if (! find_char (ch, short_options,
-				       short_options + hol_so_len))
-			/* The short option CH isn't shadowed by HOL's options,
-			   so add it to the sum.  */
-			*so++ = ch;
-		      more_so++;
-		    }
-		}
-	    }
-
-	  *so = '\0';
-
-	  free (hol->entries);
-	  free (hol->short_options);
-
-	  hol->entries = entries;
-	  hol->num_entries = num_entries;
-	  hol->short_options = short_options;
-	}
-    }
-
-  hol_free (more);
-}
-
-/* Inserts enough spaces to make sure STREAM is at column COL.  */
-static void
-indent_to (argp_fmtstream_t stream, unsigned col)
-{
-  int needed = col - __argp_fmtstream_point (stream);
-  while (needed-- > 0)
-    __argp_fmtstream_putc (stream, ' ');
-}
-
-/* Output to STREAM either a space, or a newline if there isn't room for at
-   least ENSURE characters before the right margin.  */
-static void
-space (argp_fmtstream_t stream, size_t ensure)
-{
-  if (__argp_fmtstream_point (stream) + ensure
-      >= __argp_fmtstream_rmargin (stream))
-    __argp_fmtstream_putc (stream, '\n');
-  else
-    __argp_fmtstream_putc (stream, ' ');
-}
-
-/* If the option REAL has an argument, we print it in using the printf
-   format REQ_FMT or OPT_FMT depending on whether it's a required or
-   optional argument.  */
-static void
-arg (const struct argp_option *real, const char *req_fmt, const char *opt_fmt,
-     const char *domain UNUSED, argp_fmtstream_t stream)
-{
-  if (real->arg)
-    {
-      if (real->flags & OPTION_ARG_OPTIONAL)
-	__argp_fmtstream_printf (stream, opt_fmt,
-				 dgettext (domain, real->arg));
-      else
-	__argp_fmtstream_printf (stream, req_fmt,
-				 dgettext (domain, real->arg));
-    }
-}
-
-/* Helper functions for hol_entry_help.  */
-
-/* State used during the execution of hol_help.  */
-struct hol_help_state
-{
-  /* PREV_ENTRY should contain the previous entry printed, or 0.  */
-  struct hol_entry *prev_entry;
-
-  /* If an entry is in a different group from the previous one, and SEP_GROUPS
-     is true, then a blank line will be printed before any output. */
-  int sep_groups;
-
-  /* True if a duplicate option argument was suppressed (only ever set if
-     UPARAMS.dup_args is false).  */
-  int suppressed_dup_arg;
-};
-
-/* Some state used while printing a help entry (used to communicate with
-   helper functions).  See the doc for hol_entry_help for more info, as most
-   of the fields are copied from its arguments.  */
-struct pentry_state
-{
-  const struct hol_entry *entry;
-  argp_fmtstream_t stream;
-  struct hol_help_state *hhstate;
-
-  /* True if nothing's been printed so far.  */
-  int first;
-
-  /* If non-zero, the state that was used to print this help.  */
-  const struct argp_state *state;
-};
-
-/* If a user doc filter should be applied to DOC, do so.  */
-static const char *
-filter_doc (const char *doc, int key, const struct argp *argp,
-	    const struct argp_state *state)
-{
-  if (argp->help_filter)
-    /* We must apply a user filter to this output.  */
-    {
-      void *input = __argp_input (argp, state);
-      return (*argp->help_filter) (key, doc, input);
-    }
-  else
-    /* No filter.  */
-    return doc;
-}
-
-/* Prints STR as a header line, with the margin lines set appropiately, and
-   notes the fact that groups should be separated with a blank line.  ARGP is
-   the argp that should dictate any user doc filtering to take place.  Note
-   that the previous wrap margin isn't restored, but the left margin is reset
-   to 0.  */
-static void
-print_header (const char *str, const struct argp *argp,
-	      struct pentry_state *pest)
-{
-  const char *tstr = dgettext (argp->argp_domain, str);
-  const char *fstr = filter_doc (tstr, ARGP_KEY_HELP_HEADER, argp, pest->state);
-
-  if (fstr)
-    {
-      if (*fstr)
-	{
-	  if (pest->hhstate->prev_entry)
-	    /* Precede with a blank line.  */
-	    __argp_fmtstream_putc (pest->stream, '\n');
-	  indent_to (pest->stream, uparams.header_col);
-	  __argp_fmtstream_set_lmargin (pest->stream, uparams.header_col);
-	  __argp_fmtstream_set_wmargin (pest->stream, uparams.header_col);
-	  __argp_fmtstream_puts (pest->stream, fstr);
-	  __argp_fmtstream_set_lmargin (pest->stream, 0);
-	  __argp_fmtstream_putc (pest->stream, '\n');
-	}
-
-      pest->hhstate->sep_groups = 1; /* Separate subsequent groups. */
-    }
-
-  if (fstr != tstr)
-    free ((char *) fstr);
-}
-
-/* Inserts a comma if this isn't the first item on the line, and then makes
-   sure we're at least to column COL.  If this *is* the first item on a line,
-   prints any pending whitespace/headers that should precede this line. Also
-   clears FIRST.  */
-static void
-comma (unsigned col, struct pentry_state *pest)
-{
-  if (pest->first)
-    {
-      const struct hol_entry *pe = pest->hhstate->prev_entry;
-      const struct hol_cluster *cl = pest->entry->cluster;
-
-      if (pest->hhstate->sep_groups && pe && pest->entry->group != pe->group)
-	__argp_fmtstream_putc (pest->stream, '\n');
-
-      if (cl && cl->header && *cl->header
-	  && (!pe
-	      || (pe->cluster != cl
-		  && !hol_cluster_is_child (pe->cluster, cl))))
-	/* If we're changing clusters, then this must be the start of the
-	   ENTRY's cluster unless that is an ancestor of the previous one
-	   (in which case we had just popped into a sub-cluster for a bit).
-	   If so, then print the cluster's header line.  */
-	{
-	  int old_wm = __argp_fmtstream_wmargin (pest->stream);
-	  print_header (cl->header, cl->argp, pest);
-	  __argp_fmtstream_set_wmargin (pest->stream, old_wm);
-	}
-
-      pest->first = 0;
-    }
-  else
-    __argp_fmtstream_puts (pest->stream, ", ");
-
-  indent_to (pest->stream, col);
-}
-
-/* Print help for ENTRY to STREAM.  */
-static void
-hol_entry_help (struct hol_entry *entry, const struct argp_state *state,
-		argp_fmtstream_t stream, struct hol_help_state *hhstate)
-{
-  unsigned num;
-  const struct argp_option *real = entry->opt, *opt;
-  char *so = entry->short_options;
-  int have_long_opt = 0;	/* We have any long options.  */
-  /* Saved margins.  */
-  int old_lm = __argp_fmtstream_set_lmargin (stream, 0);
-  int old_wm = __argp_fmtstream_wmargin (stream);
-  /* PEST is a state block holding some of our variables that we'd like to
-     share with helper functions.  */
-
-  /* Decent initializers are a GNU extension, so don't use it here. */
-  struct pentry_state pest;
-  pest.entry = entry;
-  pest.stream = stream;
-  pest.hhstate = hhstate;
-  pest.first = 1;
-  pest.state = state;
-
-  if (! odoc (real))
-    for (opt = real, num = entry->num; num > 0; opt++, num--)
-      if (opt->name && ovisible (opt))
-	{
-	  have_long_opt = 1;
-	  break;
-	}
-
-  /* First emit short options.  */
-  __argp_fmtstream_set_wmargin (stream, uparams.short_opt_col); /* For truly bizarre cases. */
-  for (opt = real, num = entry->num; num > 0; opt++, num--)
-    if (oshort (opt) && opt->key == *so)
-      /* OPT has a valid (non shadowed) short option.  */
-      {
-	if (ovisible (opt))
-	  {
-	    comma (uparams.short_opt_col, &pest);
-	    __argp_fmtstream_putc (stream, '-');
-	    __argp_fmtstream_putc (stream, *so);
-	    if (!have_long_opt || uparams.dup_args)
-	      arg (real, " %s", "[%s]", state->root_argp->argp_domain, stream);
-	    else if (real->arg)
-	      hhstate->suppressed_dup_arg = 1;
-	  }
-	so++;
-      }
-
-  /* Now, long options.  */
-  if (odoc (real))
-    /* A `documentation' option.  */
-    {
-      __argp_fmtstream_set_wmargin (stream, uparams.doc_opt_col);
-      for (opt = real, num = entry->num; num > 0; opt++, num--)
-	if (opt->name && ovisible (opt))
-	  {
-	    comma (uparams.doc_opt_col, &pest);
-	    /* Calling gettext here isn't quite right, since sorting will
-	       have been done on the original; but documentation options
-	       should be pretty rare anyway...  */
-	    __argp_fmtstream_puts (stream,
-				   dgettext (state->root_argp->argp_domain,
-					     opt->name));
-	  }
-    }
-  else
-    /* A real long option.  */
-    {
-      int first_long_opt = 1;
-
-      __argp_fmtstream_set_wmargin (stream, uparams.long_opt_col);
-      for (opt = real, num = entry->num; num > 0; opt++, num--)
-	if (opt->name && ovisible (opt))
-	  {
-	    comma (uparams.long_opt_col, &pest);
-	    __argp_fmtstream_printf (stream, "--%s", opt->name);
-	    if (first_long_opt || uparams.dup_args)
-	      arg (real, "=%s", "[=%s]", state->root_argp->argp_domain,
-		   stream);
-	    else if (real->arg)
-	      hhstate->suppressed_dup_arg = 1;
-	  }
-    }
-
-  /* Next, documentation strings.  */
-  __argp_fmtstream_set_lmargin (stream, 0);
-
-  if (pest.first)
-    {
-      /* Didn't print any switches, what's up?  */
-      if (!oshort (real) && !real->name)
-	/* This is a group header, print it nicely.  */
-	print_header (real->doc, entry->argp, &pest);
-      else
-	/* Just a totally shadowed option or null header; print nothing.  */
-	goto cleanup;		/* Just return, after cleaning up.  */
-    }
-  else
-    {
-      const char *tstr = real->doc ? dgettext (state->root_argp->argp_domain,
-					       real->doc) : 0;
-      const char *fstr = filter_doc (tstr, real->key, entry->argp, state);
-      if (fstr && *fstr)
-	{
-	  unsigned int col = __argp_fmtstream_point (stream);
-
-	  __argp_fmtstream_set_lmargin (stream, uparams.opt_doc_col);
-	  __argp_fmtstream_set_wmargin (stream, uparams.opt_doc_col);
-
-	  if (col > (unsigned int) (uparams.opt_doc_col + 3))
-	    __argp_fmtstream_putc (stream, '\n');
-	  else if (col >= (unsigned int) uparams.opt_doc_col)
-	    __argp_fmtstream_puts (stream, "   ");
-	  else
-	    indent_to (stream, uparams.opt_doc_col);
-
-	  __argp_fmtstream_puts (stream, fstr);
-	}
-      if (fstr && fstr != tstr)
-	free ((char *) fstr);
-
-      /* Reset the left margin.  */
-      __argp_fmtstream_set_lmargin (stream, 0);
-      __argp_fmtstream_putc (stream, '\n');
-    }
-
-  hhstate->prev_entry = entry;
-
-cleanup:
-  __argp_fmtstream_set_lmargin (stream, old_lm);
-  __argp_fmtstream_set_wmargin (stream, old_wm);
-}
-
-/* Output a long help message about the options in HOL to STREAM.  */
-static void
-hol_help (struct hol *hol, const struct argp_state *state,
-	  argp_fmtstream_t stream)
-{
-  unsigned num;
-  struct hol_entry *entry;
-  struct hol_help_state hhstate = { 0, 0, 0 };
-
-  for (entry = hol->entries, num = hol->num_entries; num > 0; entry++, num--)
-    hol_entry_help (entry, state, stream, &hhstate);
-
-  if (hhstate.suppressed_dup_arg && uparams.dup_args_note)
-    {
-      const char *tstr = dgettext (state->root_argp->argp_domain, "\
-Mandatory or optional arguments to long options are also mandatory or \
-optional for any corresponding short options.");
-      const char *fstr = filter_doc (tstr, ARGP_KEY_HELP_DUP_ARGS_NOTE,
-				     state ? state->root_argp : 0, state);
-      if (fstr && *fstr)
-	{
-	  __argp_fmtstream_putc (stream, '\n');
-	  __argp_fmtstream_puts (stream, fstr);
-	  __argp_fmtstream_putc (stream, '\n');
-	}
-      if (fstr && fstr != tstr)
-	free ((char *) fstr);
-    }
-}
-
-/* Helper functions for hol_usage.  */
-
-/* If OPT is a short option without an arg, append its key to the string
-   pointer pointer to by COOKIE, and advance the pointer.  */
-static int
-add_argless_short_opt (const struct argp_option *opt,
-		       const struct argp_option *real,
-		       const char *domain UNUSED, void *cookie)
-{
-  char **snao_end = cookie;
-  if (!(opt->arg || real->arg)
-      && !((opt->flags | real->flags) & OPTION_NO_USAGE))
-    *(*snao_end)++ = opt->key;
-  return 0;
-}
-
-/* If OPT is a short option with an arg, output a usage entry for it to the
-   stream pointed at by COOKIE.  */
-static int
-usage_argful_short_opt (const struct argp_option *opt,
-			const struct argp_option *real,
-			const char *domain UNUSED, void *cookie)
-{
-  argp_fmtstream_t stream = cookie;
-  const char *arg = opt->arg;
-  int flags = opt->flags | real->flags;
-
-  if (! arg)
-    arg = real->arg;
-
-  if (arg && !(flags & OPTION_NO_USAGE))
-    {
-      arg = dgettext (domain, arg);
-
-      if (flags & OPTION_ARG_OPTIONAL)
-	__argp_fmtstream_printf (stream, " [-%c[%s]]", opt->key, arg);
-      else
-	{
-	  /* Manually do line wrapping so that it (probably) won't
-	     get wrapped at the embedded space.  */
-	  space (stream, 6 + strlen (arg));
-	  __argp_fmtstream_printf (stream, "[-%c %s]", opt->key, arg);
-	}
-    }
-
-  return 0;
-}
-
-/* Output a usage entry for the long option opt to the stream pointed at by
-   COOKIE.  */
-static int
-usage_long_opt (const struct argp_option *opt,
-		const struct argp_option *real,
-		const char *domain UNUSED, void *cookie)
-{
-  argp_fmtstream_t stream = cookie;
-  const char *arg = opt->arg;
-  int flags = opt->flags | real->flags;
-
-  if (! arg)
-    arg = real->arg;
-
-  if (! (flags & OPTION_NO_USAGE))
-    {
-      if (arg)
-	{
-	  arg = dgettext (domain, arg);
-	  if (flags & OPTION_ARG_OPTIONAL)
-	    __argp_fmtstream_printf (stream, " [--%s[=%s]]", opt->name, arg);
-	  else
-	    __argp_fmtstream_printf (stream, " [--%s=%s]", opt->name, arg);
-	}
-      else
-	__argp_fmtstream_printf (stream, " [--%s]", opt->name);
-    }
-
-  return 0;
-}
-
-/* Print a short usage description for the arguments in HOL to STREAM.  */
-static void
-hol_usage (struct hol *hol, argp_fmtstream_t stream)
-{
-  if (hol->num_entries > 0)
-    {
-      unsigned nentries;
-      struct hol_entry *entry;
-      char *short_no_arg_opts = alloca (strlen (hol->short_options) + 1);
-      char *snao_end = short_no_arg_opts;
-
-      /* First we put a list of short options without arguments.  */
-      for (entry = hol->entries, nentries = hol->num_entries
-	   ; nentries > 0
-	   ; entry++, nentries--)
-	hol_entry_short_iterate (entry, add_argless_short_opt,
-				 entry->argp->argp_domain, &snao_end);
-      if (snao_end > short_no_arg_opts)
-	{
-	  *snao_end++ = 0;
-	  __argp_fmtstream_printf (stream, " [-%s]", short_no_arg_opts);
-	}
-
-      /* Now a list of short options *with* arguments.  */
-      for (entry = hol->entries, nentries = hol->num_entries
-	   ; nentries > 0
-	   ; entry++, nentries--)
-	hol_entry_short_iterate (entry, usage_argful_short_opt,
-				 entry->argp->argp_domain, stream);
-
-      /* Finally, a list of long options (whew!).  */
-      for (entry = hol->entries, nentries = hol->num_entries
-	   ; nentries > 0
-	   ; entry++, nentries--)
-	hol_entry_long_iterate (entry, usage_long_opt,
-				entry->argp->argp_domain, stream);
-    }
-}
-
-/* Make a HOL containing all levels of options in ARGP.  CLUSTER is the
-   cluster in which ARGP's entries should be clustered, or 0.  */
-static struct hol *
-argp_hol (const struct argp *argp, struct hol_cluster *cluster)
-{
-  const struct argp_child *child = argp->children;
-  struct hol *hol = make_hol (argp, cluster);
-  if (child)
-    while (child->argp)
-      {
-	struct hol_cluster *child_cluster =
-	  ((child->group || child->header)
-	   /* Put CHILD->argp within its own cluster.  */
-	   ? hol_add_cluster (hol, child->group, child->header,
-			      child - argp->children, cluster, argp)
-	   /* Just merge it into the parent's cluster.  */
-	   : cluster);
-	hol_append (hol, argp_hol (child->argp, child_cluster)) ;
-	child++;
-      }
-  return hol;
-}
-
-/* Calculate how many different levels with alternative args strings exist in
-   ARGP.  */
-static size_t
-argp_args_levels (const struct argp *argp)
-{
-  size_t levels = 0;
-  const struct argp_child *child = argp->children;
-
-  if (argp->args_doc && strchr (argp->args_doc, '\n'))
-    levels++;
-
-  if (child)
-    while (child->argp)
-      levels += argp_args_levels ((child++)->argp);
-
-  return levels;
-}
-
-/* Print all the non-option args documented in ARGP to STREAM.  Any output is
-   preceded by a space.  LEVELS is a pointer to a byte vector the length
-   returned by argp_args_levels; it should be initialized to zero, and
-   updated by this routine for the next call if ADVANCE is true.  True is
-   returned as long as there are more patterns to output.  */
-static int
-argp_args_usage (const struct argp *argp, const struct argp_state *state,
-		 char **levels, int advance, argp_fmtstream_t stream)
-{
-  char *our_level = *levels;
-  int multiple = 0;
-  const struct argp_child *child = argp->children;
-  const char *tdoc = dgettext (argp->argp_domain, argp->args_doc), *nl = 0;
-  const char *fdoc = filter_doc (tdoc, ARGP_KEY_HELP_ARGS_DOC, argp, state);
-
-  if (fdoc)
-    {
-      const char *cp = fdoc;
-      nl = __strchrnul (cp, '\n');
-      if (*nl != '\0')
-	/* This is a `multi-level' args doc; advance to the correct position
-	   as determined by our state in LEVELS, and update LEVELS.  */
-	{
-	  int i;
-	  multiple = 1;
-	  for (i = 0; i < *our_level; i++)
-	    cp = nl + 1, nl = __strchrnul (cp, '\n');
-	  (*levels)++;
-	}
-
-      /* Manually do line wrapping so that it (probably) won't get wrapped at
-	 any embedded spaces.  */
-      space (stream, 1 + nl - cp);
-
-      __argp_fmtstream_write (stream, cp, nl - cp);
-    }
-  if (fdoc && fdoc != tdoc)
-    free ((char *)fdoc);	/* Free user's modified doc string.  */
-
-  if (child)
-    while (child->argp)
-      advance = !argp_args_usage ((child++)->argp, state, levels, advance, stream);
-
-  if (advance && multiple)
-    {
-      /* Need to increment our level.  */
-      if (*nl)
-	/* There's more we can do here.  */
-	{
-	  (*our_level)++;
-	  advance = 0;		/* Our parent shouldn't advance also. */
-	}
-      else if (*our_level > 0)
-	/* We had multiple levels, but used them up; reset to zero.  */
-	*our_level = 0;
-    }
-
-  return !advance;
-}
-
-/* Print the documentation for ARGP to STREAM; if POST is false, then
-   everything preceeding a `\v' character in the documentation strings (or
-   the whole string, for those with none) is printed, otherwise, everything
-   following the `\v' character (nothing for strings without).  Each separate
-   bit of documentation is separated a blank line, and if PRE_BLANK is true,
-   then the first is as well.  If FIRST_ONLY is true, only the first
-   occurrence is output.  Returns true if anything was output.  */
-static int
-argp_doc (const struct argp *argp, const struct argp_state *state,
-	  int post, int pre_blank, int first_only,
-	  argp_fmtstream_t stream)
-{
-  const char *text;
-  const char *inp_text;
-  void *input = 0;
-  int anything = 0;
-  size_t inp_text_limit = 0;
-  const char *doc = dgettext (argp->argp_domain, argp->doc);
-  const struct argp_child *child = argp->children;
-
-  if (doc)
-    {
-      char *vt = strchr (doc, '\v');
-      inp_text = post ? (vt ? vt + 1 : 0) : doc;
-      inp_text_limit = (!post && vt) ? (vt - doc) : 0;
-    }
-  else
-    inp_text = 0;
-
-  if (argp->help_filter)
-    /* We have to filter the doc strings.  */
-    {
-      if (inp_text_limit)
-	/* Copy INP_TEXT so that it's nul-terminated.  */
-	inp_text = STRNDUP (inp_text, inp_text_limit);
-      input = __argp_input (argp, state);
-      text =
-	(*argp->help_filter) (post
-			      ? ARGP_KEY_HELP_POST_DOC
-			      : ARGP_KEY_HELP_PRE_DOC,
-			      inp_text, input);
-    }
-  else
-    text = (const char *) inp_text;
-
-  if (text)
-    {
-      if (pre_blank)
-	__argp_fmtstream_putc (stream, '\n');
-
-      if (text == inp_text && inp_text_limit)
-	__argp_fmtstream_write (stream, inp_text, inp_text_limit);
-      else
-	__argp_fmtstream_puts (stream, text);
-
-      if (__argp_fmtstream_point (stream) > __argp_fmtstream_lmargin (stream))
-	__argp_fmtstream_putc (stream, '\n');
-
-      anything = 1;
-    }
-
-  if (text && text != inp_text)
-    free ((char *) text);	/* Free TEXT returned from the help filter.  */
-  if (inp_text && inp_text_limit && argp->help_filter)
-    free ((char *) inp_text);	/* We copied INP_TEXT, so free it now.  */
-
-  if (post && argp->help_filter)
-    /* Now see if we have to output a ARGP_KEY_HELP_EXTRA text.  */
-    {
-      text = (*argp->help_filter) (ARGP_KEY_HELP_EXTRA, 0, input);
-      if (text)
-	{
-	  if (anything || pre_blank)
-	    __argp_fmtstream_putc (stream, '\n');
-	  __argp_fmtstream_puts (stream, text);
-	  free ((char *) text);
-	  if (__argp_fmtstream_point (stream)
-	      > __argp_fmtstream_lmargin (stream))
-	    __argp_fmtstream_putc (stream, '\n');
-	  anything = 1;
-	}
-    }
-
-  if (child)
-    while (child->argp && !(first_only && anything))
-      anything |=
-	argp_doc ((child++)->argp, state,
-		  post, anything || pre_blank, first_only,
-		  stream);
-
-  return anything;
-}
-
-/* Output a usage message for ARGP to STREAM.  If called from
-   argp_state_help, STATE is the relevent parsing state.  FLAGS are from the
-   set ARGP_HELP_*.  NAME is what to use wherever a `program name' is
-   needed. */
-
-static void
-_help (const struct argp *argp, const struct argp_state *state, FILE *stream,
-       unsigned flags, const char *name)
-{
-  int anything = 0;		/* Whether we've output anything.  */
-  struct hol *hol = 0;
-  argp_fmtstream_t fs;
-
-  if (! stream)
-    return;
-
-  FLOCKFILE (stream);
-
-  if (! uparams.valid)
-    fill_in_uparams (state);
-
-  fs = __argp_make_fmtstream (stream, 0, uparams.rmargin, 0);
-  if (! fs)
-    {
-      FUNLOCKFILE (stream);
-      return;
-    }
-
-  if (flags & (ARGP_HELP_USAGE | ARGP_HELP_SHORT_USAGE | ARGP_HELP_LONG))
-    {
-      hol = argp_hol (argp, 0);
-
-      /* If present, these options always come last.  */
-      hol_set_group (hol, "help", -1);
-      hol_set_group (hol, "version", -1);
-
-      hol_sort (hol);
-    }
-
-  if (flags & (ARGP_HELP_USAGE | ARGP_HELP_SHORT_USAGE))
-    /* Print a short `Usage:' message.  */
-    {
-      int first_pattern = 1, more_patterns;
-      size_t num_pattern_levels = argp_args_levels (argp);
-      char *pattern_levels = alloca (num_pattern_levels);
-
-      memset (pattern_levels, 0, num_pattern_levels);
-
-      do
-	{
-	  int old_lm;
-	  int old_wm = __argp_fmtstream_set_wmargin (fs, uparams.usage_indent);
-	  char *levels = pattern_levels;
-
-	  if (first_pattern)
-	    __argp_fmtstream_printf (fs, "%s %s",
-				     dgettext (argp->argp_domain, "Usage:"),
-				     name);
-	  else
-	    __argp_fmtstream_printf (fs, "%s %s",
-				     dgettext (argp->argp_domain, "  or: "),
-				     name);
-
-	  /* We set the lmargin as well as the wmargin, because hol_usage
-	     manually wraps options with newline to avoid annoying breaks.  */
-	  old_lm = __argp_fmtstream_set_lmargin (fs, uparams.usage_indent);
-
-	  if (flags & ARGP_HELP_SHORT_USAGE)
-	    /* Just show where the options go.  */
-	    {
-	      if (hol->num_entries > 0)
-		__argp_fmtstream_puts (fs, dgettext (argp->argp_domain,
-						     " [OPTION...]"));
-	    }
-	  else
-	    /* Actually print the options.  */
-	    {
-	      hol_usage (hol, fs);
-	      flags |= ARGP_HELP_SHORT_USAGE; /* But only do so once.  */
-	    }
-
-	  more_patterns = argp_args_usage (argp, state, &levels, 1, fs);
-
-	  __argp_fmtstream_set_wmargin (fs, old_wm);
-	  __argp_fmtstream_set_lmargin (fs, old_lm);
-
-	  __argp_fmtstream_putc (fs, '\n');
-	  anything = 1;
-
-	  first_pattern = 0;
-	}
-      while (more_patterns);
-    }
-
-  if (flags & ARGP_HELP_PRE_DOC)
-    anything |= argp_doc (argp, state, 0, 0, 1, fs);
-
-  if (flags & ARGP_HELP_SEE)
-    {
-      __argp_fmtstream_printf (fs, dgettext (argp->argp_domain, "\
-Try `%s --help' or `%s --usage' for more information.\n"),
-			       name, name);
-      anything = 1;
-    }
-
-  if (flags & ARGP_HELP_LONG)
-    /* Print a long, detailed help message.  */
-    {
-      /* Print info about all the options.  */
-      if (hol->num_entries > 0)
-	{
-	  if (anything)
-	    __argp_fmtstream_putc (fs, '\n');
-	  hol_help (hol, state, fs);
-	  anything = 1;
-	}
-    }
-
-  if (flags & ARGP_HELP_POST_DOC)
-    /* Print any documentation strings at the end.  */
-    anything |= argp_doc (argp, state, 1, anything, 0, fs);
-
-  if ((flags & ARGP_HELP_BUG_ADDR) && argp_program_bug_address)
-    {
-      if (anything)
-	__argp_fmtstream_putc (fs, '\n');
-      __argp_fmtstream_printf (fs, dgettext (argp->argp_domain,
-					     "Report bugs to %s.\n"),
- 			       argp_program_bug_address);
-      anything = 1;
-    }
-
-  FUNLOCKFILE (stream);
-
-  if (hol)
-    hol_free (hol);
-
-  __argp_fmtstream_free (fs);
-}
-
-/* Output a usage message for ARGP to STREAM.  FLAGS are from the set
-   ARGP_HELP_*.  NAME is what to use wherever a `program name' is needed. */
-void __argp_help (const struct argp *argp, FILE *stream,
-		  unsigned flags, char *name)
-{
-  _help (argp, 0, stream, flags, name);
-}
-#ifdef weak_alias
-weak_alias (__argp_help, argp_help)
-#endif
-
-char *__argp_basename(char *name)
-{
-  char *short_name = strrchr(name, '/');
-  return short_name ? short_name + 1 : name;
-}
-
-char *
-__argp_short_program_name(const struct argp_state *state)
-{
-  if (state)
-    return state->name;
-#if HAVE_DECL_PROGRAM_INVOCATION_SHORT_NAME
-  return program_invocation_short_name;
-#elif HAVE_DECL_PROGRAM_INVOCATION_NAME
-  return __argp_basename(program_invocation_name);
-#else /* !HAVE_DECL_PROGRAM_INVOCATION_NAME */
-  /* FIXME: What now? Miles suggests that it is better to use NULL,
-     but currently the value is passed on directly to fputs_unlocked,
-     so that requires more changes. */
-# if __GNUC__
-  return "";
-# endif /* __GNUC__ */
-#endif /* !HAVE_DECL_PROGRAM_INVOCATION_NAME */
-}
-
-/* Output, if appropriate, a usage message for STATE to STREAM.  FLAGS are
-   from the set ARGP_HELP_*.  */
-void
-__argp_state_help (const struct argp_state *state, FILE *stream, unsigned flags)
-{
-  if ((!state || ! (state->flags & ARGP_NO_ERRS)) && stream)
-    {
-      if (state && (state->flags & ARGP_LONG_ONLY))
-	flags |= ARGP_HELP_LONG_ONLY;
-
-      _help (state ? state->root_argp : 0, state, stream, flags,
-	     __argp_short_program_name(state));
-
-      if (!state || ! (state->flags & ARGP_NO_EXIT))
-	{
-	  if (flags & ARGP_HELP_EXIT_ERR)
-	    exit (argp_err_exit_status);
-	  if (flags & ARGP_HELP_EXIT_OK)
-	    exit (0);
-	}
-  }
-}
-#ifdef weak_alias
-weak_alias (__argp_state_help, argp_state_help)
-#endif
-
-/* If appropriate, print the printf string FMT and following args, preceded
-   by the program name and `:', to stderr, and followed by a `Try ... --help'
-   message, then exit (1).  */
-void
-__argp_error (const struct argp_state *state, const char *fmt, ...)
-{
-  if (!state || !(state->flags & ARGP_NO_ERRS))
-    {
-      FILE *stream = state ? state->err_stream : stderr;
-
-      if (stream)
-	{
-	  va_list ap;
-
-	  FLOCKFILE (stream);
-
-	  FPUTS_UNLOCKED (__argp_short_program_name(state),
-			  stream);
-	  PUTC_UNLOCKED (':', stream);
-	  PUTC_UNLOCKED (' ', stream);
-
-	  va_start (ap, fmt);
-	  vfprintf (stream, fmt, ap);
-	  va_end (ap);
-
-	  PUTC_UNLOCKED ('\n', stream);
-
-	  __argp_state_help (state, stream, ARGP_HELP_STD_ERR);
-
-	  FUNLOCKFILE (stream);
-	}
-    }
-}
-#ifdef weak_alias
-weak_alias (__argp_error, argp_error)
-#endif
-
-/* Similar to the standard gnu error-reporting function error(), but will
-   respect the ARGP_NO_EXIT and ARGP_NO_ERRS flags in STATE, and will print
-   to STATE->err_stream.  This is useful for argument parsing code that is
-   shared between program startup (when exiting is desired) and runtime
-   option parsing (when typically an error code is returned instead).  The
-   difference between this function and argp_error is that the latter is for
-   *parsing errors*, and the former is for other problems that occur during
-   parsing but don't reflect a (syntactic) problem with the input.  */
-void
-__argp_failure (const struct argp_state *state, int status, int errnum,
-		const char *fmt, ...)
-{
-  if (!state || !(state->flags & ARGP_NO_ERRS))
-    {
-      FILE *stream = state ? state->err_stream : stderr;
-
-      if (stream)
-	{
-	  FLOCKFILE (stream);
-
-	  FPUTS_UNLOCKED (__argp_short_program_name(state),
-			  stream);
-
-	  if (fmt)
-	    {
-	      va_list ap;
-
-	      PUTC_UNLOCKED (':', stream);
-	      PUTC_UNLOCKED (' ', stream);
-
-	      va_start (ap, fmt);
-	      vfprintf (stream, fmt, ap);
-	      va_end (ap);
-	    }
-
-	  if (errnum)
-	    {
-	      PUTC_UNLOCKED (':', stream);
-	      PUTC_UNLOCKED (' ', stream);
-	      fputs (STRERROR (errnum), stream);
-	    }
-
-	  PUTC_UNLOCKED ('\n', stream);
-
-	  FUNLOCKFILE (stream);
-
-	  if (status && (!state || !(state->flags & ARGP_NO_EXIT)))
-	    exit (status);
-	}
-    }
-}
-#ifdef weak_alias
-weak_alias (__argp_failure, argp_failure)
-#endif
diff --git a/contrib/argp-standalone/argp-namefrob.h b/contrib/argp-standalone/argp-namefrob.h
deleted file mode 100644
index 0ce11481a7b..00000000000
--- a/contrib/argp-standalone/argp-namefrob.h
+++ /dev/null
@@ -1,96 +0,0 @@
-/* Name frobnication for compiling argp outside of glibc
-   Copyright (C) 1997 Free Software Foundation, Inc.
-   This file is part of the GNU C Library.
-   Written by Miles Bader <miles@gnu.ai.mit.edu>.
-
-   The GNU C Library is free software; you can redistribute it and/or
-   modify it under the terms of the GNU Library General Public License as
-   published by the Free Software Foundation; either version 2 of the
-   License, or (at your option) any later version.
-
-   The GNU C Library is distributed in the hope that it will be useful,
-   but WITHOUT ANY WARRANTY; without even the implied warranty of
-   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-   Library General Public License for more details.
-
-   You should have received a copy of the GNU Library General Public
-   License along with the GNU C Library; see the file COPYING.LIB.  If not,
-   write to the Free Software Foundation, Inc., 59 Temple Place - Suite 330,
-   Boston, MA 02111-1307, USA.  */
-
-#if !_LIBC
-/* This code is written for inclusion in gnu-libc, and uses names in the
-   namespace reserved for libc.  If we're not compiling in libc, define those
-   names to be the normal ones instead.  */
-
-/* argp-parse functions */
-#undef __argp_parse
-#define __argp_parse argp_parse
-#undef __option_is_end
-#define __option_is_end _option_is_end
-#undef __option_is_short
-#define __option_is_short _option_is_short
-#undef __argp_input
-#define __argp_input _argp_input
-
-/* argp-help functions */
-#undef __argp_help
-#define __argp_help argp_help
-#undef __argp_error
-#define __argp_error argp_error
-#undef __argp_failure
-#define __argp_failure argp_failure
-#undef __argp_state_help
-#define __argp_state_help argp_state_help
-#undef __argp_usage
-#define __argp_usage argp_usage
-#undef __argp_basename
-#define __argp_basename _argp_basename
-#undef __argp_short_program_name
-#define __argp_short_program_name _argp_short_program_name
-
-/* argp-fmtstream functions */
-#undef __argp_make_fmtstream
-#define __argp_make_fmtstream argp_make_fmtstream
-#undef __argp_fmtstream_free
-#define __argp_fmtstream_free argp_fmtstream_free
-#undef __argp_fmtstream_putc
-#define __argp_fmtstream_putc argp_fmtstream_putc
-#undef __argp_fmtstream_puts
-#define __argp_fmtstream_puts argp_fmtstream_puts
-#undef __argp_fmtstream_write
-#define __argp_fmtstream_write argp_fmtstream_write
-#undef __argp_fmtstream_printf
-#define __argp_fmtstream_printf argp_fmtstream_printf
-#undef __argp_fmtstream_set_lmargin
-#define __argp_fmtstream_set_lmargin argp_fmtstream_set_lmargin
-#undef __argp_fmtstream_set_rmargin
-#define __argp_fmtstream_set_rmargin argp_fmtstream_set_rmargin
-#undef __argp_fmtstream_set_wmargin
-#define __argp_fmtstream_set_wmargin argp_fmtstream_set_wmargin
-#undef __argp_fmtstream_point
-#define __argp_fmtstream_point argp_fmtstream_point
-#undef __argp_fmtstream_update
-#define __argp_fmtstream_update _argp_fmtstream_update
-#undef __argp_fmtstream_ensure
-#define __argp_fmtstream_ensure _argp_fmtstream_ensure
-#undef __argp_fmtstream_lmargin
-#define __argp_fmtstream_lmargin argp_fmtstream_lmargin
-#undef __argp_fmtstream_rmargin
-#define __argp_fmtstream_rmargin argp_fmtstream_rmargin
-#undef __argp_fmtstream_wmargin
-#define __argp_fmtstream_wmargin argp_fmtstream_wmargin
-
-/* normal libc functions we call */
-#undef __sleep
-#define __sleep sleep
-#undef __strcasecmp
-#define __strcasecmp strcasecmp
-#undef __vsnprintf
-#define __vsnprintf vsnprintf
-
-#endif /* !_LIBC */
-
-#ifndef __set_errno
-#define __set_errno(e) (errno = (e))
-#endif
diff --git a/contrib/argp-standalone/argp-parse.c b/contrib/argp-standalone/argp-parse.c
deleted file mode 100644
index 78f7bf139b6..00000000000
--- a/contrib/argp-standalone/argp-parse.c
+++ /dev/null
@@ -1,1305 +0,0 @@
-/* Hierarchial argument parsing
-   Copyright (C) 1995, 96, 97, 98, 99, 2000,2003 Free Software Foundation, Inc.
-   This file is part of the GNU C Library.
-   Written by Miles Bader <miles@gnu.ai.mit.edu>.
-
-   The GNU C Library is free software; you can redistribute it and/or
-   modify it under the terms of the GNU Library General Public License as
-   published by the Free Software Foundation; either version 2 of the
-   License, or (at your option) any later version.
-
-   The GNU C Library is distributed in the hope that it will be useful,
-   but WITHOUT ANY WARRANTY; without even the implied warranty of
-   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-   Library General Public License for more details.
-
-   You should have received a copy of the GNU Library General Public
-   License along with the GNU C Library; see the file COPYING.LIB.  If not,
-   write to the Free Software Foundation, Inc., 59 Temple Place - Suite 330,
-   Boston, MA 02111-1307, USA.  */
-
-#ifndef _GNU_SOURCE
-# define _GNU_SOURCE	1
-#endif
-
-#ifdef HAVE_CONFIG_H
-#include <config.h>
-#endif
-
-#if HAVE_ALLOCA_H
-#include <alloca.h>
-#endif
-
-#include <stdlib.h>
-#include <string.h>
-#if HAVE_UNISTD_H
-# include <unistd.h>
-#endif
-#include <limits.h>
-#include <assert.h>
-
-#if HAVE_MALLOC_H
-/* Needed, for alloca on windows */
-# include <malloc.h>
-#endif
-
-#ifndef _
-/* This is for other GNU distributions with internationalized messages.
-   When compiling libc, the _ macro is predefined.  */
-# if defined HAVE_LIBINTL_H || defined _LIBC
-#  include <libintl.h>
-#  ifdef _LIBC
-#   undef dgettext
-#   define dgettext(domain, msgid) __dcgettext (domain, msgid, LC_MESSAGES)
-#  endif
-# else
-#  define dgettext(domain, msgid) (msgid)
-#  define gettext(msgid) (msgid)
-# endif
-#endif
-#ifndef N_
-# define N_(msgid) (msgid)
-#endif
-
-#if _LIBC - 0
-#include <bits/libc-lock.h>
-#else
-#ifdef HAVE_CTHREADS_H
-#include <cthreads.h>
-#endif
-#endif /* _LIBC */
-
-#include "argp.h"
-#include "argp-namefrob.h"
-
-
-/* The meta-argument used to prevent any further arguments being interpreted
-   as options.  */
-#define QUOTE "--"
-
-/* EZ alias for ARGP_ERR_UNKNOWN.  */
-#define EBADKEY ARGP_ERR_UNKNOWN
-
-
-/* Default options.  */
-
-/* When argp is given the --HANG switch, _ARGP_HANG is set and argp will sleep
-   for one second intervals, decrementing _ARGP_HANG until it's zero.  Thus
-   you can force the program to continue by attaching a debugger and setting
-   it to 0 yourself.  */
-volatile int _argp_hang;
-
-#define OPT_PROGNAME	-2
-#define OPT_USAGE	-3
-#if HAVE_SLEEP && HAVE_GETPID
-#define OPT_HANG	-4
-#endif
-
-static const struct argp_option argp_default_options[] =
-{
-  {"help",	  '?',    	0, 0,  N_("Give this help list"), -1},
-  {"usage",	  OPT_USAGE,	0, 0,  N_("Give a short usage message"), 0 },
-  {"program-name",OPT_PROGNAME,"NAME", OPTION_HIDDEN,
-     N_("Set the program name"), 0},
-#if OPT_HANG
-  {"HANG",	  OPT_HANG,    "SECS", OPTION_ARG_OPTIONAL | OPTION_HIDDEN,
-     N_("Hang for SECS seconds (default 3600)"), 0 },
-#endif
-  {0, 0, 0, 0, 0, 0}
-};
-
-static error_t
-argp_default_parser (int key, char *arg, struct argp_state *state)
-{
-  switch (key)
-    {
-    case '?':
-      __argp_state_help (state, state->out_stream, ARGP_HELP_STD_HELP);
-      break;
-    case OPT_USAGE:
-      __argp_state_help (state, state->out_stream,
-		       ARGP_HELP_USAGE | ARGP_HELP_EXIT_OK);
-      break;
-
-    case OPT_PROGNAME:		/* Set the program name.  */
-#if HAVE_DECL_PROGRAM_INVOCATION_NAME
-      program_invocation_name = arg;
-#endif
-      /* [Note that some systems only have PROGRAM_INVOCATION_SHORT_NAME (aka
-	 __PROGNAME), in which case, PROGRAM_INVOCATION_NAME is just defined
-	 to be that, so we have to be a bit careful here.]  */
-
-      /* Update what we use for messages.  */
-
-      state->name = __argp_basename(arg);
-      
-#if HAVE_DECL_PROGRAM_INVOCATION_SHORT_NAME
-      program_invocation_short_name = state->name;
-#endif
-
-      if ((state->flags & (ARGP_PARSE_ARGV0 | ARGP_NO_ERRS))
-	  == ARGP_PARSE_ARGV0)
-	/* Update what getopt uses too.  */
-	state->argv[0] = arg;
-
-      break;
-      
-#if OPT_HANG
-    case OPT_HANG:
-      _argp_hang = atoi (arg ? arg : "3600");
-      fprintf(state->err_stream, "%s: pid = %ld\n",
-	      state->name, (long) getpid());
-      while (_argp_hang-- > 0)
-	__sleep (1);
-      break;
-#endif
-      
-    default:
-      return EBADKEY;
-    }
-  return 0;
-}
-
-static const struct argp argp_default_argp =
-  {argp_default_options, &argp_default_parser, NULL, NULL, NULL, NULL, "libc"};
-
-
-static const struct argp_option argp_version_options[] =
-{
-  {"version",	  'V',    	0, 0,  N_("Print program version"), -1},
-  {0, 0, 0, 0, 0, 0 }
-};
-
-static error_t
-argp_version_parser (int key, char *arg UNUSED, struct argp_state *state)
-{
-  switch (key)
-    {
-    case 'V':
-      if (argp_program_version_hook)
-	(*argp_program_version_hook) (state->out_stream, state);
-      else if (argp_program_version)
-	fprintf (state->out_stream, "%s\n", argp_program_version);
-      else
-	__argp_error (state, dgettext (state->root_argp->argp_domain,
-				       "(PROGRAM ERROR) No version known!?"));
-      if (! (state->flags & ARGP_NO_EXIT))
-	exit (0);
-      break;
-    default:
-      return EBADKEY;
-    }
-  return 0;
-}
-
-static const struct argp argp_version_argp =
-  {argp_version_options, &argp_version_parser, NULL, NULL, NULL, NULL, "libc"};
-
-
-
-/* The state of a `group' during parsing.  Each group corresponds to a
-   particular argp structure from the tree of such descending from the top
-   level argp passed to argp_parse.  */
-struct group
-{
-  /* This group's parsing function.  */
-  argp_parser_t parser;
-
-  /* Which argp this group is from.  */
-  const struct argp *argp;
-
-  /* The number of non-option args sucessfully handled by this parser.  */
-  unsigned args_processed;
-
-  /* This group's parser's parent's group.  */
-  struct group *parent;
-  unsigned parent_index;	/* And the our position in the parent.   */
-
-  /* These fields are swapped into and out of the state structure when
-     calling this group's parser.  */
-  void *input, **child_inputs;
-  void *hook;
-};
-
-/* Call GROUP's parser with KEY and ARG, swapping any group-specific info
-   from STATE before calling, and back into state afterwards.  If GROUP has
-   no parser, EBADKEY is returned.  */
-static error_t
-group_parse (struct group *group, struct argp_state *state, int key, char *arg)
-{
-  if (group->parser)
-    {
-      error_t err;
-      state->hook = group->hook;
-      state->input = group->input;
-      state->child_inputs = group->child_inputs;
-      state->arg_num = group->args_processed;
-      err = (*group->parser)(key, arg, state);
-      group->hook = state->hook;
-      return err;
-    }
-  else
-    return EBADKEY;
-}
-
-struct parser
-{
-  const struct argp *argp;
-
-  const char *posixly_correct;
-  
-  /* True if there are only no-option arguments left, which are just
-     passed verbatim with ARGP_KEY_ARG. This is set if we encounter a
-     quote, or the end of the proper options, but may be cleared again
-     if the user moves the next argument pointer backwards. */
-  int args_only;
-
-  /* Describe how to deal with options that follow non-option ARGV-elements.
-
-     If the caller did not specify anything, the default is
-     REQUIRE_ORDER if the environment variable POSIXLY_CORRECT is
-     defined, PERMUTE otherwise.
-
-     REQUIRE_ORDER means don't recognize them as options; stop option
-     processing when the first non-option is seen. This is what Unix
-     does. This mode of operation is selected by either setting the
-     environment variable POSIXLY_CORRECT, or using `+' as the first
-     character of the list of option characters.
-
-     PERMUTE is the default. We permute the contents of ARGV as we
-     scan, so that eventually all the non-options are at the end. This
-     allows options to be given in any order, even with programs that
-     were not written to expect this.
-
-     RETURN_IN_ORDER is an option available to programs that were
-     written to expect options and other ARGV-elements in any order
-     and that care about the ordering of the two. We describe each
-     non-option ARGV-element as if it were the argument of an option
-     with character code 1. Using `-' as the first character of the
-     list of option characters selects this mode of operation.
-
-  */
-  enum { REQUIRE_ORDER, PERMUTE, RETURN_IN_ORDER } ordering;
-
-  /* A segment of non-option arguments that have been skipped for
-     later processing, after all options. `first_nonopt' is the index
-     in ARGV of the first of them; `last_nonopt' is the index after
-     the last of them.
-
-     If quoted or args_only is non-zero, this segment should be empty. */
-
-  /* FIXME: I'd prefer to use unsigned, but it's more consistent to
-     use the same type as for state.next. */
-  int first_nonopt;
-  int last_nonopt;
-  
-  /* String of all recognized short options. Needed for ARGP_LONG_ONLY. */
-  /* FIXME: Perhaps change to a pointer to a suitable bitmap instead? */
-  char *short_opts;
-
-  /* For parsing combined short options. */
-  char *nextchar;
-  
-  /* States of the various parsing groups.  */
-  struct group *groups;
-  /* The end of the GROUPS array.  */
-  struct group *egroup;
-  /* An vector containing storage for the CHILD_INPUTS field in all groups.  */
-  void **child_inputs;
-  
-  /* State block supplied to parsing routines.  */
-  struct argp_state state;
-
-  /* Memory used by this parser.  */
-  void *storage;
-};
-
-/* Search for a group defining a short option. */
-static const struct argp_option *
-find_short_option(struct parser *parser, int key, struct group **p)
-{
-  struct group *group;
-  
-  assert(key >= 0);
-  assert(isascii(key));
-
-  for (group = parser->groups; group < parser->egroup; group++)
-    {
-      const struct argp_option *opts;
-
-      for (opts = group->argp->options; !__option_is_end(opts); opts++)
-	if (opts->key == key)
-	  {
-	    *p = group;
-	    return opts;
-	  }
-    }
-  return NULL;
-}
-
-enum match_result { MATCH_EXACT, MATCH_PARTIAL, MATCH_NO };
-
-/* If defined, allow complete.el-like abbreviations of long options. */
-#ifndef ARGP_COMPLETE
-#define ARGP_COMPLETE 0
-#endif
-
-/* Matches an encountern long-option argument ARG against an option NAME.
- * ARG is terminated by NUL or '='. */
-static enum match_result
-match_option(const char *arg, const char *name)
-{
-  unsigned i, j;
-  for (i = j = 0;; i++, j++)
-    {
-      switch(arg[i])
-	{
-	case '\0':
-	case '=':
-	  return name[j] ? MATCH_PARTIAL : MATCH_EXACT;
-#if ARGP_COMPLETE
-	case '-':
-	  while (name[j] != '-')
-	    if (!name[j++])
-	      return MATCH_NO;
-	  break;
-#endif
-	default:
-	  if (arg[i] != name[j])
-	    return MATCH_NO;
-	}
-    }
-}
-
-static const struct argp_option *
-find_long_option(struct parser *parser,
-		 const char *arg,
-		 struct group **p)
-{
-  struct group *group;
-
-  /* Partial match found so far. */
-  struct group *matched_group = NULL;
-  const struct argp_option *matched_option = NULL;
-
-  /* Number of partial matches. */
-  int num_partial = 0;
-
-  for (group = parser->groups; group < parser->egroup; group++)
-    {
-      const struct argp_option *opts;
-
-      for (opts = group->argp->options; !__option_is_end(opts); opts++)
-	{
-	  if (!opts->name)
-	    continue;
-	  switch (match_option(arg, opts->name))
-	    {
-	    case MATCH_NO:
-	      break;
-	    case MATCH_PARTIAL:
-	      num_partial++;
-
-	      matched_group = group;
-	      matched_option = opts;
-
-	      break;
-	    case MATCH_EXACT:
-	      /* Exact match. */
-	      *p = group;
-	      return opts;
-	    }
-	}
-    }
-  if (num_partial == 1)
-    {
-      *p = matched_group;
-      return matched_option;
-    }
-
-  return NULL;
-}
-
-
-/* The next usable entries in the various parser tables being filled in by
-   convert_options.  */
-struct parser_convert_state
-{
-  struct parser *parser;
-  char *short_end;
-  void **child_inputs_end;
-};
-
-/* Initialize GROUP from ARGP. If CVT->SHORT_END is non-NULL, short
-   options are recorded in the short options string. Returns the next
-   unused group entry. CVT holds state used during the conversion. */
-static struct group *
-convert_options (const struct argp *argp,
-		 struct group *parent, unsigned parent_index,
-		 struct group *group, struct parser_convert_state *cvt)
-{
-  const struct argp_option *opt = argp->options;
-  const struct argp_child *children = argp->children;
-
-  if (opt || argp->parser)
-    {
-      /* This parser needs a group. */
-      if (cvt->short_end)
-	{
-	  /* Record any short options. */
-	  for ( ; !__option_is_end (opt); opt++)
-	    if (__option_is_short(opt))
-	      *cvt->short_end++ = opt->key;
-	}
-      
-      group->parser = argp->parser;
-      group->argp = argp;
-      group->args_processed = 0;
-      group->parent = parent;
-      group->parent_index = parent_index;
-      group->input = 0;
-      group->hook = 0;
-      group->child_inputs = 0;
-      
-      if (children)
-	/* Assign GROUP's CHILD_INPUTS field some space from
-	   CVT->child_inputs_end.*/
-	{
-	  unsigned num_children = 0;
-	  while (children[num_children].argp)
-	    num_children++;
-	  group->child_inputs = cvt->child_inputs_end;
-	  cvt->child_inputs_end += num_children;
-	}
-      parent = group++;
-    }
-  else
-    parent = 0;
-
-  if (children)
-    {
-      unsigned index = 0;
-      while (children->argp)
-	group =
-	  convert_options (children++->argp, parent, index++, group, cvt);
-    }
-
-  return group;
-}
-/* Allocate and initialize the group structures, so that they are
-   ordered as if by traversing the corresponding argp parser tree in
-   pre-order. Also build the list of short options, if that is needed. */
-static void
-parser_convert (struct parser *parser, const struct argp *argp)
-{
-  struct parser_convert_state cvt;
-
-  cvt.parser = parser;
-  cvt.short_end = parser->short_opts;
-  cvt.child_inputs_end = parser->child_inputs;
-
-  parser->argp = argp;
-
-  if (argp)
-    parser->egroup = convert_options (argp, 0, 0, parser->groups, &cvt);
-  else
-    parser->egroup = parser->groups; /* No parsers at all! */
-
-  if (parser->short_opts)
-    *cvt.short_end ='\0';
-}
-
-/* Lengths of various parser fields which we will allocated.  */
-struct parser_sizes
-{
-  /* Needed only ARGP_LONG_ONLY */
-  size_t short_len;		/* Number of short options.  */
-
-  size_t num_groups;		/* Group structures we allocate.  */
-  size_t num_child_inputs;	/* Child input slots.  */
-};
-
-/* For ARGP, increments the NUM_GROUPS field in SZS by the total
-   number of argp structures descended from it, and the SHORT_LEN by
-   the total number of short options. */
-static void
-calc_sizes (const struct argp *argp,  struct parser_sizes *szs)
-{
-  const struct argp_child *child = argp->children;
-  const struct argp_option *opt = argp->options;
-
-  if (opt || argp->parser)
-    {
-      /* This parser needs a group. */
-      szs->num_groups++;
-      if (opt)
-	{
-	  while (__option_is_short (opt++))
-	    szs->short_len++;
-	}
-    }
-
-  if (child)
-    while (child->argp)
-      {
-	calc_sizes ((child++)->argp, szs);
-	szs->num_child_inputs++;
-      }
-}
-
-/* Initializes PARSER to parse ARGP in a manner described by FLAGS.  */
-static error_t
-parser_init (struct parser *parser, const struct argp *argp,
-	     int argc, char **argv, int flags, void *input)
-{
-  error_t err = 0;
-  struct group *group;
-  struct parser_sizes szs;
-
-  parser->posixly_correct = getenv ("POSIXLY_CORRECT");
-
-  if (flags & ARGP_IN_ORDER)
-    parser->ordering = RETURN_IN_ORDER;
-  else if (flags & ARGP_NO_ARGS)
-    parser->ordering = REQUIRE_ORDER;
-  else if (parser->posixly_correct)
-    parser->ordering = REQUIRE_ORDER;
-  else
-    parser->ordering = PERMUTE;
-  
-  szs.short_len = 0;
-  szs.num_groups = 0;
-  szs.num_child_inputs = 0;
-
-  if (argp)
-    calc_sizes (argp, &szs);
-
-  if (!(flags & ARGP_LONG_ONLY))
-    /* We have no use for the short option array. */
-    szs.short_len = 0;
-  
-  /* Lengths of the various bits of storage used by PARSER.  */
-#define GLEN (szs.num_groups + 1) * sizeof (struct group)
-#define CLEN (szs.num_child_inputs * sizeof (void *))
-#define SLEN (szs.short_len + 1)
-#define STORAGE(offset) ((void *) (((char *) parser->storage) + (offset)))
-
-  parser->storage = malloc (GLEN + CLEN + SLEN);
-  if (! parser->storage)
-    return ENOMEM;
-
-  parser->groups = parser->storage;
-
-  parser->child_inputs = STORAGE(GLEN);
-  memset (parser->child_inputs, 0, szs.num_child_inputs * sizeof (void *));
-
-  if (flags & ARGP_LONG_ONLY)
-    parser->short_opts = STORAGE(GLEN + CLEN);
-  else
-    parser->short_opts = NULL;
-
-  parser_convert (parser, argp);
-
-  memset (&parser->state, 0, sizeof (struct argp_state));
-  
-  parser->state.root_argp = parser->argp;
-  parser->state.argc = argc;
-  parser->state.argv = argv;
-  parser->state.flags = flags;
-  parser->state.err_stream = stderr;
-  parser->state.out_stream = stdout;
-  parser->state.pstate = parser;
-
-  parser->args_only = 0;
-  parser->nextchar = NULL;
-  parser->first_nonopt = parser->last_nonopt = 0;
-    
-  /* Call each parser for the first time, giving it a chance to propagate
-     values to child parsers.  */
-  if (parser->groups < parser->egroup)
-    parser->groups->input = input;
-  for (group = parser->groups;
-       group < parser->egroup && (!err || err == EBADKEY);
-       group++)
-    {
-      if (group->parent)
-	/* If a child parser, get the initial input value from the parent. */
-	group->input = group->parent->child_inputs[group->parent_index];
-
-      if (!group->parser
-	  && group->argp->children && group->argp->children->argp)
-	/* For the special case where no parsing function is supplied for an
-	   argp, propagate its input to its first child, if any (this just
-	   makes very simple wrapper argps more convenient).  */
-	group->child_inputs[0] = group->input;
-
-      err = group_parse (group, &parser->state, ARGP_KEY_INIT, 0);
-    }
-  if (err == EBADKEY)
-    err = 0;			/* Some parser didn't understand.  */
-
-  if (err)
-    return err;
-
-  if (argv[0] && !(parser->state.flags & ARGP_PARSE_ARGV0))
-    /* There's an argv[0]; use it for messages.  */
-    {
-      parser->state.name = __argp_basename(argv[0]);
-
-      /* Don't parse it as an argument. */
-      parser->state.next = 1;
-    }
-  else
-    parser->state.name = __argp_short_program_name(NULL);
-  
-  return 0;
-}
-
-/* Free any storage consumed by PARSER (but not PARSER itself).  */
-static error_t
-parser_finalize (struct parser *parser,
-		 error_t err, int arg_ebadkey, int *end_index)
-{
-  struct group *group;
-
-  if (err == EBADKEY && arg_ebadkey)
-    /* Suppress errors generated by unparsed arguments.  */
-    err = 0;
-
-  if (! err)
-    {
-      if (parser->state.next == parser->state.argc)
-	/* We successfully parsed all arguments!  Call all the parsers again,
-	   just a few more times... */
-	{
-	  for (group = parser->groups;
-	       group < parser->egroup && (!err || err==EBADKEY);
-	       group++)
-	    if (group->args_processed == 0)
-	      err = group_parse (group, &parser->state, ARGP_KEY_NO_ARGS, 0);
-	  for (group = parser->egroup - 1;
-	       group >= parser->groups && (!err || err==EBADKEY);
-	       group--)
-	    err = group_parse (group, &parser->state, ARGP_KEY_END, 0);
-
-	  if (err == EBADKEY)
-	    err = 0;		/* Some parser didn't understand.  */
-
-	  /* Tell the user that all arguments are parsed.  */
-	  if (end_index)
-	    *end_index = parser->state.next;
-	}
-      else if (end_index)
-	/* Return any remaining arguments to the user.  */
-	*end_index = parser->state.next;
-      else
-	/* No way to return the remaining arguments, they must be bogus. */
-	{
-	  if (!(parser->state.flags & ARGP_NO_ERRS)
-	      && parser->state.err_stream)
-	    fprintf (parser->state.err_stream,
-		     dgettext (parser->argp->argp_domain,
-			       "%s: Too many arguments\n"),
-		     parser->state.name);
-	  err = EBADKEY;
-	}
-    }
-
-  /* Okay, we're all done, with either an error or success; call the parsers
-     to indicate which one.  */
-
-  if (err)
-    {
-      /* Maybe print an error message.  */
-      if (err == EBADKEY)
-	/* An appropriate message describing what the error was should have
-	   been printed earlier.  */
-	__argp_state_help (&parser->state, parser->state.err_stream,
-			   ARGP_HELP_STD_ERR);
-
-      /* Since we didn't exit, give each parser an error indication.  */
-      for (group = parser->groups; group < parser->egroup; group++)
-	group_parse (group, &parser->state, ARGP_KEY_ERROR, 0);
-    }
-  else
-    /* Notify parsers of success, and propagate back values from parsers.  */
-    {
-      /* We pass over the groups in reverse order so that child groups are
-	 given a chance to do there processing before passing back a value to
-	 the parent.  */
-      for (group = parser->egroup - 1
-	   ; group >= parser->groups && (!err || err == EBADKEY)
-	   ; group--)
-	err = group_parse (group, &parser->state, ARGP_KEY_SUCCESS, 0);
-      if (err == EBADKEY)
-	err = 0;		/* Some parser didn't understand.  */
-    }
-
-  /* Call parsers once more, to do any final cleanup.  Errors are ignored.  */
-  for (group = parser->egroup - 1; group >= parser->groups; group--)
-    group_parse (group, &parser->state, ARGP_KEY_FINI, 0);
-
-  if (err == EBADKEY)
-    err = EINVAL;
-
-  free (parser->storage);
-
-  return err;
-}
-
-/* Call the user parsers to parse the non-option argument VAL, at the
-   current position, returning any error. The state NEXT pointer
-   should point to the argument; this function will adjust it
-   correctly to reflect however many args actually end up being
-   consumed. */
-static error_t
-parser_parse_arg (struct parser *parser, char *val)
-{
-  /* Save the starting value of NEXT */
-  int index = parser->state.next;
-  error_t err = EBADKEY;
-  struct group *group;
-  int key = 0;			/* Which of ARGP_KEY_ARG[S] we used.  */
-
-  /* Try to parse the argument in each parser.  */
-  for (group = parser->groups
-       ; group < parser->egroup && err == EBADKEY
-       ; group++)
-    {
-      parser->state.next++;	/* For ARGP_KEY_ARG, consume the arg.  */
-      key = ARGP_KEY_ARG;
-      err = group_parse (group, &parser->state, key, val);
-
-      if (err == EBADKEY)
-	/* This parser doesn't like ARGP_KEY_ARG; try ARGP_KEY_ARGS instead. */
-	{
-	  parser->state.next--;	/* For ARGP_KEY_ARGS, put back the arg.  */
-	  key = ARGP_KEY_ARGS;
-	  err = group_parse (group, &parser->state, key, 0);
-	}
-    }
-
-  if (! err)
-    {
-      if (key == ARGP_KEY_ARGS)
-	/* The default for ARGP_KEY_ARGS is to assume that if NEXT isn't
-	   changed by the user, *all* arguments should be considered
-	   consumed.  */
-	parser->state.next = parser->state.argc;
-
-      if (parser->state.next > index)
-	/* Remember that we successfully processed a non-option
-	   argument -- but only if the user hasn't gotten tricky and set
-	   the clock back.  */
-	(--group)->args_processed += (parser->state.next - index);
-      else
-	/* The user wants to reparse some args, so try looking for options again.  */
-	parser->args_only = 0;
-    }
-
-  return err;
-}
-
-/* Exchange two adjacent subsequences of ARGV.
-   One subsequence is elements [first_nonopt,last_nonopt)
-   which contains all the non-options that have been skipped so far.
-   The other is elements [last_nonopt,next), which contains all
-   the options processed since those non-options were skipped.
-
-   `first_nonopt' and `last_nonopt' are relocated so that they describe
-   the new indices of the non-options in ARGV after they are moved.  */
-
-static void
-exchange (struct parser *parser)
-{
-  int bottom = parser->first_nonopt;
-  int middle = parser->last_nonopt;
-  int top = parser->state.next;
-  char **argv = parser->state.argv;
-  
-  char *tem;
-
-  /* Exchange the shorter segment with the far end of the longer segment.
-     That puts the shorter segment into the right place.
-     It leaves the longer segment in the right place overall,
-     but it consists of two parts that need to be swapped next.  */
-
-  while (top > middle && middle > bottom)
-    {
-      if (top - middle > middle - bottom)
-	{
-	  /* Bottom segment is the short one.  */
-	  int len = middle - bottom;
-	  register int i;
-
-	  /* Swap it with the top part of the top segment.  */
-	  for (i = 0; i < len; i++)
-	    {
-	      tem = argv[bottom + i];
-	      argv[bottom + i] = argv[top - (middle - bottom) + i];
-	      argv[top - (middle - bottom) + i] = tem;
-	    }
-	  /* Exclude the moved bottom segment from further swapping.  */
-	  top -= len;
-	}
-      else
-	{
-	  /* Top segment is the short one.  */
-	  int len = top - middle;
-	  register int i;
-
-	  /* Swap it with the bottom part of the bottom segment.  */
-	  for (i = 0; i < len; i++)
-	    {
-	      tem = argv[bottom + i];
-	      argv[bottom + i] = argv[middle + i];
-	      argv[middle + i] = tem;
-	    }
-	  /* Exclude the moved top segment from further swapping.  */
-	  bottom += len;
-	}
-    }
-
-  /* Update records for the slots the non-options now occupy.  */
-
-  parser->first_nonopt += (parser->state.next - parser->last_nonopt);
-  parser->last_nonopt = parser->state.next;
-}
-
-
-
-enum arg_type { ARG_ARG, ARG_SHORT_OPTION,
-		ARG_LONG_OPTION, ARG_LONG_ONLY_OPTION,
-		ARG_QUOTE };
-
-static enum arg_type
-classify_arg(struct parser *parser, char *arg, char **opt)
-{
-  if (arg[0] == '-')
-    /* Looks like an option... */
-    switch (arg[1])
-      {
-      case '\0':
-	/* "-" is not an option. */
-	return ARG_ARG;
-      case '-':
-	/* Long option, or quote. */
-	if (!arg[2])
-	  return ARG_QUOTE;
-	  
-	/* A long option. */
-	if (opt)
-	  *opt = arg + 2;
-	return ARG_LONG_OPTION;
-
-      default:
-	/* Short option. But if ARGP_LONG_ONLY, it can also be a long option. */
-
-	if (opt)
-	  *opt = arg + 1;
-
-	if (parser->state.flags & ARGP_LONG_ONLY)
-	  {
-	    /* Rules from getopt.c:
-
-	       If long_only and the ARGV-element has the form "-f",
-	       where f is a valid short option, don't consider it an
-	       abbreviated form of a long option that starts with f.
-	       Otherwise there would be no way to give the -f short
-	       option.
-
-	       On the other hand, if there's a long option "fubar" and
-	       the ARGV-element is "-fu", do consider that an
-	       abbreviation of the long option, just like "--fu", and
-	       not "-f" with arg "u".
-
-	       This distinction seems to be the most useful approach. */
-
-	    assert(parser->short_opts);
-	    
-	    if (arg[2] || !strchr(parser->short_opts, arg[1]))
-	      return ARG_LONG_ONLY_OPTION;
-	  }
-
-	return ARG_SHORT_OPTION;
-      }
-  
-  else
-    return ARG_ARG;
-}
-
-/* Parse the next argument in PARSER (as indicated by PARSER->state.next).
-   Any error from the parsers is returned, and *ARGP_EBADKEY indicates
-   whether a value of EBADKEY is due to an unrecognized argument (which is
-   generally not fatal).  */
-static error_t
-parser_parse_next (struct parser *parser, int *arg_ebadkey)
-{
-  if (parser->state.quoted && parser->state.next < parser->state.quoted)
-    /* The next argument pointer has been moved to before the quoted
-       region, so pretend we never saw the quoting `--', and start
-       looking for options again. If the `--' is still there we'll just
-       process it one more time. */
-    parser->state.quoted = parser->args_only = 0;
-
-  /* Give FIRST_NONOPT & LAST_NONOPT rational values if NEXT has been
-     moved back by the user (who may also have changed the arguments).  */
-  if (parser->last_nonopt > parser->state.next)
-    parser->last_nonopt = parser->state.next;
-  if (parser->first_nonopt > parser->state.next)
-    parser->first_nonopt = parser->state.next;
-
-  if (parser->nextchar)
-    /* Deal with short options. */
-    {
-      struct group *group;
-      char c;
-      const struct argp_option *option;
-      char *value = NULL;;
-
-      assert(!parser->args_only);
-
-      c = *parser->nextchar++;
-      
-      option = find_short_option(parser, c, &group);
-      if (!option)
-	{
-	  if (parser->posixly_correct)
-	    /* 1003.2 specifies the format of this message.  */
-	    fprintf (parser->state.err_stream,
-		     dgettext(parser->state.root_argp->argp_domain,
-			      "%s: illegal option -- %c\n"),
-		     parser->state.name, c);
-	  else
-	    fprintf (parser->state.err_stream,
-		     dgettext(parser->state.root_argp->argp_domain,
-			      "%s: invalid option -- %c\n"),
-		     parser->state.name, c);
-
-	  *arg_ebadkey = 0;
-	  return EBADKEY;
-	}
-
-      if (!*parser->nextchar)
-	parser->nextchar = NULL;
-
-      if (option->arg)
-	{
-	  value = parser->nextchar;
-	  parser->nextchar = NULL;
-	      
-	  if (!value
-	      && !(option->flags & OPTION_ARG_OPTIONAL))
-	    /* We need an mandatory argument. */
-	    {
-	      if (parser->state.next == parser->state.argc)
-		/* Missing argument */
-		{
-		  /* 1003.2 specifies the format of this message.  */
-		  fprintf (parser->state.err_stream,
-			   dgettext(parser->state.root_argp->argp_domain,
-				    "%s: option requires an argument -- %c\n"),
-			   parser->state.name, c);
-
-		  *arg_ebadkey = 0;
-		  return EBADKEY;
-		}
-	      value = parser->state.argv[parser->state.next++];
-	    }
-	}
-      return group_parse(group, &parser->state,
-			 option->key, value);
-    }
-  else
-    /* Advance to the next ARGV-element.  */
-    {
-      if (parser->args_only)
-	{
-	  *arg_ebadkey = 1;
-	  if (parser->state.next >= parser->state.argc)
-	    /* We're done. */
-	    return EBADKEY;
-	  else
-	    return parser_parse_arg(parser,
-				    parser->state.argv[parser->state.next]);
-	}
-      
-      if (parser->state.next >= parser->state.argc)
-	/* Almost done. If there are non-options that we skipped
-	   previously, we should process them now. */
-	{
-	  *arg_ebadkey = 1;
-	  if (parser->first_nonopt != parser->last_nonopt)
-	    {
-	      exchange(parser);
-	      
-	      /* Start processing the arguments we skipped previously. */
-	      parser->state.next = parser->first_nonopt;
-	      
-	      parser->first_nonopt = parser->last_nonopt = 0;
-
-	      parser->args_only = 1;
-	      return 0;
-	    }
-	  else
-	    /* Indicate that we're really done. */
-	    return EBADKEY;
-	}
-      else
-	/* Look for options. */
-	{
-	  char *arg = parser->state.argv[parser->state.next];
-
-	  char *optstart;
-	  enum arg_type token = classify_arg(parser, arg, &optstart);
-	  
-	  switch (token)
-	    {
-	    case ARG_ARG:
-	      switch (parser->ordering)
-		{
-		case PERMUTE:
-		  if (parser->first_nonopt == parser->last_nonopt)
-		    /* Skipped sequence is empty; start a new one. */
-		    parser->first_nonopt = parser->last_nonopt = parser->state.next;
-
-		  else if (parser->last_nonopt != parser->state.next)
-		    /* We have a non-empty skipped sequence, and
-		       we're not at the end-point, so move it. */
-		    exchange(parser);
-
-		  assert(parser->last_nonopt == parser->state.next);
-		  
-		  /* Skip this argument for now. */
-		  parser->state.next++;
-		  parser->last_nonopt = parser->state.next; 
-		  
-		  return 0;
-
-		case REQUIRE_ORDER:
-		  /* Implicit quote before the first argument. */
-		   parser->args_only = 1;
-		   return 0;
-		   
-		case RETURN_IN_ORDER:
-		  *arg_ebadkey = 1;
-		  return parser_parse_arg(parser, arg);
-
-		default:
-		  abort();
-		}
-	    case ARG_QUOTE:
-	      /* Skip it, then exchange with any previous non-options. */
-	      parser->state.next++;
-	      assert (parser->last_nonopt != parser->state.next);
-
-	      if (parser->first_nonopt != parser->last_nonopt)
-		{
-		  exchange(parser);
-		  
-		  /* Start processing the skipped and the quoted
-		     arguments. */
-
-		  parser->state.quoted = parser->state.next = parser->first_nonopt;
-
-		  /* Also empty the skipped-list, to avoid confusion
-		     if the user resets the next pointer. */
-		  parser->first_nonopt = parser->last_nonopt = 0;
-		}
-	      else
-		parser->state.quoted = parser->state.next;
-
-	      parser->args_only = 1;	      
-	      return 0;
-
-	    case ARG_LONG_ONLY_OPTION:
-	    case ARG_LONG_OPTION:
-	      {
-		struct group *group;
-		const struct argp_option *option;
-		char *value;
-
-		parser->state.next++;
-		option = find_long_option(parser, optstart, &group);
-		
-		if (!option)
-		  {
-		    /* NOTE: This includes any "=something" in the output. */
-		    fprintf (parser->state.err_stream,
-			     dgettext(parser->state.root_argp->argp_domain,
-				      "%s: unrecognized option `%s'\n"),
-			     parser->state.name, arg);
-		    *arg_ebadkey = 0;
-		    return EBADKEY;
-		  }
-
-		value = strchr(optstart, '=');
-		if (value)
-		  value++;
-		
-		if (value && !option->arg)
-		  /* Unexpected argument. */
-		  {
-		    if (token == ARG_LONG_OPTION)
-		      /* --option */
-		      fprintf (parser->state.err_stream,
-			       dgettext(parser->state.root_argp->argp_domain,
-					"%s: option `--%s' doesn't allow an argument\n"),
-			       parser->state.name, option->name);
-		    else
-		      /* +option or -option */
-		      fprintf (parser->state.err_stream,
-			       dgettext(parser->state.root_argp->argp_domain,
-					"%s: option `%c%s' doesn't allow an argument\n"),
-			       parser->state.name, arg[0], option->name);
-
-		    *arg_ebadkey = 0;
-		    return EBADKEY;
-		  }
-		
-		if (option->arg && !value
-		    && !(option->flags & OPTION_ARG_OPTIONAL))
-		  /* We need an mandatory argument. */
-		  {
-		    if (parser->state.next == parser->state.argc)
-		      /* Missing argument */
-		      {
-			if (token == ARG_LONG_OPTION)
-			  /* --option */
-			  fprintf (parser->state.err_stream,
-				   dgettext(parser->state.root_argp->argp_domain,
-					    "%s: option `--%s' requires an argument\n"),
-				 parser->state.name, option->name);
-			else
-			  /* +option or -option */
-			  fprintf (parser->state.err_stream,
-				   dgettext(parser->state.root_argp->argp_domain,
-					    "%s: option `%c%s' requires an argument\n"),
-				   parser->state.name, arg[0], option->name);
-
-			*arg_ebadkey = 0;
-			return EBADKEY;
-		      }
-
-		    value = parser->state.argv[parser->state.next++];
-		  }
-		*arg_ebadkey = 0;
-		return group_parse(group, &parser->state,
-				   option->key, value);
-	      }
-	    case ARG_SHORT_OPTION:
-	      parser->state.next++;
-	      parser->nextchar = optstart;
-	      return 0;
-
-	    default:
-	      abort();
-	    }
-	}
-    }
-}
-
-/* Parse the options strings in ARGC & ARGV according to the argp in ARGP.
-   FLAGS is one of the ARGP_ flags above.  If END_INDEX is non-NULL, the
-   index in ARGV of the first unparsed option is returned in it.  If an
-   unknown option is present, EINVAL is returned; if some parser routine
-   returned a non-zero value, it is returned; otherwise 0 is returned.  */
-error_t
-__argp_parse (const struct argp *argp, int argc, char **argv, unsigned flags,
-	      int *end_index, void *input)
-{
-  error_t err;
-  struct parser parser;
-
-  /* If true, then err == EBADKEY is a result of a non-option argument failing
-     to be parsed (which in some cases isn't actually an error).  */
-  int arg_ebadkey = 0;
-
-  if (! (flags & ARGP_NO_HELP))
-    /* Add our own options.  */
-    {
-      struct argp_child *child = alloca (4 * sizeof (struct argp_child));
-      struct argp *top_argp = alloca (sizeof (struct argp));
-
-      /* TOP_ARGP has no options, it just serves to group the user & default
-	 argps.  */
-      memset (top_argp, 0, sizeof (*top_argp));
-      top_argp->children = child;
-
-      memset (child, 0, 4 * sizeof (struct argp_child));
-
-      if (argp)
-	(child++)->argp = argp;
-      (child++)->argp = &argp_default_argp;
-      if (argp_program_version || argp_program_version_hook)
-	(child++)->argp = &argp_version_argp;
-      child->argp = 0;
-
-      argp = top_argp;
-    }
-
-  /* Construct a parser for these arguments.  */
-  err = parser_init (&parser, argp, argc, argv, flags, input);
-
-  if (! err)
-    /* Parse! */
-    {
-      while (! err)
-	err = parser_parse_next (&parser, &arg_ebadkey);
-      err = parser_finalize (&parser, err, arg_ebadkey, end_index);
-    }
-
-  return err;
-}
-#ifdef weak_alias
-weak_alias (__argp_parse, argp_parse)
-#endif
-
-/* Return the input field for ARGP in the parser corresponding to STATE; used
-   by the help routines.  */
-void *
-__argp_input (const struct argp *argp, const struct argp_state *state)
-{
-  if (state)
-    {
-      struct group *group;
-      struct parser *parser = state->pstate;
-
-      for (group = parser->groups; group < parser->egroup; group++)
-	if (group->argp == argp)
-	  return group->input;
-    }
-
-  return 0;
-}
-#ifdef weak_alias
-weak_alias (__argp_input, _argp_input)
-#endif
-
-/* Defined here, in case a user is not inlining the definitions in
- * argp.h */
-void
-__argp_usage (__const struct argp_state *__state)
-{
-  __argp_state_help (__state, stderr, ARGP_HELP_STD_USAGE);
-}
-
-int
-__option_is_short (__const struct argp_option *__opt)
-{
-  if (__opt->flags & OPTION_DOC)
-    return 0;
-  else
-    {
-      int __key = __opt->key;
-      /* FIXME: whether or not a particular key implies a short option
-       * ought not to be locale dependent. */
-      return __key > 0 && isprint (__key);
-    }
-}
-
-int
-__option_is_end (__const struct argp_option *__opt)
-{
-  return !__opt->key && !__opt->name && !__opt->doc && !__opt->group;
-}
diff --git a/contrib/argp-standalone/argp-pv.c b/contrib/argp-standalone/argp-pv.c
deleted file mode 100644
index d7d374a66bd..00000000000
--- a/contrib/argp-standalone/argp-pv.c
+++ /dev/null
@@ -1,25 +0,0 @@
-/* Default definition for ARGP_PROGRAM_VERSION.
-   Copyright (C) 1996, 1997, 1999, 2004 Free Software Foundation, Inc.
-   This file is part of the GNU C Library.
-   Written by Miles Bader <miles@gnu.ai.mit.edu>.
-
-   The GNU C Library is free software; you can redistribute it and/or
-   modify it under the terms of the GNU Library General Public License as
-   published by the Free Software Foundation; either version 2 of the
-   License, or (at your option) any later version.
-
-   The GNU C Library is distributed in the hope that it will be useful,
-   but WITHOUT ANY WARRANTY; without even the implied warranty of
-   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-   Library General Public License for more details.
-
-   You should have received a copy of the GNU Library General Public
-   License along with the GNU C Library; see the file COPYING.LIB.  If not,
-   write to the Free Software Foundation, Inc., 59 Temple Place - Suite 330,
-   Boston, MA 02111-1307, USA.  */
-
-/* If set by the user program to a non-zero value, then a default option
-   --version is added (unless the ARGP_NO_HELP flag is used), which will
-   print this this string followed by a newline and exit (unless the
-   ARGP_NO_EXIT flag is used).  Overridden by ARGP_PROGRAM_VERSION_HOOK.  */
-const char *argp_program_version = 0;
diff --git a/contrib/argp-standalone/argp-pvh.c b/contrib/argp-standalone/argp-pvh.c
deleted file mode 100644
index 829a1cda80d..00000000000
--- a/contrib/argp-standalone/argp-pvh.c
+++ /dev/null
@@ -1,32 +0,0 @@
-/* Default definition for ARGP_PROGRAM_VERSION_HOOK.
-   Copyright (C) 1996, 1997, 1999, 2004 Free Software Foundation, Inc.
-   This file is part of the GNU C Library.
-   Written by Miles Bader <miles@gnu.ai.mit.edu>.
-
-   The GNU C Library is free software; you can redistribute it and/or
-   modify it under the terms of the GNU Library General Public License as
-   published by the Free Software Foundation; either version 2 of the
-   License, or (at your option) any later version.
-
-   The GNU C Library is distributed in the hope that it will be useful,
-   but WITHOUT ANY WARRANTY; without even the implied warranty of
-   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-   Library General Public License for more details.
-
-   You should have received a copy of the GNU Library General Public
-   License along with the GNU C Library; see the file COPYING.LIB.  If not,
-   write to the Free Software Foundation, Inc., 59 Temple Place - Suite 330,
-   Boston, MA 02111-1307, USA.  */
-
-#ifdef HAVE_CONFIG_H
-#include <config.h>
-#endif
-
-#include "argp.h"
-
-/* If set by the user program to a non-zero value, then a default option
-   --version is added (unless the ARGP_NO_HELP flag is used), which calls
-   this function with a stream to print the version to and a pointer to the
-   current parsing state, and then exits (unless the ARGP_NO_EXIT flag is
-   used).  This variable takes precedent over ARGP_PROGRAM_VERSION.  */
-void (*argp_program_version_hook) (FILE *stream, struct argp_state *state) = 0;
diff --git a/contrib/argp-standalone/argp.h b/contrib/argp-standalone/argp.h
deleted file mode 100644
index 29d3dfe9720..00000000000
--- a/contrib/argp-standalone/argp.h
+++ /dev/null
@@ -1,602 +0,0 @@
-/* Hierarchial argument parsing.
-   Copyright (C) 1995, 96, 97, 98, 99, 2003 Free Software Foundation, Inc.
-   This file is part of the GNU C Library.
-   Written by Miles Bader <miles@gnu.ai.mit.edu>.
-
-   The GNU C Library is free software; you can redistribute it and/or
-   modify it under the terms of the GNU Library General Public License as
-   published by the Free Software Foundation; either version 2 of the
-   License, or (at your option) any later version.
-
-   The GNU C Library is distributed in the hope that it will be useful,
-   but WITHOUT ANY WARRANTY; without even the implied warranty of
-   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-   Library General Public License for more details.
-
-   You should have received a copy of the GNU Library General Public
-   License along with the GNU C Library; see the file COPYING.LIB.  If not,
-   write to the Free Software Foundation, Inc., 59 Temple Place - Suite 330,
-   Boston, MA 02111-1307, USA.  */
-
-#ifndef _ARGP_H
-#define _ARGP_H
-
-#include <stdio.h>
-#include <ctype.h>
-
-#define __need_error_t
-#include <errno.h>
-
-#ifndef __THROW
-# define __THROW
-#endif
-
-#ifndef __const
-# define __const const
-#endif
-
-#ifndef __error_t_defined
-typedef int error_t;
-# define __error_t_defined
-#endif
-
-/* FIXME: What's the right way to check for __restrict? Sun's cc seems
-   not to have it. Perhaps it's easiest to just delete the use of
-   __restrict from the prototypes. */
-#ifndef __restrict
-# ifndef __GNUC___
-#  define __restrict
-# endif
-#endif
-
-/* NOTE: We can't use the autoconf tests, since this is supposed to be
-   an installed header file and argp's config.h is of course not
-   installed. */
-#ifndef PRINTF_STYLE
-# if __GNUC__ >= 2
-#  define PRINTF_STYLE(f, a) __attribute__ ((__format__ (__printf__, f, a)))
-# else
-#  define PRINTF_STYLE(f, a)
-# endif
-#endif
-
-
-#ifdef  __cplusplus
-extern "C" {
-#endif
-
-/* A description of a particular option.  A pointer to an array of
-   these is passed in the OPTIONS field of an argp structure.  Each option
-   entry can correspond to one long option and/or one short option; more
-   names for the same option can be added by following an entry in an option
-   array with options having the OPTION_ALIAS flag set.  */
-struct argp_option
-{
-  /* The long option name.  For more than one name for the same option, you
-     can use following options with the OPTION_ALIAS flag set.  */
-  __const char *name;
-
-  /* What key is returned for this option.  If > 0 and printable, then it's
-     also accepted as a short option.  */
-  int key;
-
-  /* If non-NULL, this is the name of the argument associated with this
-     option, which is required unless the OPTION_ARG_OPTIONAL flag is set. */
-  __const char *arg;
-
-  /* OPTION_ flags.  */
-  int flags;
-
-  /* The doc string for this option.  If both NAME and KEY are 0, This string
-     will be printed outdented from the normal option column, making it
-     useful as a group header (it will be the first thing printed in its
-     group); in this usage, it's conventional to end the string with a `:'.  */
-  __const char *doc;
-
-  /* The group this option is in.  In a long help message, options are sorted
-     alphabetically within each group, and the groups presented in the order
-     0, 1, 2, ..., n, -m, ..., -2, -1.  Every entry in an options array with
-     if this field 0 will inherit the group number of the previous entry, or
-     zero if it's the first one, unless its a group header (NAME and KEY both
-     0), in which case, the previous entry + 1 is the default.  Automagic
-     options such as --help are put into group -1.  */
-  int group;
-};
-
-/* The argument associated with this option is optional.  */
-#define OPTION_ARG_OPTIONAL	0x1
-
-/* This option isn't displayed in any help messages.  */
-#define OPTION_HIDDEN	       	0x2
-
-/* This option is an alias for the closest previous non-alias option.  This
-   means that it will be displayed in the same help entry, and will inherit
-   fields other than NAME and KEY from the aliased option.  */
-#define OPTION_ALIAS		0x4
-
-/* This option isn't actually an option (and so should be ignored by the
-   actual option parser), but rather an arbitrary piece of documentation that
-   should be displayed in much the same manner as the options.  If this flag
-   is set, then the option NAME field is displayed unmodified (e.g., no `--'
-   prefix is added) at the left-margin (where a *short* option would normally
-   be displayed), and the documentation string in the normal place.  For
-   purposes of sorting, any leading whitespace and puncuation is ignored,
-   except that if the first non-whitespace character is not `-', this entry
-   is displayed after all options (and OPTION_DOC entries with a leading `-')
-   in the same group.  */
-#define OPTION_DOC		0x8
-
-/* This option shouldn't be included in `long' usage messages (but is still
-   included in help messages).  This is mainly intended for options that are
-   completely documented in an argp's ARGS_DOC field, in which case including
-   the option in the generic usage list would be redundant.  For instance,
-   if ARGS_DOC is "FOO BAR\n-x BLAH", and the `-x' option's purpose is to
-   distinguish these two cases, -x should probably be marked
-   OPTION_NO_USAGE.  */
-#define OPTION_NO_USAGE		0x10
-
-struct argp;			/* fwd declare this type */
-struct argp_state;		/* " */
-struct argp_child;		/* " */
-
-/* The type of a pointer to an argp parsing function.  */
-typedef error_t (*argp_parser_t) (int key, char *arg,
-				  struct argp_state *state);
-
-/* What to return for unrecognized keys.  For special ARGP_KEY_ keys, such
-   returns will simply be ignored.  For user keys, this error will be turned
-   into EINVAL (if the call to argp_parse is such that errors are propagated
-   back to the user instead of exiting); returning EINVAL itself would result
-   in an immediate stop to parsing in *all* cases.  */
-#define ARGP_ERR_UNKNOWN	E2BIG /* Hurd should never need E2BIG.  XXX */
-
-/* Special values for the KEY argument to an argument parsing function.
-   ARGP_ERR_UNKNOWN should be returned if they aren't understood.
-
-   The sequence of keys to a parsing function is either (where each
-   uppercased word should be prefixed by `ARGP_KEY_' and opt is a user key):
-
-       INIT opt... NO_ARGS END SUCCESS  -- No non-option arguments at all
-   or  INIT (opt | ARG)... END SUCCESS  -- All non-option args parsed
-   or  INIT (opt | ARG)... SUCCESS      -- Some non-option arg unrecognized
-
-   The third case is where every parser returned ARGP_KEY_UNKNOWN for an
-   argument, in which case parsing stops at that argument (returning the
-   unparsed arguments to the caller of argp_parse if requested, or stopping
-   with an error message if not).
-
-   If an error occurs (either detected by argp, or because the parsing
-   function returned an error value), then the parser is called with
-   ARGP_KEY_ERROR, and no further calls are made.  */
-
-/* This is not an option at all, but rather a command line argument.  If a
-   parser receiving this key returns success, the fact is recorded, and the
-   ARGP_KEY_NO_ARGS case won't be used.  HOWEVER, if while processing the
-   argument, a parser function decrements the NEXT field of the state it's
-   passed, the option won't be considered processed; this is to allow you to
-   actually modify the argument (perhaps into an option), and have it
-   processed again.  */
-#define ARGP_KEY_ARG		0
-/* There are remaining arguments not parsed by any parser, which may be found
-   starting at (STATE->argv + STATE->next).  If success is returned, but
-   STATE->next left untouched, it's assumed that all arguments were consume,
-   otherwise, the parser should adjust STATE->next to reflect any arguments
-   consumed.  */
-#define ARGP_KEY_ARGS		0x1000006
-/* There are no more command line arguments at all.  */
-#define ARGP_KEY_END		0x1000001
-/* Because it's common to want to do some special processing if there aren't
-   any non-option args, user parsers are called with this key if they didn't
-   successfully process any non-option arguments.  Called just before
-   ARGP_KEY_END (where more general validity checks on previously parsed
-   arguments can take place).  */
-#define ARGP_KEY_NO_ARGS	0x1000002
-/* Passed in before any parsing is done.  Afterwards, the values of each
-   element of the CHILD_INPUT field, if any, in the state structure is
-   copied to each child's state to be the initial value of the INPUT field.  */
-#define ARGP_KEY_INIT		0x1000003
-/* Use after all other keys, including SUCCESS & END.  */
-#define ARGP_KEY_FINI		0x1000007
-/* Passed in when parsing has successfully been completed (even if there are
-   still arguments remaining).  */
-#define ARGP_KEY_SUCCESS	0x1000004
-/* Passed in if an error occurs.  */
-#define ARGP_KEY_ERROR		0x1000005
-
-/* An argp structure contains a set of options declarations, a function to
-   deal with parsing one, documentation string, a possible vector of child
-   argp's, and perhaps a function to filter help output.  When actually
-   parsing options, getopt is called with the union of all the argp
-   structures chained together through their CHILD pointers, with conflicts
-   being resolved in favor of the first occurrence in the chain.  */
-struct argp
-{
-  /* An array of argp_option structures, terminated by an entry with both
-     NAME and KEY having a value of 0.  */
-  __const struct argp_option *options;
-
-  /* What to do with an option from this structure.  KEY is the key
-     associated with the option, and ARG is any associated argument (NULL if
-     none was supplied).  If KEY isn't understood, ARGP_ERR_UNKNOWN should be
-     returned.  If a non-zero, non-ARGP_ERR_UNKNOWN value is returned, then
-     parsing is stopped immediately, and that value is returned from
-     argp_parse().  For special (non-user-supplied) values of KEY, see the
-     ARGP_KEY_ definitions below.  */
-  argp_parser_t parser;
-
-  /* A string describing what other arguments are wanted by this program.  It
-     is only used by argp_usage to print the `Usage:' message.  If it
-     contains newlines, the strings separated by them are considered
-     alternative usage patterns, and printed on separate lines (lines after
-     the first are prefix by `  or: ' instead of `Usage:').  */
-  __const char *args_doc;
-
-  /* If non-NULL, a string containing extra text to be printed before and
-     after the options in a long help message (separated by a vertical tab
-     `\v' character).  */
-  __const char *doc;
-
-  /* A vector of argp_children structures, terminated by a member with a 0
-     argp field, pointing to child argps should be parsed with this one.  Any
-     conflicts are resolved in favor of this argp, or early argps in the
-     CHILDREN list.  This field is useful if you use libraries that supply
-     their own argp structure, which you want to use in conjunction with your
-     own.  */
-  __const struct argp_child *children;
-
-  /* If non-zero, this should be a function to filter the output of help
-     messages.  KEY is either a key from an option, in which case TEXT is
-     that option's help text, or a special key from the ARGP_KEY_HELP_
-     defines, below, describing which other help text TEXT is.  The function
-     should return either TEXT, if it should be used as-is, a replacement
-     string, which should be malloced, and will be freed by argp, or NULL,
-     meaning `print nothing'.  The value for TEXT is *after* any translation
-     has been done, so if any of the replacement text also needs translation,
-     that should be done by the filter function.  INPUT is either the input
-     supplied to argp_parse, or NULL, if argp_help was called directly.  */
-  char *(*help_filter) (int __key, __const char *__text, void *__input);
-
-  /* If non-zero the strings used in the argp library are translated using
-     the domain described by this string.  Otherwise the currently installed
-     default domain is used.  */
-  const char *argp_domain;
-};
-
-/* Possible KEY arguments to a help filter function.  */
-#define ARGP_KEY_HELP_PRE_DOC	0x2000001 /* Help text preceeding options. */
-#define ARGP_KEY_HELP_POST_DOC	0x2000002 /* Help text following options. */
-#define ARGP_KEY_HELP_HEADER	0x2000003 /* Option header string. */
-#define ARGP_KEY_HELP_EXTRA	0x2000004 /* After all other documentation;
-					     TEXT is NULL for this key.  */
-/* Explanatory note emitted when duplicate option arguments have been
-   suppressed.  */
-#define ARGP_KEY_HELP_DUP_ARGS_NOTE 0x2000005
-#define ARGP_KEY_HELP_ARGS_DOC	0x2000006 /* Argument doc string.  */
-
-/* When an argp has a non-zero CHILDREN field, it should point to a vector of
-   argp_child structures, each of which describes a subsidiary argp.  */
-struct argp_child
-{
-  /* The child parser.  */
-  __const struct argp *argp;
-
-  /* Flags for this child.  */
-  int flags;
-
-  /* If non-zero, an optional header to be printed in help output before the
-     child options.  As a side-effect, a non-zero value forces the child
-     options to be grouped together; to achieve this effect without actually
-     printing a header string, use a value of "".  */
-  __const char *header;
-
-  /* Where to group the child options relative to the other (`consolidated')
-     options in the parent argp; the values are the same as the GROUP field
-     in argp_option structs, but all child-groupings follow parent options at
-     a particular group level.  If both this field and HEADER are zero, then
-     they aren't grouped at all, but rather merged with the parent options
-     (merging the child's grouping levels with the parents).  */
-  int group;
-};
-
-/* Parsing state.  This is provided to parsing functions called by argp,
-   which may examine and, as noted, modify fields.  */
-struct argp_state
-{
-  /* The top level ARGP being parsed.  */
-  __const struct argp *root_argp;
-
-  /* The argument vector being parsed.  May be modified.  */
-  int argc;
-  char **argv;
-
-  /* The index in ARGV of the next arg that to be parsed.  May be modified. */
-  int next;
-
-  /* The flags supplied to argp_parse.  May be modified.  */
-  unsigned flags;
-
-  /* While calling a parsing function with a key of ARGP_KEY_ARG, this is the
-     number of the current arg, starting at zero, and incremented after each
-     such call returns.  At all other times, this is the number of such
-     arguments that have been processed.  */
-  unsigned arg_num;
-
-  /* If non-zero, the index in ARGV of the first argument following a special
-     `--' argument (which prevents anything following being interpreted as an
-     option).  Only set once argument parsing has proceeded past this point. */
-  int quoted;
-
-  /* An arbitrary pointer passed in from the user.  */
-  void *input;
-  /* Values to pass to child parsers.  This vector will be the same length as
-     the number of children for the current parser.  */
-  void **child_inputs;
-
-  /* For the parser's use.  Initialized to 0.  */
-  void *hook;
-
-  /* The name used when printing messages.  This is initialized to ARGV[0],
-     or PROGRAM_INVOCATION_NAME if that is unavailable.  */
-  char *name;
-
-  /* Streams used when argp prints something.  */
-  FILE *err_stream;		/* For errors; initialized to stderr. */
-  FILE *out_stream;		/* For information; initialized to stdout. */
-
-  void *pstate;			/* Private, for use by argp.  */
-};
-
-/* Flags for argp_parse (note that the defaults are those that are
-   convenient for program command line parsing): */
-
-/* Don't ignore the first element of ARGV.  Normally (and always unless
-   ARGP_NO_ERRS is set) the first element of the argument vector is
-   skipped for option parsing purposes, as it corresponds to the program name
-   in a command line.  */
-#define ARGP_PARSE_ARGV0  0x01
-
-/* Don't print error messages for unknown options to stderr; unless this flag
-   is set, ARGP_PARSE_ARGV0 is ignored, as ARGV[0] is used as the program
-   name in the error messages.  This flag implies ARGP_NO_EXIT (on the
-   assumption that silent exiting upon errors is bad behaviour).  */
-#define ARGP_NO_ERRS	0x02
-
-/* Don't parse any non-option args.  Normally non-option args are parsed by
-   calling the parse functions with a key of ARGP_KEY_ARG, and the actual arg
-   as the value.  Since it's impossible to know which parse function wants to
-   handle it, each one is called in turn, until one returns 0 or an error
-   other than ARGP_ERR_UNKNOWN; if an argument is handled by no one, the
-   argp_parse returns prematurely (but with a return value of 0).  If all
-   args have been parsed without error, all parsing functions are called one
-   last time with a key of ARGP_KEY_END.  This flag needn't normally be set,
-   as the normal behavior is to stop parsing as soon as some argument can't
-   be handled.  */
-#define ARGP_NO_ARGS	0x04
-
-/* Parse options and arguments in the same order they occur on the command
-   line -- normally they're rearranged so that all options come first. */
-#define ARGP_IN_ORDER	0x08
-
-/* Don't provide the standard long option --help, which causes usage and
-      option help information to be output to stdout, and exit (0) called. */
-#define ARGP_NO_HELP	0x10
-
-/* Don't exit on errors (they may still result in error messages).  */
-#define ARGP_NO_EXIT	0x20
-
-/* Use the gnu getopt `long-only' rules for parsing arguments.  */
-#define ARGP_LONG_ONLY	0x40
-
-/* Turns off any message-printing/exiting options.  */
-#define ARGP_SILENT    (ARGP_NO_EXIT | ARGP_NO_ERRS | ARGP_NO_HELP)
-
-/* Parse the options strings in ARGC & ARGV according to the options in ARGP.
-   FLAGS is one of the ARGP_ flags above.  If ARG_INDEX is non-NULL, the
-   index in ARGV of the first unparsed option is returned in it.  If an
-   unknown option is present, ARGP_ERR_UNKNOWN is returned; if some parser
-   routine returned a non-zero value, it is returned; otherwise 0 is
-   returned.  This function may also call exit unless the ARGP_NO_HELP flag
-   is set.  INPUT is a pointer to a value to be passed in to the parser.  */
-extern error_t argp_parse (__const struct argp *__restrict argp,
-			   int argc, char **__restrict argv,
-			   unsigned flags, int *__restrict arg_index,
-			   void *__restrict input) __THROW;
-extern error_t __argp_parse (__const struct argp *__restrict argp,
-			     int argc, char **__restrict argv,
-			     unsigned flags, int *__restrict arg_index,
-			     void *__restrict input) __THROW;
-
-/* Global variables.  */
-
-/* If defined or set by the user program to a non-zero value, then a default
-   option --version is added (unless the ARGP_NO_HELP flag is used), which
-   will print this string followed by a newline and exit (unless the
-   ARGP_NO_EXIT flag is used).  Overridden by ARGP_PROGRAM_VERSION_HOOK.  */
-extern __const char *argp_program_version;
-
-/* If defined or set by the user program to a non-zero value, then a default
-   option --version is added (unless the ARGP_NO_HELP flag is used), which
-   calls this function with a stream to print the version to and a pointer to
-   the current parsing state, and then exits (unless the ARGP_NO_EXIT flag is
-   used).  This variable takes precedent over ARGP_PROGRAM_VERSION.  */
-extern void (*argp_program_version_hook) (FILE *__restrict __stream,
-					  struct argp_state *__restrict
-					  __state);
-
-/* If defined or set by the user program, it should point to string that is
-   the bug-reporting address for the program.  It will be printed by
-   argp_help if the ARGP_HELP_BUG_ADDR flag is set (as it is by various
-   standard help messages), embedded in a sentence that says something like
-   `Report bugs to ADDR.'.  */
-extern __const char *argp_program_bug_address;
-
-/* The exit status that argp will use when exiting due to a parsing error.
-   If not defined or set by the user program, this defaults to EX_USAGE from
-   <sysexits.h>.  */
-extern error_t argp_err_exit_status;
-
-/* Flags for argp_help.  */
-#define ARGP_HELP_USAGE		0x01 /* a Usage: message. */
-#define ARGP_HELP_SHORT_USAGE	0x02 /*  " but don't actually print options. */
-#define ARGP_HELP_SEE		0x04 /* a `Try ... for more help' message. */
-#define ARGP_HELP_LONG		0x08 /* a long help message. */
-#define ARGP_HELP_PRE_DOC	0x10 /* doc string preceding long help.  */
-#define ARGP_HELP_POST_DOC	0x20 /* doc string following long help.  */
-#define ARGP_HELP_DOC		(ARGP_HELP_PRE_DOC | ARGP_HELP_POST_DOC)
-#define ARGP_HELP_BUG_ADDR	0x40 /* bug report address */
-#define ARGP_HELP_LONG_ONLY	0x80 /* modify output appropriately to
-					reflect ARGP_LONG_ONLY mode.  */
-
-/* These ARGP_HELP flags are only understood by argp_state_help.  */
-#define ARGP_HELP_EXIT_ERR	0x100 /* Call exit(1) instead of returning.  */
-#define ARGP_HELP_EXIT_OK	0x200 /* Call exit(0) instead of returning.  */
-
-/* The standard thing to do after a program command line parsing error, if an
-   error message has already been printed.  */
-#define ARGP_HELP_STD_ERR \
-  (ARGP_HELP_SEE | ARGP_HELP_EXIT_ERR)
-/* The standard thing to do after a program command line parsing error, if no
-   more specific error message has been printed.  */
-#define ARGP_HELP_STD_USAGE \
-  (ARGP_HELP_SHORT_USAGE | ARGP_HELP_SEE | ARGP_HELP_EXIT_ERR)
-/* The standard thing to do in response to a --help option.  */
-#define ARGP_HELP_STD_HELP \
-  (ARGP_HELP_SHORT_USAGE | ARGP_HELP_LONG | ARGP_HELP_EXIT_OK \
-   | ARGP_HELP_DOC | ARGP_HELP_BUG_ADDR)
-
-/* Output a usage message for ARGP to STREAM.  FLAGS are from the set
-   ARGP_HELP_*.  */
-extern void argp_help (__const struct argp *__restrict __argp,
-		       FILE *__restrict __stream,
-		       unsigned __flags, char *__restrict __name) __THROW;
-extern void __argp_help (__const struct argp *__restrict __argp,
-			 FILE *__restrict __stream, unsigned __flags,
-			 char *__name) __THROW;
-
-/* The following routines are intended to be called from within an argp
-   parsing routine (thus taking an argp_state structure as the first
-   argument).  They may or may not print an error message and exit, depending
-   on the flags in STATE -- in any case, the caller should be prepared for
-   them *not* to exit, and should return an appropiate error after calling
-   them.  [argp_usage & argp_error should probably be called argp_state_...,
-   but they're used often enough that they should be short]  */
-
-/* Output, if appropriate, a usage message for STATE to STREAM.  FLAGS are
-   from the set ARGP_HELP_*.  */
-extern void argp_state_help (__const struct argp_state *__restrict __state,
-			     FILE *__restrict __stream,
-			     unsigned int __flags) __THROW;
-extern void __argp_state_help (__const struct argp_state *__restrict __state,
-			       FILE *__restrict __stream,
-			       unsigned int __flags) __THROW;
-
-/* Possibly output the standard usage message for ARGP to stderr and exit.  */
-extern void argp_usage (__const struct argp_state *__state) __THROW;
-extern void __argp_usage (__const struct argp_state *__state) __THROW;
-
-/* If appropriate, print the printf string FMT and following args, preceded
-   by the program name and `:', to stderr, and followed by a `Try ... --help'
-   message, then exit (1).  */
-extern void argp_error (__const struct argp_state *__restrict __state,
-			__const char *__restrict __fmt, ...) __THROW
-     PRINTF_STYLE(2,3);
-extern void __argp_error (__const struct argp_state *__restrict __state,
-			  __const char *__restrict __fmt, ...) __THROW
-     PRINTF_STYLE(2,3);
-
-/* Similar to the standard gnu error-reporting function error(), but will
-   respect the ARGP_NO_EXIT and ARGP_NO_ERRS flags in STATE, and will print
-   to STATE->err_stream.  This is useful for argument parsing code that is
-   shared between program startup (when exiting is desired) and runtime
-   option parsing (when typically an error code is returned instead).  The
-   difference between this function and argp_error is that the latter is for
-   *parsing errors*, and the former is for other problems that occur during
-   parsing but don't reflect a (syntactic) problem with the input.  */
-extern void argp_failure (__const struct argp_state *__restrict __state,
-			  int __status, int __errnum,
-			  __const char *__restrict __fmt, ...) __THROW
-     PRINTF_STYLE(4,5);
-extern void __argp_failure (__const struct argp_state *__restrict __state,
-			    int __status, int __errnum,
-			    __const char *__restrict __fmt, ...) __THROW
-     PRINTF_STYLE(4,5);
-
-/* Returns true if the option OPT is a valid short option.  */
-extern int _option_is_short (__const struct argp_option *__opt) __THROW;
-extern int __option_is_short (__const struct argp_option *__opt) __THROW;
-
-/* Returns true if the option OPT is in fact the last (unused) entry in an
-   options array.  */
-extern int _option_is_end (__const struct argp_option *__opt) __THROW;
-extern int __option_is_end (__const struct argp_option *__opt) __THROW;
-
-/* Return the input field for ARGP in the parser corresponding to STATE; used
-   by the help routines.  */
-extern void *_argp_input (__const struct argp *__restrict __argp,
-			  __const struct argp_state *__restrict __state)
-     __THROW;
-extern void *__argp_input (__const struct argp *__restrict __argp,
-			   __const struct argp_state *__restrict __state)
-     __THROW;
-
-/* Used for extracting the program name from argv[0] */
-extern char *_argp_basename(char *name) __THROW;
-extern char *__argp_basename(char *name) __THROW;
-
-/* Getting the program name given an argp state */
-extern char *
-_argp_short_program_name(const struct argp_state *state) __THROW;
-extern char *
-__argp_short_program_name(const struct argp_state *state) __THROW;
-
-
-#ifdef __USE_EXTERN_INLINES
-
-# if !_LIBC
-#  define __argp_usage argp_usage
-#  define __argp_state_help argp_state_help
-#  define __option_is_short _option_is_short
-#  define __option_is_end _option_is_end
-# endif
-
-# ifndef ARGP_EI
-#  define ARGP_EI extern __inline__
-# endif
-
-ARGP_EI void
-__argp_usage (__const struct argp_state *__state)
-{
-  __argp_state_help (__state, stderr, ARGP_HELP_STD_USAGE);
-}
-
-ARGP_EI int
-__option_is_short (__const struct argp_option *__opt)
-{
-  if (__opt->flags & OPTION_DOC)
-    return 0;
-  else
-    {
-      int __key = __opt->key;
-      return __key > 0 && isprint (__key);
-    }
-}
-
-ARGP_EI int
-__option_is_end (__const struct argp_option *__opt)
-{
-  return !__opt->key && !__opt->name && !__opt->doc && !__opt->group;
-}
-
-# if !_LIBC
-#  undef __argp_usage
-#  undef __argp_state_help
-#  undef __option_is_short
-#  undef __option_is_end
-# endif
-#endif /* Use extern inlines.  */
-
-#ifdef  __cplusplus
-}
-#endif
-
-#endif /* argp.h */
diff --git a/contrib/argp-standalone/autogen.sh b/contrib/argp-standalone/autogen.sh
deleted file mode 100755
index 8337353b5ae..00000000000
--- a/contrib/argp-standalone/autogen.sh
+++ /dev/null
@@ -1,6 +0,0 @@
-#!/bin/sh
-
-aclocal -I .
-autoheader
-autoconf
-automake --add-missing --copy --foreign
diff --git a/contrib/argp-standalone/configure.ac b/contrib/argp-standalone/configure.ac
deleted file mode 100644
index c0867eb5b35..00000000000
--- a/contrib/argp-standalone/configure.ac
+++ /dev/null
@@ -1,105 +0,0 @@
-dnl Process this file with autoconf to produce a configure script.
-
-dnl This configure.ac is only for building a standalone argp library.
-AC_INIT([argp], [standalone-1.3])
-AC_PREREQ(2.54)
-AC_CONFIG_SRCDIR([argp-ba.c])
-# Needed to stop autoconf from looking for files in parent directories.
-AC_CONFIG_AUX_DIR([.])
-
-AM_INIT_AUTOMAKE
-AC_CONFIG_HEADERS(config.h)
-
-m4_ifdef([AM_SILENT_RULES], [AM_SILENT_RULES(yes)])
-
-# GNU libc defaults to supplying the ISO C library functions only. The
-# _GNU_SOURCE define enables these extensions, in particular we want
-# errno.h to declare program_invocation_name. Enable it on all
-# systems; no problems have been reported with it so far.
-AC_GNU_SOURCE
-
-# Checks for programs.
-AC_PROG_CC
-AC_PROG_MAKE_SET
-AC_PROG_RANLIB
-AC_PROG_CC
-
-if test "x$am_cv_prog_cc_stdc" = xno ; then
-  AC_ERROR([the C compiler doesn't handle ANSI-C])
-fi
-
-# Checks for libraries.
-
-# Checks for header files.
-AC_HEADER_STDC
-AC_CHECK_HEADERS(libintl.h limits.h malloc.h unistd.h sysexits.h stdarg.h)
-
-# Checks for typedefs, structures, and compiler characteristics.
-AC_C_CONST
-AC_C_INLINE
-AC_TYPE_SIZE_T
-
-LSH_GCC_ATTRIBUTES
-
-# Checks for library functions.
-AC_FUNC_ALLOCA
-AC_FUNC_VPRINTF
-AC_CHECK_FUNCS(strerror sleep getpid snprintf)
-
-AC_REPLACE_FUNCS(mempcpy strndup strchrnul strcasecmp vsnprintf)
-
-dnl ARGP_CHECK_FUNC(includes, function-call [, if-found [, if-not-found]])
-AC_DEFUN([ARGP_CHECK_FUNC],
-  [AS_VAR_PUSHDEF([ac_func], m4_substr([$2], 0, m4_index([$2], [(])))
-   AS_VAR_PUSHDEF([ac_var], [ac_cv_func_call_]ac_func)
-   AH_TEMPLATE(AS_TR_CPP(HAVE_[]ac_func),
-               [Define to 1 if you have the `]ac_func[' function.])
-   AC_CACHE_CHECK([for $2], ac_var,
-     [AC_TRY_LINK([$1], [$2],
-                  [AS_VAR_SET(ac_var, yes)],
-                  [AS_VAR_SET(ac_var, no)])])
-   if test AS_VAR_GET(ac_var) = yes ; then
-     ifelse([$3],,
-            [AC_DEFINE_UNQUOTED(AS_TR_CPP(HAVE_[]ac_func))],
-            [$3
-])
-   else
-     ifelse([$4],, true, [$4])
-   fi
-   AS_VAR_POPDEF([ac_var])
-   AS_VAR_POPDEF([ac_func])
-   ])
-
-# At least on freebsd, putc_unlocked is a macro, so the standard
-# AC_CHECK_FUNCS doesn't work well.
-ARGP_CHECK_FUNC([#include <stdio.h>], [putc_unlocked('x', stdout)])
-
-AC_CHECK_FUNCS(flockfile)
-AC_CHECK_FUNCS(fputs_unlocked fwrite_unlocked)
-
-# Used only by argp-test.c, so don't use AC_REPLACE_FUNCS.
-AC_CHECK_FUNCS(strdup asprintf)
-
-AC_CHECK_DECLS([program_invocation_name, program_invocation_short_name],
-               [], [], [[#include <errno.h>]])
-
-# Set these flags *last*, or else the test programs won't compile
-if test x$GCC = xyes ; then
-  # Using -ggdb3 makes (some versions of) Redhat's gcc-2.96 dump core
-  if "$CC" --version | grep '^2\.96$' 1>/dev/null 2>&1; then
-    true
-  else
-    CFLAGS="$CFLAGS -ggdb3"
-  fi
-  CFLAGS="$CFLAGS -Wall -W \
- -Wmissing-prototypes -Wmissing-declarations -Wstrict-prototypes \
- -Waggregate-return \
- -Wpointer-arith -Wbad-function-cast -Wnested-externs"
-fi
-
-CPPFLAGS="$CPPFLAGS -I$srcdir"
-
-dnl Added for C99 standards
-CFLAGS="$CFLAGS -std=gnu89 -static"
-
-AC_OUTPUT(Makefile)
diff --git a/contrib/argp-standalone/mempcpy.c b/contrib/argp-standalone/mempcpy.c
deleted file mode 100644
index 21d8bd2ed94..00000000000
--- a/contrib/argp-standalone/mempcpy.c
+++ /dev/null
@@ -1,21 +0,0 @@
-/* strndup.c
- *
- */
-
-/* Written by Niels Möller <nisse@lysator.liu.se>
- *
- * This file is hereby placed in the public domain.
- */
-
-#include <string.h>
-
-void *
-mempcpy (void *, const void *, size_t) ;
-
-void *
-mempcpy (void *to, const void *from, size_t size)
-{
-  memcpy(to, from, size);
-  return (char *) to + size;
-}
-
diff --git a/contrib/argp-standalone/strcasecmp.c b/contrib/argp-standalone/strcasecmp.c
deleted file mode 100644
index 9c1637232fd..00000000000
--- a/contrib/argp-standalone/strcasecmp.c
+++ /dev/null
@@ -1,29 +0,0 @@
-/* strcasecmp.c
- *
- */
-
-/* Written by Niels Möller <nisse@lysator.liu.se>
- *
- * This file is hereby placed in the public domain.
- */
-
-#include <ctype.h>
-int strcasecmp(const char *, const char *);
-
-int strcasecmp(const char *s1, const char *s2)
-{
-  unsigned i;
-  
-  for (i = 0; s1[i] && s2[i]; i++)
-    {
-      unsigned char c1 = tolower( (unsigned char) s1[i]);
-      unsigned char c2 = tolower( (unsigned char) s2[i]);
-
-      if (c1 < c2)
-	return -1;
-      else if (c1 > c2)
-	return 1;
-    }
-
-  return !s2[i] - !s1[i];	
-}
diff --git a/contrib/argp-standalone/strchrnul.c b/contrib/argp-standalone/strchrnul.c
deleted file mode 100644
index ee4145e4eda..00000000000
--- a/contrib/argp-standalone/strchrnul.c
+++ /dev/null
@@ -1,23 +0,0 @@
-/* strchrnul.c
- *
- */
-
-/* Written by Niels Möller <nisse@lysator.liu.se>
- *
- * This file is hereby placed in the public domain.
- */
-
-/* FIXME: What is this function supposed to do? My guess is that it is
- * like strchr, but returns a pointer to the NUL character, not a NULL
- * pointer, if the character isn't found. */
-
-char *strchrnul(const char *, int );
-
-char *strchrnul(const char *s, int c)
-{
-  const char *p = s;
-  while (*p && (*p != c))
-    p++;
-
-  return (char *) p;
-}
diff --git a/contrib/argp-standalone/strndup.c b/contrib/argp-standalone/strndup.c
deleted file mode 100644
index 4147b7a2051..00000000000
--- a/contrib/argp-standalone/strndup.c
+++ /dev/null
@@ -1,34 +0,0 @@
-/* strndup.c
- *
- */
-
-/* Written by Niels Möller <nisse@lysator.liu.se>
- *
- * This file is hereby placed in the public domain.
- */
-
-#include <stdlib.h>
-#include <string.h>
-
-char *
-strndup (const char *, size_t);
-
-char *
-strndup (const char *s, size_t size)
-{
-  char *r;
-  char *end = memchr(s, 0, size);
-  
-  if (end)
-    /* Length + 1 */
-    size = end - s + 1;
-  
-  r = malloc(size);
-
-  if (size)
-    {
-      memcpy(r, s, size-1);
-      r[size-1] = '\0';
-    }
-  return r;
-}
diff --git a/contrib/argp-standalone/vsnprintf.c b/contrib/argp-standalone/vsnprintf.c
deleted file mode 100644
index 33c9a5d0042..00000000000
--- a/contrib/argp-standalone/vsnprintf.c
+++ /dev/null
@@ -1,839 +0,0 @@
-/* Copied from http://www.fiction.net/blong/programs/snprintf.c */
-
-/*
- * Copyright Patrick Powell 1995
- * This code is based on code written by Patrick Powell (papowell@astart.com)
- * It may be used for any purpose as long as this notice remains intact
- * on all source code distributions
- */
-
-/**************************************************************
- * Original:
- * Patrick Powell Tue Apr 11 09:48:21 PDT 1995
- * A bombproof version of doprnt (dopr) included.
- * Sigh.  This sort of thing is always nasty do deal with.  Note that
- * the version here does not include floating point...
- *
- * snprintf() is used instead of sprintf() as it does limit checks
- * for string length.  This covers a nasty loophole.
- *
- * The other functions are there to prevent NULL pointers from
- * causing nast effects.
- *
- * More Recently:
- *  Brandon Long <blong@fiction.net> 9/15/96 for mutt 0.43
- *  This was ugly.  It is still ugly.  I opted out of floating point
- *  numbers, but the formatter understands just about everything
- *  from the normal C string format, at least as far as I can tell from
- *  the Solaris 2.5 printf(3S) man page.
- *
- *  Brandon Long <blong@fiction.net> 10/22/97 for mutt 0.87.1
- *    Ok, added some minimal floating point support, which means this
- *    probably requires libm on most operating systems.  Don't yet
- *    support the exponent (e,E) and sigfig (g,G).  Also, fmtint()
- *    was pretty badly broken, it just wasn't being exercised in ways
- *    which showed it, so that's been fixed.  Also, formated the code
- *    to mutt conventions, and removed dead code left over from the
- *    original.  Also, there is now a builtin-test, just compile with:
- *           gcc -DTEST_SNPRINTF -o snprintf snprintf.c -lm
- *    and run snprintf for results.
- * 
- *  Thomas Roessler <roessler@guug.de> 01/27/98 for mutt 0.89i
- *    The PGP code was using unsigned hexadecimal formats. 
- *    Unfortunately, unsigned formats simply didn't work.
- *
- *  Michael Elkins <me@cs.hmc.edu> 03/05/98 for mutt 0.90.8
- *    The original code assumed that both snprintf() and vsnprintf() were
- *    missing.  Some systems only have snprintf() but not vsnprintf(), so
- *    the code is now broken down under HAVE_SNPRINTF and HAVE_VSNPRINTF.
- *
- *  Andrew Tridgell (tridge@samba.org) Oct 1998
- *    fixed handling of %.0f
- *    added test for HAVE_LONG_DOUBLE
- *
- *  Russ Allbery <rra@stanford.edu> 2000-08-26
- *    fixed return value to comply with C99
- *    fixed handling of snprintf(NULL, ...)
- *
- * Niels Möller <nisse@lysator.liu.se> 2004-03-05
- *    fixed calls to isdigit to use unsigned char.
- *    fixed calls to va_arg; short arguments are always passed as int.
- *
- **************************************************************/
-
-#if HAVE_CONFIG_H
-# include "config.h"
-#endif
-
-#if !defined(HAVE_SNPRINTF) || !defined(HAVE_VSNPRINTF)
-
-#include <string.h>
-#include <ctype.h>
-#include <sys/types.h>
-
-/* Define this as a fall through, HAVE_STDARG_H is probably already set */
-
-#define HAVE_VARARGS_H
-
-
-/* varargs declarations: */
-
-#if defined(HAVE_STDARG_H)
-# include <stdarg.h>
-# define HAVE_STDARGS    /* let's hope that works everywhere (mj) */
-# define VA_LOCAL_DECL   va_list ap
-# define VA_START(f)     va_start(ap, f)
-# define VA_SHIFT(v,t)  ;   /* no-op for ANSI */
-# define VA_END          va_end(ap)
-#else
-# if defined(HAVE_VARARGS_H)
-#  include <varargs.h>
-#  undef HAVE_STDARGS
-#  define VA_LOCAL_DECL   va_list ap
-#  define VA_START(f)     va_start(ap)      /* f is ignored! */
-#  define VA_SHIFT(v,t) v = va_arg(ap,t)
-#  define VA_END        va_end(ap)
-# else
-/*XX ** NO VARARGS ** XX*/
-# endif
-#endif
-
-#ifdef HAVE_LONG_DOUBLE
-#define LDOUBLE long double
-#else
-#define LDOUBLE double
-#endif
-
-int snprintf (char *str, size_t count, const char *fmt, ...);
-int vsnprintf (char *str, size_t count, const char *fmt, va_list arg);
-
-static int dopr (char *buffer, size_t maxlen, const char *format, 
-                 va_list args);
-static int fmtstr (char *buffer, size_t *currlen, size_t maxlen,
-		   char *value, int flags, int min, int max);
-static int fmtint (char *buffer, size_t *currlen, size_t maxlen,
-		   long value, int base, int min, int max, int flags);
-static int fmtfp (char *buffer, size_t *currlen, size_t maxlen,
-		  LDOUBLE fvalue, int min, int max, int flags);
-static int dopr_outch (char *buffer, size_t *currlen, size_t maxlen, char c );
-
-/*
- * dopr(): poor man's version of doprintf
- */
-
-/* format read states */
-#define DP_S_DEFAULT 0
-#define DP_S_FLAGS   1
-#define DP_S_MIN     2
-#define DP_S_DOT     3
-#define DP_S_MAX     4
-#define DP_S_MOD     5
-#define DP_S_CONV    6
-#define DP_S_DONE    7
-
-/* format flags - Bits */
-#define DP_F_MINUS 	(1 << 0)
-#define DP_F_PLUS  	(1 << 1)
-#define DP_F_SPACE 	(1 << 2)
-#define DP_F_NUM   	(1 << 3)
-#define DP_F_ZERO  	(1 << 4)
-#define DP_F_UP    	(1 << 5)
-#define DP_F_UNSIGNED 	(1 << 6)
-
-/* Conversion Flags */
-#define DP_C_SHORT   1
-#define DP_C_LONG    2
-#define DP_C_LDOUBLE 3
-
-#define char_to_int(p) (p - '0')
-#define MAX(p,q) ((p >= q) ? p : q)
-#define MIN(p,q) ((p <= q) ? p : q)
-
-static int dopr (char *buffer, size_t maxlen, const char *format, va_list args)
-{
-  unsigned char ch;
-  long value;
-  LDOUBLE fvalue;
-  char *strvalue;
-  int min;
-  int max;
-  int state;
-  int flags;
-  int cflags;
-  int total;
-  size_t currlen;
-  
-  state = DP_S_DEFAULT;
-  currlen = flags = cflags = min = 0;
-  max = -1;
-  ch = *format++;
-  total = 0;
-
-  while (state != DP_S_DONE)
-  {
-    if (ch == '\0')
-      state = DP_S_DONE;
-
-    switch(state) 
-    {
-    case DP_S_DEFAULT:
-      if (ch == '%') 
-	state = DP_S_FLAGS;
-      else 
-	total += dopr_outch (buffer, &currlen, maxlen, ch);
-      ch = *format++;
-      break;
-    case DP_S_FLAGS:
-      switch (ch) 
-      {
-      case '-':
-	flags |= DP_F_MINUS;
-        ch = *format++;
-	break;
-      case '+':
-	flags |= DP_F_PLUS;
-        ch = *format++;
-	break;
-      case ' ':
-	flags |= DP_F_SPACE;
-        ch = *format++;
-	break;
-      case '#':
-	flags |= DP_F_NUM;
-        ch = *format++;
-	break;
-      case '0':
-	flags |= DP_F_ZERO;
-        ch = *format++;
-	break;
-      default:
-	state = DP_S_MIN;
-	break;
-      }
-      break;
-    case DP_S_MIN:
-      if (isdigit(ch)) 
-      {
-	min = 10*min + char_to_int (ch);
-	ch = *format++;
-      } 
-      else if (ch == '*') 
-      {
-	min = va_arg (args, int);
-	ch = *format++;
-	state = DP_S_DOT;
-      } 
-      else 
-	state = DP_S_DOT;
-      break;
-    case DP_S_DOT:
-      if (ch == '.') 
-      {
-	state = DP_S_MAX;
-	ch = *format++;
-      } 
-      else 
-	state = DP_S_MOD;
-      break;
-    case DP_S_MAX:
-      if (isdigit(ch)) 
-      {
-	if (max < 0)
-	  max = 0;
-	max = 10*max + char_to_int (ch);
-	ch = *format++;
-      } 
-      else if (ch == '*') 
-      {
-	max = va_arg (args, int);
-	ch = *format++;
-	state = DP_S_MOD;
-      } 
-      else 
-	state = DP_S_MOD;
-      break;
-    case DP_S_MOD:
-      /* Currently, we don't support Long Long, bummer */
-      switch (ch) 
-      {
-      case 'h':
-	cflags = DP_C_SHORT;
-	ch = *format++;
-	break;
-      case 'l':
-	cflags = DP_C_LONG;
-	ch = *format++;
-	break;
-      case 'L':
-	cflags = DP_C_LDOUBLE;
-	ch = *format++;
-	break;
-      default:
-	break;
-      }
-      state = DP_S_CONV;
-      break;
-    case DP_S_CONV:
-      switch (ch) 
-      {
-      case 'd':
-      case 'i':
-	if (cflags == DP_C_SHORT) 
-	  value = (short) va_arg (args, int);
-	else if (cflags == DP_C_LONG)
-	  value = va_arg (args, long int);
-	else
-	  value = va_arg (args, int);
-	total += fmtint (buffer, &currlen, maxlen, value, 10, min, max, flags);
-	break;
-      case 'o':
-	flags |= DP_F_UNSIGNED;
-	if (cflags == DP_C_SHORT)
-	  value = (unsigned short) va_arg (args, unsigned);
-	else if (cflags == DP_C_LONG)
-	  value = va_arg (args, unsigned long int);
-	else
-	  value = va_arg (args, unsigned int);
-	total += fmtint (buffer, &currlen, maxlen, value, 8, min, max, flags);
-	break;
-      case 'u':
-	flags |= DP_F_UNSIGNED;
-	if (cflags == DP_C_SHORT)
-	  value = (unsigned short) va_arg (args, unsigned);
-	else if (cflags == DP_C_LONG)
-	  value = va_arg (args, unsigned long int);
-	else
-	  value = va_arg (args, unsigned int);
-	total += fmtint (buffer, &currlen, maxlen, value, 10, min, max, flags);
-	break;
-      case 'X':
-	flags |= DP_F_UP;
-      case 'x':
-	flags |= DP_F_UNSIGNED;
-	if (cflags == DP_C_SHORT)
-	  value = (unsigned short) va_arg (args, unsigned);
-	else if (cflags == DP_C_LONG)
-	  value = va_arg (args, unsigned long int);
-	else
-	  value = va_arg (args, unsigned int);
-	total += fmtint (buffer, &currlen, maxlen, value, 16, min, max, flags);
-	break;
-      case 'f':
-	if (cflags == DP_C_LDOUBLE)
-	  fvalue = va_arg (args, LDOUBLE);
-	else
-	  fvalue = va_arg (args, double);
-	/* um, floating point? */
-	total += fmtfp (buffer, &currlen, maxlen, fvalue, min, max, flags);
-	break;
-      case 'E':
-	flags |= DP_F_UP;
-      case 'e':
-	if (cflags == DP_C_LDOUBLE)
-	  fvalue = va_arg (args, LDOUBLE);
-	else
-	  fvalue = va_arg (args, double);
-	break;
-      case 'G':
-	flags |= DP_F_UP;
-      case 'g':
-	if (cflags == DP_C_LDOUBLE)
-	  fvalue = va_arg (args, LDOUBLE);
-	else
-	  fvalue = va_arg (args, double);
-	break;
-      case 'c':
-	total += dopr_outch (buffer, &currlen, maxlen, va_arg (args, int));
-	break;
-      case 's':
-	strvalue = va_arg (args, char *);
-	total += fmtstr (buffer, &currlen, maxlen, strvalue, flags, min, max);
-	break;
-      case 'p':
-	strvalue = va_arg (args, void *);
-	total += fmtint (buffer, &currlen, maxlen, (long) strvalue, 16, min,
-                         max, flags);
-	break;
-      case 'n':
-	if (cflags == DP_C_SHORT) 
-	{
-	  short int *num;
-	  num = va_arg (args, short int *);
-	  *num = currlen;
-        } 
-	else if (cflags == DP_C_LONG) 
-	{
-	  long int *num;
-	  num = va_arg (args, long int *);
-	  *num = currlen;
-        } 
-	else 
-	{
-	  int *num;
-	  num = va_arg (args, int *);
-	  *num = currlen;
-        }
-	break;
-      case '%':
-	total += dopr_outch (buffer, &currlen, maxlen, ch);
-	break;
-      case 'w':
-	/* not supported yet, treat as next char */
-	ch = *format++;
-	break;
-      default:
-	/* Unknown, skip */
-	break;
-      }
-      ch = *format++;
-      state = DP_S_DEFAULT;
-      flags = cflags = min = 0;
-      max = -1;
-      break;
-    case DP_S_DONE:
-      break;
-    default:
-      /* hmm? */
-      break; /* some picky compilers need this */
-    }
-  }
-  if (buffer != NULL)
-  {
-    if (currlen < maxlen - 1) 
-      buffer[currlen] = '\0';
-    else 
-      buffer[maxlen - 1] = '\0';
-  }
-  return total;
-}
-
-static int fmtstr (char *buffer, size_t *currlen, size_t maxlen,
-                   char *value, int flags, int min, int max)
-{
-  int padlen, strln;     /* amount to pad */
-  int cnt = 0;
-  int total = 0;
-  
-  if (value == 0)
-  {
-    value = "<NULL>";
-  }
-
-  for (strln = 0; value[strln]; ++strln); /* strlen */
-  if (max >= 0 && max < strln)
-    strln = max;
-  padlen = min - strln;
-  if (padlen < 0) 
-    padlen = 0;
-  if (flags & DP_F_MINUS) 
-    padlen = -padlen; /* Left Justify */
-
-  while (padlen > 0)
-  {
-    total += dopr_outch (buffer, currlen, maxlen, ' ');
-    --padlen;
-  }
-  while (*value && ((max < 0) || (cnt < max)))
-  {
-    total += dopr_outch (buffer, currlen, maxlen, *value++);
-    ++cnt;
-  }
-  while (padlen < 0)
-  {
-    total += dopr_outch (buffer, currlen, maxlen, ' ');
-    ++padlen;
-  }
-  return total;
-}
-
-/* Have to handle DP_F_NUM (ie 0x and 0 alternates) */
-
-static int fmtint (char *buffer, size_t *currlen, size_t maxlen,
-		   long value, int base, int min, int max, int flags)
-{
-  int signvalue = 0;
-  unsigned long uvalue;
-  char convert[20];
-  int place = 0;
-  int spadlen = 0; /* amount to space pad */
-  int zpadlen = 0; /* amount to zero pad */
-  int caps = 0;
-  int total = 0;
-  
-  if (max < 0)
-    max = 0;
-
-  uvalue = value;
-
-  if(!(flags & DP_F_UNSIGNED))
-  {
-    if( value < 0 ) {
-      signvalue = '-';
-      uvalue = -value;
-    }
-    else
-      if (flags & DP_F_PLUS)  /* Do a sign (+/i) */
-	signvalue = '+';
-    else
-      if (flags & DP_F_SPACE)
-	signvalue = ' ';
-  }
-  
-  if (flags & DP_F_UP) caps = 1; /* Should characters be upper case? */
-
-  do {
-    convert[place++] =
-      (caps? "0123456789ABCDEF":"0123456789abcdef")
-      [uvalue % (unsigned)base  ];
-    uvalue = (uvalue / (unsigned)base );
-  } while(uvalue && (place < 20));
-  if (place == 20) place--;
-  convert[place] = 0;
-
-  zpadlen = max - place;
-  spadlen = min - MAX (max, place) - (signvalue ? 1 : 0);
-  if (zpadlen < 0) zpadlen = 0;
-  if (spadlen < 0) spadlen = 0;
-  if (flags & DP_F_ZERO)
-  {
-    zpadlen = MAX(zpadlen, spadlen);
-    spadlen = 0;
-  }
-  if (flags & DP_F_MINUS) 
-    spadlen = -spadlen; /* Left Justifty */
-
-#ifdef DEBUG_SNPRINTF
-  dprint (1, (debugfile, "zpad: %d, spad: %d, min: %d, max: %d, place: %d\n",
-      zpadlen, spadlen, min, max, place));
-#endif
-
-  /* Spaces */
-  while (spadlen > 0) 
-  {
-    total += dopr_outch (buffer, currlen, maxlen, ' ');
-    --spadlen;
-  }
-
-  /* Sign */
-  if (signvalue) 
-    total += dopr_outch (buffer, currlen, maxlen, signvalue);
-
-  /* Zeros */
-  if (zpadlen > 0) 
-  {
-    while (zpadlen > 0)
-    {
-      total += dopr_outch (buffer, currlen, maxlen, '0');
-      --zpadlen;
-    }
-  }
-
-  /* Digits */
-  while (place > 0) 
-    total += dopr_outch (buffer, currlen, maxlen, convert[--place]);
-  
-  /* Left Justified spaces */
-  while (spadlen < 0) {
-    total += dopr_outch (buffer, currlen, maxlen, ' ');
-    ++spadlen;
-  }
-
-  return total;
-}
-
-static LDOUBLE abs_val (LDOUBLE value)
-{
-  LDOUBLE result = value;
-
-  if (value < 0)
-    result = -value;
-
-  return result;
-}
-
-static LDOUBLE pow10_argp (int exp)
-{
-  LDOUBLE result = 1;
-
-  while (exp)
-  {
-    result *= 10;
-    exp--;
-  }
-  
-  return result;
-}
-
-static long round_argp (LDOUBLE value)
-{
-  long intpart;
-
-  intpart = value;
-  value = value - intpart;
-  if (value >= 0.5)
-    intpart++;
-
-  return intpart;
-}
-
-static int fmtfp (char *buffer, size_t *currlen, size_t maxlen,
-		  LDOUBLE fvalue, int min, int max, int flags)
-{
-  int signvalue = 0;
-  LDOUBLE ufvalue;
-  char iconvert[20];
-  char fconvert[20];
-  int iplace = 0;
-  int fplace = 0;
-  int padlen = 0; /* amount to pad */
-  int zpadlen = 0; 
-  int caps = 0;
-  int total = 0;
-  long intpart;
-  long fracpart;
-  
-  /* 
-   * AIX manpage says the default is 0, but Solaris says the default
-   * is 6, and sprintf on AIX defaults to 6
-   */
-  if (max < 0)
-    max = 6;
-
-  ufvalue = abs_val (fvalue);
-
-  if (fvalue < 0)
-    signvalue = '-';
-  else
-    if (flags & DP_F_PLUS)  /* Do a sign (+/i) */
-      signvalue = '+';
-    else
-      if (flags & DP_F_SPACE)
-	signvalue = ' ';
-
-#if 0
-  if (flags & DP_F_UP) caps = 1; /* Should characters be upper case? */
-#endif
-
-  intpart = ufvalue;
-
-  /* 
-   * Sorry, we only support 9 digits past the decimal because of our 
-   * conversion method
-   */
-  if (max > 9)
-    max = 9;
-
-  /* We "cheat" by converting the fractional part to integer by
-   * multiplying by a factor of 10
-   */
-  fracpart = round_argp ((pow10_argp (max)) * (ufvalue - intpart));
-
-  if (fracpart >= pow10_argp (max))
-  {
-    intpart++;
-    fracpart -= pow10_argp (max);
-  }
-
-#ifdef DEBUG_SNPRINTF
-  dprint (1, (debugfile, "fmtfp: %f =? %d.%d\n", fvalue, intpart, fracpart));
-#endif
-
-  /* Convert integer part */
-  do {
-    iconvert[iplace++] =
-      (caps? "0123456789ABCDEF":"0123456789abcdef")[intpart % 10];
-    intpart = (intpart / 10);
-  } while(intpart && (iplace < 20));
-  if (iplace == 20) iplace--;
-  iconvert[iplace] = 0;
-
-  /* Convert fractional part */
-  do {
-    fconvert[fplace++] =
-      (caps? "0123456789ABCDEF":"0123456789abcdef")[fracpart % 10];
-    fracpart = (fracpart / 10);
-  } while(fracpart && (fplace < 20));
-  if (fplace == 20) fplace--;
-  fconvert[fplace] = 0;
-
-  /* -1 for decimal point, another -1 if we are printing a sign */
-  padlen = min - iplace - max - 1 - ((signvalue) ? 1 : 0); 
-  zpadlen = max - fplace;
-  if (zpadlen < 0)
-    zpadlen = 0;
-  if (padlen < 0) 
-    padlen = 0;
-  if (flags & DP_F_MINUS) 
-    padlen = -padlen; /* Left Justifty */
-
-  if ((flags & DP_F_ZERO) && (padlen > 0)) 
-  {
-    if (signvalue) 
-    {
-      total += dopr_outch (buffer, currlen, maxlen, signvalue);
-      --padlen;
-      signvalue = 0;
-    }
-    while (padlen > 0)
-    {
-      total += dopr_outch (buffer, currlen, maxlen, '0');
-      --padlen;
-    }
-  }
-  while (padlen > 0)
-  {
-    total += dopr_outch (buffer, currlen, maxlen, ' ');
-    --padlen;
-  }
-  if (signvalue) 
-    total += dopr_outch (buffer, currlen, maxlen, signvalue);
-
-  while (iplace > 0) 
-    total += dopr_outch (buffer, currlen, maxlen, iconvert[--iplace]);
-
-  /*
-   * Decimal point.  This should probably use locale to find the correct
-   * char to print out.
-   */
-  if (max > 0)
-  {
-    total += dopr_outch (buffer, currlen, maxlen, '.');
-
-    while (fplace > 0) 
-      total += dopr_outch (buffer, currlen, maxlen, fconvert[--fplace]);
-  }
-
-  while (zpadlen > 0)
-  {
-    total += dopr_outch (buffer, currlen, maxlen, '0');
-    --zpadlen;
-  }
-
-  while (padlen < 0) 
-  {
-    total += dopr_outch (buffer, currlen, maxlen, ' ');
-    ++padlen;
-  }
-
-  return total;
-}
-
-static int dopr_outch (char *buffer, size_t *currlen, size_t maxlen, char c)
-{
-  if (*currlen + 1 < maxlen)
-    buffer[(*currlen)++] = c;
-  return 1;
-}
-
-#ifndef HAVE_VSNPRINTF
-int vsnprintf (char *str, size_t count, const char *fmt, va_list args)
-{
-  if (str != NULL)
-    str[0] = 0;
-  return dopr(str, count, fmt, args);
-}
-#endif /* !HAVE_VSNPRINTF */
-
-#ifndef HAVE_SNPRINTF
-/* VARARGS3 */
-#ifdef HAVE_STDARGS
-int snprintf (char *str,size_t count,const char *fmt,...)
-#else
-int snprintf (va_alist) va_dcl
-#endif
-{
-#ifndef HAVE_STDARGS
-  char *str;
-  size_t count;
-  char *fmt;
-#endif
-  VA_LOCAL_DECL;
-  int total;
-    
-  VA_START (fmt);
-  VA_SHIFT (str, char *);
-  VA_SHIFT (count, size_t );
-  VA_SHIFT (fmt, char *);
-  total = vsnprintf(str, count, fmt, ap);
-  VA_END;
-  return total;
-}
-#endif /* !HAVE_SNPRINTF */
-
-#ifdef TEST_SNPRINTF
-#ifndef LONG_STRING
-#define LONG_STRING 1024
-#endif
-int main (void)
-{
-  char buf1[LONG_STRING];
-  char buf2[LONG_STRING];
-  char *fp_fmt[] = {
-    "%-1.5f",
-    "%1.5f",
-    "%123.9f",
-    "%10.5f",
-    "% 10.5f",
-    "%+22.9f",
-    "%+4.9f",
-    "%01.3f",
-    "%4f",
-    "%3.1f",
-    "%3.2f",
-    "%.0f",
-    "%.1f",
-    NULL
-  };
-  double fp_nums[] = { -1.5, 134.21, 91340.2, 341.1234, 0203.9, 0.96, 0.996, 
-    0.9996, 1.996, 4.136, 0};
-  char *int_fmt[] = {
-    "%-1.5d",
-    "%1.5d",
-    "%123.9d",
-    "%5.5d",
-    "%10.5d",
-    "% 10.5d",
-    "%+22.33d",
-    "%01.3d",
-    "%4d",
-    NULL
-  };
-  long int_nums[] = { -1, 134, 91340, 341, 0203, 0};
-  int x, y;
-  int fail = 0;
-  int num = 0;
-
-  printf ("Testing snprintf format codes against system sprintf...\n");
-
-  for (x = 0; fp_fmt[x] != NULL ; x++)
-    for (y = 0; fp_nums[y] != 0 ; y++)
-    {
-      snprintf (buf1, sizeof (buf1), fp_fmt[x], fp_nums[y]);
-      sprintf (buf2, fp_fmt[x], fp_nums[y]);
-      if (strcmp (buf1, buf2))
-      {
-	printf("snprintf doesn't match Format: %s\n\tsnprintf = %s\n\tsprintf  = %s\n", 
-	    fp_fmt[x], buf1, buf2);
-	fail++;
-      }
-      num++;
-    }
-
-  for (x = 0; int_fmt[x] != NULL ; x++)
-    for (y = 0; int_nums[y] != 0 ; y++)
-    {
-      snprintf (buf1, sizeof (buf1), int_fmt[x], int_nums[y]);
-      sprintf (buf2, int_fmt[x], int_nums[y]);
-      if (strcmp (buf1, buf2))
-      {
-	printf("snprintf doesn't match Format: %s\n\tsnprintf = %s\n\tsprintf  = %s\n", 
-	    int_fmt[x], buf1, buf2);
-	fail++;
-      }
-      num++;
-    }
-  printf ("%d tests failed out of %d.\n", fail, num);
-}
-#endif /* SNPRINTF_TEST */
-
-#endif /* !HAVE_SNPRINTF */
diff --git a/contrib/fuse-include/fuse-mount.h b/contrib/fuse-include/fuse-mount.h
index 9358ac810e1..7d28462de47 100644
--- a/contrib/fuse-include/fuse-mount.h
+++ b/contrib/fuse-include/fuse-mount.h
@@ -8,6 +8,6 @@
 */
 
 void gf_fuse_unmount (const char *mountpoint, int fd);
-int gf_fuse_mount (const char *mountpoint, char *fsname,
-                   unsigned long mountflags, char *mnt_param,
+int gf_fuse_unmount_daemon (const char *mountpoint, int fd);
+int gf_fuse_mount (const char *mountpoint, char *fsname, char *mnt_param,
                    pid_t *mtab_pid, int status_fd);
diff --git a/contrib/fuse-include/fuse_kernel.h b/contrib/fuse-include/fuse_kernel.h
index 60bb2f9f7b7..1e41e237e6f 100644
--- a/contrib/fuse-include/fuse_kernel.h
+++ b/contrib/fuse-include/fuse_kernel.h
@@ -93,6 +93,18 @@
  *
  * 7.22
  *  - add FUSE_ASYNC_DIO
+ *
+ * 7.23
+ *  - add FUSE_WRITEBACK_CACHE
+ *  - add time_gran to fuse_init_out
+ *  - add reserved space to fuse_init_out
+ *  - add FATTR_CTIME
+ *  - add ctime and ctimensec to fuse_setattr_in
+ *  - add FUSE_RENAME2 request
+ *  - add FUSE_NO_OPEN_SUPPORT flag
+ *
+ *  7.24
+ *  - add FUSE_LSEEK for SEEK_HOLE and SEEK_DATA support
  */
 
 #ifndef _LINUX_FUSE_H
@@ -128,7 +140,7 @@
 #define FUSE_KERNEL_VERSION 7
 
 /** Minor version number of this interface */
-#define FUSE_KERNEL_MINOR_VERSION 22
+#define FUSE_KERNEL_MINOR_VERSION 24
 
 /** The node ID of the root inode */
 #define FUSE_ROOT_ID 1
@@ -188,6 +200,7 @@ struct fuse_file_lock {
 #define FATTR_ATIME_NOW	(1 << 7)
 #define FATTR_MTIME_NOW	(1 << 8)
 #define FATTR_LOCKOWNER	(1 << 9)
+#define FATTR_CTIME	(1 << 10)
 
 /**
  * Flags returned by the OPEN request
@@ -219,6 +232,8 @@ struct fuse_file_lock {
  * FUSE_DO_READDIRPLUS: do READDIRPLUS (READDIR+LOOKUP in one)
  * FUSE_READDIRPLUS_AUTO: adaptive readdirplus
  * FUSE_ASYNC_DIO: asynchronous direct I/O submission
+ * FUSE_WRITEBACK_CACHE: use writeback cache for buffered writes
+ * FUSE_NO_OPEN_SUPPORT: kernel supports zero-message opens
  */
 #define FUSE_ASYNC_READ		(1 << 0)
 #define FUSE_POSIX_LOCKS	(1 << 1)
@@ -236,6 +251,8 @@ struct fuse_file_lock {
 #define FUSE_DO_READDIRPLUS	(1 << 13)
 #define FUSE_READDIRPLUS_AUTO	(1 << 14)
 #define FUSE_ASYNC_DIO		(1 << 15)
+#define FUSE_WRITEBACK_CACHE	(1 << 16)
+#define FUSE_NO_OPEN_SUPPORT	(1 << 17)
 
 /**
  * CUSE INIT request/reply flags
@@ -343,6 +360,8 @@ enum fuse_opcode {
 	FUSE_BATCH_FORGET  = 42,
 	FUSE_FALLOCATE     = 43,
 	FUSE_READDIRPLUS   = 44,
+	FUSE_RENAME2       = 45,
+	FUSE_LSEEK         = 46,
 
 	/* CUSE specific operations */
 	CUSE_INIT          = 4096,
@@ -421,6 +440,12 @@ struct fuse_rename_in {
 	uint64_t	newdir;
 };
 
+struct fuse_rename2_in {
+	uint64_t	newdir;
+	uint32_t	flags;
+	uint32_t	padding;
+};
+
 struct fuse_link_in {
 	uint64_t	oldnodeid;
 };
@@ -433,10 +458,10 @@ struct fuse_setattr_in {
 	uint64_t	lock_owner;
 	uint64_t	atime;
 	uint64_t	mtime;
-	uint64_t	unused2;
+	uint64_t	ctime;
 	uint32_t	atimensec;
 	uint32_t	mtimensec;
-	uint32_t	unused3;
+	uint32_t	ctimensec;
 	uint32_t	mode;
 	uint32_t	unused4;
 	uint32_t	uid;
@@ -554,6 +579,9 @@ struct fuse_init_in {
 	uint32_t	flags;
 };
 
+#define FUSE_COMPAT_INIT_OUT_SIZE 8
+#define FUSE_COMPAT_22_INIT_OUT_SIZE 24
+
 struct fuse_init_out {
 	uint32_t	major;
 	uint32_t	minor;
@@ -562,6 +590,8 @@ struct fuse_init_out {
 	uint16_t	max_background;
 	uint16_t	congestion_threshold;
 	uint32_t	max_write;
+	uint32_t	time_gran;
+	uint32_t	unused[9];
 };
 
 #define CUSE_INIT_INFO_MAX 4096
@@ -729,4 +759,15 @@ struct fuse_notify_retrieve_in {
 	uint64_t	dummy4;
 };
 
+struct fuse_lseek_in {
+	uint64_t	fh;
+	uint64_t	offset;
+	int32_t		whence;
+	uint32_t	padding;
+};
+
+struct fuse_lseek_out {
+	uint64_t	offset;
+};
+
 #endif /* _LINUX_FUSE_H */
diff --git a/contrib/fuse-lib/misc.c b/contrib/fuse-lib/misc.c
index 0c41b1a1917..1a9b418e511 100644
--- a/contrib/fuse-lib/misc.c
+++ b/contrib/fuse-lib/misc.c
@@ -10,7 +10,7 @@
 #include <string.h>
 #include <limits.h>
 #include <fcntl.h>
-#include "glusterfs.h"
+#include "glusterfs/glusterfs.h"
 #include "fuse_kernel.h"
 #include "fuse-misc.h"
 
@@ -41,7 +41,6 @@ void
 convert_fuse_file_lock (struct fuse_file_lock *fl, struct gf_flock *flock,
                         uint64_t lk_owner)
 {
-        memset (flock, 0, sizeof (struct flock));
         flock->l_type = fl->type;
         flock->l_whence = SEEK_SET;
         flock->l_start = fl->start;
diff --git a/contrib/fuse-lib/mount-common.c b/contrib/fuse-lib/mount-common.c
index e9f80fe8154..cffd4c01ed5 100644
--- a/contrib/fuse-lib/mount-common.c
+++ b/contrib/fuse-lib/mount-common.c
@@ -32,7 +32,7 @@ mtab_needs_update (const char *mnt)
         struct stat stbuf;
 
         /* If mtab is within new mount, don't touch it */
-        if (strncmp (mnt, _PATH_MOUNTED, strlen (mnt)) == 0 &&
+        if (strncmp (mnt, _PATH_MOUNTED, sizeof (_PATH_MOUNTED) - 1) == 0 &&
             _PATH_MOUNTED[strlen (mnt)] == '/')
                 return 0;
 
@@ -255,16 +255,16 @@ fuse_mnt_umount (const char *progname, const char *abs_mnt,
                         exit (1);
                 }
 #ifdef GF_LINUX_HOST_OS
-                execl ("/bin/umount", "/bin/umount", "-i", rel_mnt,
+                execl ("umount", "umount", "-i", rel_mnt,
                        lazy ? "-l" : NULL, NULL);
-                GFFUSE_LOGERR ("%s: failed to execute /bin/umount: %s",
+                GFFUSE_LOGERR ("%s: failed to execute umount: %s",
                                progname, strerror (errno));
 #elif __NetBSD__
                 /* exitting the filesystem causes the umount */
                 exit (0);
 #else
-                execl ("/sbin/umount", "/sbin/umount", "-f", rel_mnt, NULL);
-                GFFUSE_LOGERR ("%s: failed to execute /sbin/umount: %s",
+                execl ("umount", "umount", "-f", rel_mnt, NULL);
+                GFFUSE_LOGERR ("%s: failed to execute umount: %s",
                                progname, strerror (errno));
 #endif /* GF_LINUX_HOST_OS */
                 exit (1);
diff --git a/contrib/fuse-lib/mount-gluster-compat.h b/contrib/fuse-lib/mount-gluster-compat.h
index 562f089dd1f..d3646d08d8e 100644
--- a/contrib/fuse-lib/mount-gluster-compat.h
+++ b/contrib/fuse-lib/mount-gluster-compat.h
@@ -30,17 +30,59 @@
 #include <sys/wait.h>
 #include <sys/mount.h>
 
+#ifdef GF_LINUX_HOST_OS
+typedef unsigned long mount_flag_t;
+#endif
+
 #if defined(__NetBSD__)
 #include <perfuse.h>
 #define umount2(dir, flags) unmount(dir, ((flags) != 0) ? MNT_FORCE : 0)
 #define MS_RDONLY MNT_RDONLY
+#define MS_NOSUID MNT_NOSUID
+#define MS_NODEV  MNT_NODEV
+#define MS_NOATIME MNT_NOATIME
+#define MS_NOEXEC MNT_NOEXEC
+typedef int mount_flag_t;
 #endif
 
 #if defined(GF_DARWIN_HOST_OS) || defined(__FreeBSD__)
 #include <sys/param.h>
 #include <sys/mount.h>
 #define umount2(dir, flags) unmount(dir, ((flags) != 0) ? MNT_FORCE : 0)
+#endif
+
+#if defined(__FreeBSD__)
 #define MS_RDONLY MNT_RDONLY
+#define MS_NOSUID MNT_NOSUID
+/* "nodev"/MNT_NODEV was removed from FreBSD, as it became unneeded because "As
+ * of FreeBSD 6.0 device nodes may be created in regular file systems but such
+ * nodes cannot be used to access devices." (See
+ * https://freebsd.org/cgi/man.cgi?query=mknod&sektion=8 .
+ * Also see:
+ * - https://github.com/freebsd/freebsd/commit/266790a
+ * - https://github.com/freebsd/freebsd/commit/a5e716d
+ * - 700008 in
+ *   https://www.freebsd.org/doc/en/books/porters-handbook/versions-7.html .)
+ */
+#if __FreeBSD_version < 700008
+#define MS_NODEV  MNT_NODEV
+#else
+#define MS_NODEV  0
+#endif
+#define MS_NOATIME MNT_NOATIME
+#define MS_NOEXEC MNT_NOEXEC
+#if __FreeBSD_version < 1000715
+typedef int  mount_flag_t;
+#else
+/* __FreeBSD_version was not bumped for this type change. Anyway, see
+ * https://github.com/freebsd/freebsd/commit/e8d76f8
+ * and respective __FreeBSD_version:
+ * https://github.com/freebsd/freebsd/blob/e8d76f8/sys/sys/param.h#L61 .
+ * We use the subsequent value, 1000715, to switch. (Also see:
+ * https://www.freebsd.org/doc/en/books/porters-handbook/versions-10.html .)
+ */
+typedef long long mount_flag_t;
+#endif
 #endif
 
 #ifdef GF_LINUX_HOST_OS
@@ -54,9 +96,9 @@
 #define FREE(ptr) free (ptr)
 #define GFFUSE_LOGERR(...) fprintf (stderr, ## __VA_ARGS__)
 #else /* FUSE_UTIL */
-#include "glusterfs.h"
-#include "logging.h"
-#include "common-utils.h"
+#include "glusterfs/glusterfs.h"
+#include "glusterfs/logging.h"
+#include "glusterfs/common-utils.h"
 
 #define GFFUSE_LOGERR(...) \
         gf_log ("glusterfs-fuse", GF_LOG_ERROR, ## __VA_ARGS__)
diff --git a/contrib/fuse-lib/mount.c b/contrib/fuse-lib/mount.c
index 1edde86014a..06ff191f542 100644
--- a/contrib/fuse-lib/mount.c
+++ b/contrib/fuse-lib/mount.c
@@ -52,12 +52,16 @@ gf_fuse_unmount (const char *mountpoint, int fd)
         if (geteuid () == 0) {
                 fuse_mnt_umount ("fuse", mountpoint, mountpoint, 1);
                 return;
+        } else {
+            GFFUSE_LOGERR ("fuse: Effective-uid: %d", geteuid());
         }
 
         res = umount2 (mountpoint, 2);
         if (res == 0)
                 return;
 
+        GFFUSE_LOGERR ("fuse: failed to unmount %s: %s",
+                       mountpoint, strerror (errno));
         pid = fork ();
         if (pid == -1)
                 return;
@@ -67,6 +71,8 @@ gf_fuse_unmount (const char *mountpoint, int fd)
                                        "--", mountpoint, NULL };
 
                 execvp (FUSERMOUNT_PROG, (char **)argv);
+                GFFUSE_LOGERR ("fuse: failed to execute fuserumount: %s",
+                               strerror (errno));
                 _exit (1);
         }
         waitpid (pid, NULL, 0);
@@ -75,6 +81,54 @@ gf_fuse_unmount (const char *mountpoint, int fd)
 
 /* gluster-specific routines */
 
+/* Unmounting in a daemon that lurks 'till main process exits */
+int
+gf_fuse_unmount_daemon (const char *mountpoint, int fd)
+{
+        int   ret = -1;
+        pid_t pid = -1;
+
+        if (fd == -1)
+                return -1;
+
+        int ump[2] = {0,};
+
+        ret = pipe(ump);
+        if (ret == -1) {
+                close (fd);
+                return -1;
+        }
+
+        pid = fork ();
+        switch (pid) {
+        case 0:
+        {
+                char c = 0;
+                sigset_t sigset;
+
+                close_fds_except (ump, 1);
+
+                setsid();
+                (void)chdir("/");
+                sigfillset(&sigset);
+                sigprocmask(SIG_BLOCK, &sigset, NULL);
+
+                read (ump[0], &c, 1);
+
+                gf_fuse_unmount (mountpoint, fd);
+                exit (0);
+        }
+        case -1:
+                close (fd);
+                fd = -1;
+                ret = -1;
+                close (ump[1]);
+        }
+        close (ump[0]);
+
+        return ret;
+}
+
 static char *
 escape (char *s)
 {
@@ -106,8 +160,7 @@ escape (char *s)
 
 static int
 fuse_mount_fusermount (const char *mountpoint, char *fsname,
-                       unsigned long mountflags, char *mnt_param,
-                       int fd)
+                       char *mnt_param, int fd)
 {
         int  pid = -1;
         int  res = 0;
@@ -130,8 +183,7 @@ fuse_mount_fusermount (const char *mountpoint, char *fsname,
                 return -1;
         }
         ret = asprintf (&fm_mnt_params,
-                        "%s%s,fsname=%s,nonempty,subtype=glusterfs",
-                        (mountflags & MS_RDONLY) ? "ro," : "",
+                        "%s,fsname=%s,nonempty,subtype=glusterfs",
                         mnt_param, efsname);
         FREE (efsname);
         if (ret == -1) {
@@ -224,19 +276,101 @@ build_iovec_argf(struct iovec **iov, int *iovlen, const char *name,
 }
 #endif /* __FreeBSD__ */
 
+struct mount_flags {
+        const char *opt;
+        mount_flag_t flag;
+        int on;
+} mount_flags[] = {
+        /* We provide best effort cross platform support for mount flags by
+         * defining the ones which are commonly used in Unix-like OS-es.
+         */
+        {"ro",      MS_RDONLY,      1},
+        {"nosuid",  MS_NOSUID,      1},
+        {"nodev",   MS_NODEV,       1},
+        {"noatime", MS_NOATIME,     1},
+        {"noexec",  MS_NOEXEC,      1},
+#ifdef GF_LINUX_HOST_OS
+        {"rw",      MS_RDONLY,      0},
+        {"suid",    MS_NOSUID,      0},
+        {"dev",     MS_NODEV,       0},
+        {"exec",    MS_NOEXEC,      0},
+        {"async",   MS_SYNCHRONOUS, 0},
+        {"sync",    MS_SYNCHRONOUS, 1},
+        {"atime",   MS_NOATIME,     0},
+        {"dirsync", MS_DIRSYNC,     1},
+#endif
+        {NULL,      0,              0}
+};
+
+static int
+mount_param_to_flag (char *mnt_param, mount_flag_t *mntflags,
+                     char **mnt_param_new)
+{
+        gf_boolean_t found = _gf_false;
+        struct mount_flags *flag = NULL;
+        char *param_tok = NULL;
+        token_iter_t tit = {0,};
+        gf_boolean_t iter_end = _gf_false;
+
+        /* Allocate a buffer that will hold the mount parameters remaining
+         * after the ones corresponding to mount flags are processed and
+         * removed.The length of the original params are a good upper bound
+         * of the size needed.
+         */
+        *mnt_param_new = strdup (mnt_param);
+        if (!*mnt_param_new)
+                return -1;
+
+        for (param_tok = token_iter_init (*mnt_param_new, ',', &tit) ;;) {
+                iter_end = next_token (&param_tok, &tit);
+
+                found = _gf_false;
+                for (flag = mount_flags; flag->opt; flag++) {
+                        /* Compare the mount flag name to the param
+                         * name at hand.
+                         */
+                        if (strcmp (flag->opt, param_tok) == 0) {
+                                /* If there is a match, adjust mntflags
+                                 * accordingly and break.
+                                 */
+                                if (flag->on) {
+                                        *mntflags |= flag->flag;
+                                } else {
+                                        *mntflags &= ~flag->flag;
+                                }
+                                found = _gf_true;
+                                break;
+                        }
+                }
+                /* Exclude flag names from new parameter list. */
+                if (found)
+                        drop_token (param_tok, &tit);
+
+                if (iter_end)
+                        break;
+        }
+
+        return 0;
+}
+
 static int
 fuse_mount_sys (const char *mountpoint, char *fsname,
-                unsigned long mountflags, char *mnt_param, int fd)
+                char *mnt_param, int fd)
 {
         int ret = -1;
         unsigned mounted = 0;
         char *mnt_param_mnt = NULL;
         char *fstype = "fuse.glusterfs";
         char *source = fsname;
-
-        ret = asprintf (&mnt_param_mnt,
-                        "%s,fd=%i,rootmode=%o,user_id=%i,group_id=%i",
-                        mnt_param, fd, S_IFDIR, getuid (), getgid ());
+        mount_flag_t mountflags = 0;
+        char *mnt_param_new = NULL;
+
+        ret = mount_param_to_flag (mnt_param, &mountflags, &mnt_param_new);
+        if (ret == 0)
+                ret = asprintf (&mnt_param_mnt,
+                                "%s,fd=%i,rootmode=%o,user_id=%i,group_id=%i",
+                                mnt_param_new, fd, S_IFDIR, getuid (),
+                                getgid ());
         if (ret == -1) {
                 GFFUSE_LOGERR ("Out of memory");
 
@@ -256,6 +390,7 @@ fuse_mount_sys (const char *mountpoint, char *fsname,
         build_iovec (&iov, &iovlen, "from", "/dev/fuse", -1);
         build_iovec (&iov, &iovlen, "volname", source, -1);
         build_iovec (&iov, &iovlen, "fd", fdstr, -1);
+        build_iovec (&iov, &iovlen, "allow_other", NULL, -1);
         ret = nmount (iov, iovlen, mountflags);
 #else
         ret = mount (source, mountpoint, fstype, mountflags,
@@ -295,7 +430,7 @@ fuse_mount_sys (const char *mountpoint, char *fsname,
 
                 ret = asprintf (&mnt_param_mtab, "%s%s",
                                 mountflags & MS_RDONLY ? "ro," : "",
-                                mnt_param);
+                                mnt_param_new);
                 if (ret == -1)
                         GFFUSE_LOGERR ("Out of memory");
                 else {
@@ -320,6 +455,7 @@ out:
                         umount2 (mountpoint, 2); /* lazy umount */
         }
         FREE (mnt_param_mnt);
+        FREE (mnt_param_new);
         if (source != fsname)
                 FREE (source);
 
@@ -328,8 +464,7 @@ out:
 
 int
 gf_fuse_mount (const char *mountpoint, char *fsname,
-               unsigned long mountflags, char *mnt_param,
-               pid_t *mnt_pid, int status_fd)
+               char *mnt_param, pid_t *mnt_pid, int status_fd)
 {
         int   fd  = -1;
         pid_t pid = -1;
@@ -356,16 +491,19 @@ gf_fuse_mount (const char *mountpoint, char *fsname,
                                 exit (pid == -1 ? 1 : 0);
                 }
 
-                ret = fuse_mount_sys (mountpoint, fsname, mountflags, mnt_param,
-                                      fd);
+                ret = fuse_mount_sys (mountpoint, fsname, mnt_param, fd);
                 if (ret == -1) {
                         gf_log ("glusterfs-fuse", GF_LOG_INFO,
-                                "direct mount failed (%s) errno %d, "
-                                "retry to mount via fusermount",
+                                "direct mount failed (%s) errno %d",
                                 strerror (errno), errno);
 
-                        ret = fuse_mount_fusermount (mountpoint, fsname,
-                                                     mountflags, mnt_param, fd);
+                        if (errno == EPERM) {
+                                gf_log ("glusterfs-fuse", GF_LOG_INFO,
+                                        "retry to mount via fusermount");
+
+                                ret = fuse_mount_fusermount (mountpoint, fsname,
+                                                             mnt_param, fd);
+                        }
                 }
 
                 if (ret == -1)
diff --git a/contrib/fuse-util/fusermount.c b/contrib/fuse-util/fusermount.c
index a64d8e102ff..ff743f75a21 100644
--- a/contrib/fuse-util/fusermount.c
+++ b/contrib/fuse-util/fusermount.c
@@ -520,20 +520,22 @@ static void parse_line(char *line, int linenum)
 
 static void read_conf(void)
 {
+	int  len;
 	FILE *fp = fopen(FUSE_CONF, "r");
 	if (fp != NULL) {
 		int linenum = 1;
 		char line[256];
 		int isnewline = 1;
 		while (fgets(line, sizeof(line), fp) != NULL) {
+			len = strlen (line);
 			if (isnewline) {
-				if (strlen(line) && line[strlen(line)-1] == '\n') {
+				if (len && line[len-1] == '\n') {
 					strip_line(line);
 					parse_line(line, linenum);
 				} else {
 					isnewline = 0;
 				}
-			} else if(strlen(line) && line[strlen(line)-1] == '\n') {
+			} else if (len && line[len-1] == '\n') {
 				fprintf(stderr, "%s: reading %s: line %i too long\n", progname, FUSE_CONF, linenum);
 
 				isnewline = 1;
diff --git a/contrib/ipaddr-py/COPYING b/contrib/ipaddr-py/COPYING
deleted file mode 100644
index d6456956733..00000000000
--- a/contrib/ipaddr-py/COPYING
+++ /dev/null
@@ -1,202 +0,0 @@
-
-                                 Apache License
-                           Version 2.0, January 2004
-                        http://www.apache.org/licenses/
-
-   TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
-
-   1. Definitions.
-
-      "License" shall mean the terms and conditions for use, reproduction,
-      and distribution as defined by Sections 1 through 9 of this document.
-
-      "Licensor" shall mean the copyright owner or entity authorized by
-      the copyright owner that is granting the License.
-
-      "Legal Entity" shall mean the union of the acting entity and all
-      other entities that control, are controlled by, or are under common
-      control with that entity. For the purposes of this definition,
-      "control" means (i) the power, direct or indirect, to cause the
-      direction or management of such entity, whether by contract or
-      otherwise, or (ii) ownership of fifty percent (50%) or more of the
-      outstanding shares, or (iii) beneficial ownership of such entity.
-
-      "You" (or "Your") shall mean an individual or Legal Entity
-      exercising permissions granted by this License.
-
-      "Source" form shall mean the preferred form for making modifications,
-      including but not limited to software source code, documentation
-      source, and configuration files.
-
-      "Object" form shall mean any form resulting from mechanical
-      transformation or translation of a Source form, including but
-      not limited to compiled object code, generated documentation,
-      and conversions to other media types.
-
-      "Work" shall mean the work of authorship, whether in Source or
-      Object form, made available under the License, as indicated by a
-      copyright notice that is included in or attached to the work
-      (an example is provided in the Appendix below).
-
-      "Derivative Works" shall mean any work, whether in Source or Object
-      form, that is based on (or derived from) the Work and for which the
-      editorial revisions, annotations, elaborations, or other modifications
-      represent, as a whole, an original work of authorship. For the purposes
-      of this License, Derivative Works shall not include works that remain
-      separable from, or merely link (or bind by name) to the interfaces of,
-      the Work and Derivative Works thereof.
-
-      "Contribution" shall mean any work of authorship, including
-      the original version of the Work and any modifications or additions
-      to that Work or Derivative Works thereof, that is intentionally
-      submitted to Licensor for inclusion in the Work by the copyright owner
-      or by an individual or Legal Entity authorized to submit on behalf of
-      the copyright owner. For the purposes of this definition, "submitted"
-      means any form of electronic, verbal, or written communication sent
-      to the Licensor or its representatives, including but not limited to
-      communication on electronic mailing lists, source code control systems,
-      and issue tracking systems that are managed by, or on behalf of, the
-      Licensor for the purpose of discussing and improving the Work, but
-      excluding communication that is conspicuously marked or otherwise
-      designated in writing by the copyright owner as "Not a Contribution."
-
-      "Contributor" shall mean Licensor and any individual or Legal Entity
-      on behalf of whom a Contribution has been received by Licensor and
-      subsequently incorporated within the Work.
-
-   2. Grant of Copyright License. Subject to the terms and conditions of
-      this License, each Contributor hereby grants to You a perpetual,
-      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
-      copyright license to reproduce, prepare Derivative Works of,
-      publicly display, publicly perform, sublicense, and distribute the
-      Work and such Derivative Works in Source or Object form.
-
-   3. Grant of Patent License. Subject to the terms and conditions of
-      this License, each Contributor hereby grants to You a perpetual,
-      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
-      (except as stated in this section) patent license to make, have made,
-      use, offer to sell, sell, import, and otherwise transfer the Work,
-      where such license applies only to those patent claims licensable
-      by such Contributor that are necessarily infringed by their
-      Contribution(s) alone or by combination of their Contribution(s)
-      with the Work to which such Contribution(s) was submitted. If You
-      institute patent litigation against any entity (including a
-      cross-claim or counterclaim in a lawsuit) alleging that the Work
-      or a Contribution incorporated within the Work constitutes direct
-      or contributory patent infringement, then any patent licenses
-      granted to You under this License for that Work shall terminate
-      as of the date such litigation is filed.
-
-   4. Redistribution. You may reproduce and distribute copies of the
-      Work or Derivative Works thereof in any medium, with or without
-      modifications, and in Source or Object form, provided that You
-      meet the following conditions:
-
-      (a) You must give any other recipients of the Work or
-          Derivative Works a copy of this License; and
-
-      (b) You must cause any modified files to carry prominent notices
-          stating that You changed the files; and
-
-      (c) You must retain, in the Source form of any Derivative Works
-          that You distribute, all copyright, patent, trademark, and
-          attribution notices from the Source form of the Work,
-          excluding those notices that do not pertain to any part of
-          the Derivative Works; and
-
-      (d) If the Work includes a "NOTICE" text file as part of its
-          distribution, then any Derivative Works that You distribute must
-          include a readable copy of the attribution notices contained
-          within such NOTICE file, excluding those notices that do not
-          pertain to any part of the Derivative Works, in at least one
-          of the following places: within a NOTICE text file distributed
-          as part of the Derivative Works; within the Source form or
-          documentation, if provided along with the Derivative Works; or,
-          within a display generated by the Derivative Works, if and
-          wherever such third-party notices normally appear. The contents
-          of the NOTICE file are for informational purposes only and
-          do not modify the License. You may add Your own attribution
-          notices within Derivative Works that You distribute, alongside
-          or as an addendum to the NOTICE text from the Work, provided
-          that such additional attribution notices cannot be construed
-          as modifying the License.
-
-      You may add Your own copyright statement to Your modifications and
-      may provide additional or different license terms and conditions
-      for use, reproduction, or distribution of Your modifications, or
-      for any such Derivative Works as a whole, provided Your use,
-      reproduction, and distribution of the Work otherwise complies with
-      the conditions stated in this License.
-
-   5. Submission of Contributions. Unless You explicitly state otherwise,
-      any Contribution intentionally submitted for inclusion in the Work
-      by You to the Licensor shall be under the terms and conditions of
-      this License, without any additional terms or conditions.
-      Notwithstanding the above, nothing herein shall supersede or modify
-      the terms of any separate license agreement you may have executed
-      with Licensor regarding such Contributions.
-
-   6. Trademarks. This License does not grant permission to use the trade
-      names, trademarks, service marks, or product names of the Licensor,
-      except as required for reasonable and customary use in describing the
-      origin of the Work and reproducing the content of the NOTICE file.
-
-   7. Disclaimer of Warranty. Unless required by applicable law or
-      agreed to in writing, Licensor provides the Work (and each
-      Contributor provides its Contributions) on an "AS IS" BASIS,
-      WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
-      implied, including, without limitation, any warranties or conditions
-      of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
-      PARTICULAR PURPOSE. You are solely responsible for determining the
-      appropriateness of using or redistributing the Work and assume any
-      risks associated with Your exercise of permissions under this License.
-
-   8. Limitation of Liability. In no event and under no legal theory,
-      whether in tort (including negligence), contract, or otherwise,
-      unless required by applicable law (such as deliberate and grossly
-      negligent acts) or agreed to in writing, shall any Contributor be
-      liable to You for damages, including any direct, indirect, special,
-      incidental, or consequential damages of any character arising as a
-      result of this License or out of the use or inability to use the
-      Work (including but not limited to damages for loss of goodwill,
-      work stoppage, computer failure or malfunction, or any and all
-      other commercial damages or losses), even if such Contributor
-      has been advised of the possibility of such damages.
-
-   9. Accepting Warranty or Additional Liability. While redistributing
-      the Work or Derivative Works thereof, You may choose to offer,
-      and charge a fee for, acceptance of support, warranty, indemnity,
-      or other liability obligations and/or rights consistent with this
-      License. However, in accepting such obligations, You may act only
-      on Your own behalf and on Your sole responsibility, not on behalf
-      of any other Contributor, and only if You agree to indemnify,
-      defend, and hold each Contributor harmless for any liability
-      incurred by, or claims asserted against, such Contributor by reason
-      of your accepting any such warranty or additional liability.
-
-   END OF TERMS AND CONDITIONS
-
-   APPENDIX: How to apply the Apache License to your work.
-
-      To apply the Apache License to your work, attach the following
-      boilerplate notice, with the fields enclosed by brackets "[]"
-      replaced with your own identifying information. (Don't include
-      the brackets!)  The text should be enclosed in the appropriate
-      comment syntax for the file format. We also recommend that a
-      file or class name and description of purpose be included on the
-      same "printed page" as the copyright notice for easier
-      identification within third-party archives.
-
-   Copyright [yyyy] [name of copyright owner]
-
-   Licensed under the Apache License, Version 2.0 (the "License");
-   you may not use this file except in compliance with the License.
-   You may obtain a copy of the License at
-
-       http://www.apache.org/licenses/LICENSE-2.0
-
-   Unless required by applicable law or agreed to in writing, software
-   distributed under the License is distributed on an "AS IS" BASIS,
-   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-   See the License for the specific language governing permissions and
-   limitations under the License.
diff --git a/contrib/ipaddr-py/MANIFEST.in b/contrib/ipaddr-py/MANIFEST.in
deleted file mode 100644
index f572804441b..00000000000
--- a/contrib/ipaddr-py/MANIFEST.in
+++ /dev/null
@@ -1,3 +0,0 @@
-include COPYING
-include ipaddr_test.py
-include RELEASENOTES
diff --git a/contrib/ipaddr-py/OWNERS b/contrib/ipaddr-py/OWNERS
deleted file mode 100644
index 501673e0395..00000000000
--- a/contrib/ipaddr-py/OWNERS
+++ /dev/null
@@ -1,4 +0,0 @@
-pmoody
-harro
-mshields
-smart
diff --git a/contrib/ipaddr-py/README b/contrib/ipaddr-py/README
deleted file mode 100644
index 1b54294bb10..00000000000
--- a/contrib/ipaddr-py/README
+++ /dev/null
@@ -1,8 +0,0 @@
-ipaddr.py is a library for working with IP addresses, both IPv4 and IPv6.
-It was developed by Google for internal use, and is now open source.
-
-Project home page: http://code.google.com/p/ipaddr-py/
-
-Please send contributions to ipaddr-py-dev@googlegroups.com.  Code should
-include unit tests and follow the Google Python style guide:
-http://code.google.com/p/soc/wiki/PythonStyleGuide
diff --git a/contrib/ipaddr-py/ipaddr.py b/contrib/ipaddr-py/ipaddr.py
deleted file mode 100644
index a89298a315d..00000000000
--- a/contrib/ipaddr-py/ipaddr.py
+++ /dev/null
@@ -1,1907 +0,0 @@
-#!/usr/bin/python
-#
-# Copyright 2007 Google Inc.
-#  Licensed to PSF under a Contributor Agreement.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#      http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
-# implied. See the License for the specific language governing
-# permissions and limitations under the License.
-
-"""A fast, lightweight IPv4/IPv6 manipulation library in Python.
-
-This library is used to create/poke/manipulate IPv4 and IPv6 addresses
-and networks.
-
-"""
-
-__version__ = 'trunk'
-
-import struct
-
-IPV4LENGTH = 32
-IPV6LENGTH = 128
-
-
-class AddressValueError(ValueError):
-    """A Value Error related to the address."""
-
-
-class NetmaskValueError(ValueError):
-    """A Value Error related to the netmask."""
-
-
-def IPAddress(address, version=None):
-    """Take an IP string/int and return an object of the correct type.
-
-    Args:
-        address: A string or integer, the IP address.  Either IPv4 or
-          IPv6 addresses may be supplied; integers less than 2**32 will
-          be considered to be IPv4 by default.
-        version: An Integer, 4 or 6. If set, don't try to automatically
-          determine what the IP address type is. important for things
-          like IPAddress(1), which could be IPv4, '0.0.0.1',  or IPv6,
-          '::1'.
-
-    Returns:
-        An IPv4Address or IPv6Address object.
-
-    Raises:
-        ValueError: if the string passed isn't either a v4 or a v6
-          address.
-
-    """
-    if version:
-        if version == 4:
-            return IPv4Address(address)
-        elif version == 6:
-            return IPv6Address(address)
-
-    try:
-        return IPv4Address(address)
-    except (AddressValueError, NetmaskValueError):
-        pass
-
-    try:
-        return IPv6Address(address)
-    except (AddressValueError, NetmaskValueError):
-        pass
-
-    raise ValueError('%r does not appear to be an IPv4 or IPv6 address' %
-                     address)
-
-
-def IPNetwork(address, version=None, strict=False):
-    """Take an IP string/int and return an object of the correct type.
-
-    Args:
-        address: A string or integer, the IP address.  Either IPv4 or
-          IPv6 addresses may be supplied; integers less than 2**32 will
-          be considered to be IPv4 by default.
-        version: An Integer, if set, don't try to automatically
-          determine what the IP address type is. important for things
-          like IPNetwork(1), which could be IPv4, '0.0.0.1/32', or IPv6,
-          '::1/128'.
-
-    Returns:
-        An IPv4Network or IPv6Network object.
-
-    Raises:
-        ValueError: if the string passed isn't either a v4 or a v6
-          address. Or if a strict network was requested and a strict
-          network wasn't given.
-
-    """
-    if version:
-        if version == 4:
-            return IPv4Network(address, strict)
-        elif version == 6:
-            return IPv6Network(address, strict)
-
-    try:
-        return IPv4Network(address, strict)
-    except (AddressValueError, NetmaskValueError):
-        pass
-
-    try:
-        return IPv6Network(address, strict)
-    except (AddressValueError, NetmaskValueError):
-        pass
-
-    raise ValueError('%r does not appear to be an IPv4 or IPv6 network' %
-                     address)
-
-
-def v4_int_to_packed(address):
-    """The binary representation of this address.
-
-    Args:
-        address: An integer representation of an IPv4 IP address.
-
-    Returns:
-        The binary representation of this address.
-
-    Raises:
-        ValueError: If the integer is too large to be an IPv4 IP
-          address.
-    """
-    if address > _BaseV4._ALL_ONES:
-        raise ValueError('Address too large for IPv4')
-    return struct.pack('!I', address)
-
-
-def v6_int_to_packed(address):
-    """The binary representation of this address.
-
-    Args:
-        address: An integer representation of an IPv4 IP address.
-
-    Returns:
-        The binary representation of this address.
-    """
-    return struct.pack('!QQ', address >> 64, address & (2**64 - 1))
-
-
-def _find_address_range(addresses):
-    """Find a sequence of addresses.
-
-    Args:
-        addresses: a list of IPv4 or IPv6 addresses.
-
-    Returns:
-        A tuple containing the first and last IP addresses in the sequence.
-
-    """
-    first = last = addresses[0]
-    for ip in addresses[1:]:
-        if ip._ip == last._ip + 1:
-            last = ip
-        else:
-            break
-    return (first, last)
-
-def _get_prefix_length(number1, number2, bits):
-    """Get the number of leading bits that are same for two numbers.
-
-    Args:
-        number1: an integer.
-        number2: another integer.
-        bits: the maximum number of bits to compare.
-
-    Returns:
-        The number of leading bits that are the same for two numbers.
-
-    """
-    for i in range(bits):
-        if number1 >> i == number2 >> i:
-            return bits - i
-    return 0
-
-def _count_righthand_zero_bits(number, bits):
-    """Count the number of zero bits on the right hand side.
-
-    Args:
-        number: an integer.
-        bits: maximum number of bits to count.
-
-    Returns:
-        The number of zero bits on the right hand side of the number.
-
-    """
-    if number == 0:
-        return bits
-    for i in range(bits):
-        if (number >> i) % 2:
-            return i
-
-def summarize_address_range(first, last):
-    """Summarize a network range given the first and last IP addresses.
-
-    Example:
-        >>> summarize_address_range(IPv4Address('1.1.1.0'),
-            IPv4Address('1.1.1.130'))
-        [IPv4Network('1.1.1.0/25'), IPv4Network('1.1.1.128/31'),
-        IPv4Network('1.1.1.130/32')]
-
-    Args:
-        first: the first IPv4Address or IPv6Address in the range.
-        last: the last IPv4Address or IPv6Address in the range.
-
-    Returns:
-        The address range collapsed to a list of IPv4Network's or
-        IPv6Network's.
-
-    Raise:
-        TypeError:
-            If the first and last objects are not IP addresses.
-            If the first and last objects are not the same version.
-        ValueError:
-            If the last object is not greater than the first.
-            If the version is not 4 or 6.
-
-    """
-    if not (isinstance(first, _BaseIP) and isinstance(last, _BaseIP)):
-        raise TypeError('first and last must be IP addresses, not networks')
-    if first.version != last.version:
-        raise TypeError("%s and %s are not of the same version" % (
-                str(first), str(last)))
-    if first > last:
-        raise ValueError('last IP address must be greater than first')
-
-    networks = []
-
-    if first.version == 4:
-        ip = IPv4Network
-    elif first.version == 6:
-        ip = IPv6Network
-    else:
-        raise ValueError('unknown IP version')
-
-    ip_bits = first._max_prefixlen
-    first_int = first._ip
-    last_int = last._ip
-    while first_int <= last_int:
-        nbits = _count_righthand_zero_bits(first_int, ip_bits)
-        current = None
-        while nbits >= 0:
-            addend = 2**nbits - 1
-            current = first_int + addend
-            nbits -= 1
-            if current <= last_int:
-                break
-        prefix = _get_prefix_length(first_int, current, ip_bits)
-        net = ip('%s/%d' % (str(first), prefix))
-        networks.append(net)
-        if current == ip._ALL_ONES:
-            break
-        first_int = current + 1
-        first = IPAddress(first_int, version=first._version)
-    return networks
-
-def _collapse_address_list_recursive(addresses):
-    """Loops through the addresses, collapsing concurrent netblocks.
-
-    Example:
-
-        ip1 = IPv4Network('1.1.0.0/24')
-        ip2 = IPv4Network('1.1.1.0/24')
-        ip3 = IPv4Network('1.1.2.0/24')
-        ip4 = IPv4Network('1.1.3.0/24')
-        ip5 = IPv4Network('1.1.4.0/24')
-        ip6 = IPv4Network('1.1.0.1/22')
-
-        _collapse_address_list_recursive([ip1, ip2, ip3, ip4, ip5, ip6]) ->
-          [IPv4Network('1.1.0.0/22'), IPv4Network('1.1.4.0/24')]
-
-        This shouldn't be called directly; it is called via
-          collapse_address_list([]).
-
-    Args:
-        addresses: A list of IPv4Network's or IPv6Network's
-
-    Returns:
-        A list of IPv4Network's or IPv6Network's depending on what we were
-        passed.
-
-    """
-    ret_array = []
-    optimized = False
-
-    for cur_addr in addresses:
-        if not ret_array:
-            ret_array.append(cur_addr)
-            continue
-        if cur_addr in ret_array[-1]:
-            optimized = True
-        elif cur_addr == ret_array[-1].supernet().subnet()[1]:
-            ret_array.append(ret_array.pop().supernet())
-            optimized = True
-        else:
-            ret_array.append(cur_addr)
-
-    if optimized:
-        return _collapse_address_list_recursive(ret_array)
-
-    return ret_array
-
-
-def collapse_address_list(addresses):
-    """Collapse a list of IP objects.
-
-    Example:
-        collapse_address_list([IPv4('1.1.0.0/24'), IPv4('1.1.1.0/24')]) ->
-          [IPv4('1.1.0.0/23')]
-
-    Args:
-        addresses: A list of IPv4Network or IPv6Network objects.
-
-    Returns:
-        A list of IPv4Network or IPv6Network objects depending on what we
-        were passed.
-
-    Raises:
-        TypeError: If passed a list of mixed version objects.
-
-    """
-    i = 0
-    addrs = []
-    ips = []
-    nets = []
-
-    # split IP addresses and networks
-    for ip in addresses:
-        if isinstance(ip, _BaseIP):
-            if ips and ips[-1]._version != ip._version:
-                raise TypeError("%s and %s are not of the same version" % (
-                        str(ip), str(ips[-1])))
-            ips.append(ip)
-        elif ip._prefixlen == ip._max_prefixlen:
-            if ips and ips[-1]._version != ip._version:
-                raise TypeError("%s and %s are not of the same version" % (
-                        str(ip), str(ips[-1])))
-            ips.append(ip.ip)
-        else:
-            if nets and nets[-1]._version != ip._version:
-                raise TypeError("%s and %s are not of the same version" % (
-                        str(ip), str(ips[-1])))
-            nets.append(ip)
-
-    # sort and dedup
-    ips = sorted(set(ips))
-    nets = sorted(set(nets))
-
-    while i < len(ips):
-        (first, last) = _find_address_range(ips[i:])
-        i = ips.index(last) + 1
-        addrs.extend(summarize_address_range(first, last))
-
-    return _collapse_address_list_recursive(sorted(
-        addrs + nets, key=_BaseNet._get_networks_key))
-
-# backwards compatibility
-CollapseAddrList = collapse_address_list
-
-# Test whether this Python implementation supports byte objects that
-# are not identical to str ones.
-# We need to exclude platforms where bytes == str so that we can
-# distinguish between packed representations and strings, for example
-# b'12::' (the IPv4 address 49.50.58.58) and '12::' (an IPv6 address).
-try:
-    _compat_has_real_bytes = bytes is not str
-except NameError: # <Python2.6
-    _compat_has_real_bytes = False
-
-def get_mixed_type_key(obj):
-    """Return a key suitable for sorting between networks and addresses.
-
-    Address and Network objects are not sortable by default; they're
-    fundamentally different so the expression
-
-        IPv4Address('1.1.1.1') <= IPv4Network('1.1.1.1/24')
-
-    doesn't make any sense.  There are some times however, where you may wish
-    to have ipaddr sort these for you anyway. If you need to do this, you
-    can use this function as the key= argument to sorted().
-
-    Args:
-      obj: either a Network or Address object.
-    Returns:
-      appropriate key.
-
-    """
-    if isinstance(obj, _BaseNet):
-        return obj._get_networks_key()
-    elif isinstance(obj, _BaseIP):
-        return obj._get_address_key()
-    return NotImplemented
-
-class _IPAddrBase(object):
-
-    """The mother class."""
-
-    def __index__(self):
-        return self._ip
-
-    def __int__(self):
-        return self._ip
-
-    def __hex__(self):
-        return hex(self._ip)
-
-    @property
-    def exploded(self):
-        """Return the longhand version of the IP address as a string."""
-        return self._explode_shorthand_ip_string()
-
-    @property
-    def compressed(self):
-        """Return the shorthand version of the IP address as a string."""
-        return str(self)
-
-
-class _BaseIP(_IPAddrBase):
-
-    """A generic IP object.
-
-    This IP class contains the version independent methods which are
-    used by single IP addresses.
-
-    """
-
-    def __init__(self, address):
-        if (not (_compat_has_real_bytes and isinstance(address, bytes))
-            and '/' in str(address)):
-            raise AddressValueError(address)
-
-    def __eq__(self, other):
-        try:
-            return (self._ip == other._ip
-                    and self._version == other._version)
-        except AttributeError:
-            return NotImplemented
-
-    def __ne__(self, other):
-        eq = self.__eq__(other)
-        if eq is NotImplemented:
-            return NotImplemented
-        return not eq
-
-    def __le__(self, other):
-        gt = self.__gt__(other)
-        if gt is NotImplemented:
-            return NotImplemented
-        return not gt
-
-    def __ge__(self, other):
-        lt = self.__lt__(other)
-        if lt is NotImplemented:
-            return NotImplemented
-        return not lt
-
-    def __lt__(self, other):
-        if self._version != other._version:
-            raise TypeError('%s and %s are not of the same version' % (
-                    str(self), str(other)))
-        if not isinstance(other, _BaseIP):
-            raise TypeError('%s and %s are not of the same type' % (
-                    str(self), str(other)))
-        if self._ip != other._ip:
-            return self._ip < other._ip
-        return False
-
-    def __gt__(self, other):
-        if self._version != other._version:
-            raise TypeError('%s and %s are not of the same version' % (
-                    str(self), str(other)))
-        if not isinstance(other, _BaseIP):
-            raise TypeError('%s and %s are not of the same type' % (
-                    str(self), str(other)))
-        if self._ip != other._ip:
-            return self._ip > other._ip
-        return False
-
-    # Shorthand for Integer addition and subtraction. This is not
-    # meant to ever support addition/subtraction of addresses.
-    def __add__(self, other):
-        if not isinstance(other, int):
-            return NotImplemented
-        return IPAddress(int(self) + other, version=self._version)
-
-    def __sub__(self, other):
-        if not isinstance(other, int):
-            return NotImplemented
-        return IPAddress(int(self) - other, version=self._version)
-
-    def __repr__(self):
-        return '%s(%r)' % (self.__class__.__name__, str(self))
-
-    def __str__(self):
-        return  '%s' % self._string_from_ip_int(self._ip)
-
-    def __hash__(self):
-        return hash(hex(long(self._ip)))
-
-    def _get_address_key(self):
-        return (self._version, self)
-
-    @property
-    def version(self):
-        raise NotImplementedError('BaseIP has no version')
-
-
-class _BaseNet(_IPAddrBase):
-
-    """A generic IP object.
-
-    This IP class contains the version independent methods which are
-    used by networks.
-
-    """
-
-    def __init__(self, address):
-        self._cache = {}
-
-    def __repr__(self):
-        return '%s(%r)' % (self.__class__.__name__, str(self))
-
-    def iterhosts(self):
-        """Generate Iterator over usable hosts in a network.
-
-           This is like __iter__ except it doesn't return the network
-           or broadcast addresses.
-
-        """
-        cur = int(self.network) + 1
-        bcast = int(self.broadcast) - 1
-        while cur <= bcast:
-            cur += 1
-            yield IPAddress(cur - 1, version=self._version)
-
-    def __iter__(self):
-        cur = int(self.network)
-        bcast = int(self.broadcast)
-        while cur <= bcast:
-            cur += 1
-            yield IPAddress(cur - 1, version=self._version)
-
-    def __getitem__(self, n):
-        network = int(self.network)
-        broadcast = int(self.broadcast)
-        if n >= 0:
-            if network + n > broadcast:
-                raise IndexError
-            return IPAddress(network + n, version=self._version)
-        else:
-            n += 1
-            if broadcast + n < network:
-                raise IndexError
-            return IPAddress(broadcast + n, version=self._version)
-
-    def __lt__(self, other):
-        if self._version != other._version:
-            raise TypeError('%s and %s are not of the same version' % (
-                    str(self), str(other)))
-        if not isinstance(other, _BaseNet):
-            raise TypeError('%s and %s are not of the same type' % (
-                    str(self), str(other)))
-        if self.network != other.network:
-            return self.network < other.network
-        if self.netmask != other.netmask:
-            return self.netmask < other.netmask
-        return False
-
-    def __gt__(self, other):
-        if self._version != other._version:
-            raise TypeError('%s and %s are not of the same version' % (
-                    str(self), str(other)))
-        if not isinstance(other, _BaseNet):
-            raise TypeError('%s and %s are not of the same type' % (
-                    str(self), str(other)))
-        if self.network != other.network:
-            return self.network > other.network
-        if self.netmask != other.netmask:
-            return self.netmask > other.netmask
-        return False
-
-    def __le__(self, other):
-        gt = self.__gt__(other)
-        if gt is NotImplemented:
-            return NotImplemented
-        return not gt
-
-    def __ge__(self, other):
-        lt = self.__lt__(other)
-        if lt is NotImplemented:
-            return NotImplemented
-        return not lt
-
-    def __eq__(self, other):
-        try:
-            return (self._version == other._version
-                    and self.network == other.network
-                    and int(self.netmask) == int(other.netmask))
-        except AttributeError:
-            if isinstance(other, _BaseIP):
-                return (self._version == other._version
-                        and self._ip == other._ip)
-
-    def __ne__(self, other):
-        eq = self.__eq__(other)
-        if eq is NotImplemented:
-            return NotImplemented
-        return not eq
-
-    def __str__(self):
-        return  '%s/%s' % (str(self.ip),
-                           str(self._prefixlen))
-
-    def __hash__(self):
-        return hash(int(self.network) ^ int(self.netmask))
-
-    def __contains__(self, other):
-        # always false if one is v4 and the other is v6.
-        if self._version != other._version:
-          return False
-        # dealing with another network.
-        if isinstance(other, _BaseNet):
-            return (self.network <= other.network and
-                    self.broadcast >= other.broadcast)
-        # dealing with another address
-        else:
-            return (int(self.network) <= int(other._ip) <=
-                    int(self.broadcast))
-
-    def overlaps(self, other):
-        """Tell if self is partly contained in other."""
-        return self.network in other or self.broadcast in other or (
-            other.network in self or other.broadcast in self)
-
-    @property
-    def network(self):
-        x = self._cache.get('network')
-        if x is None:
-            x = IPAddress(self._ip & int(self.netmask), version=self._version)
-            self._cache['network'] = x
-        return x
-
-    @property
-    def broadcast(self):
-        x = self._cache.get('broadcast')
-        if x is None:
-            x = IPAddress(self._ip | int(self.hostmask), version=self._version)
-            self._cache['broadcast'] = x
-        return x
-
-    @property
-    def hostmask(self):
-        x = self._cache.get('hostmask')
-        if x is None:
-            x = IPAddress(int(self.netmask) ^ self._ALL_ONES,
-                          version=self._version)
-            self._cache['hostmask'] = x
-        return x
-
-    @property
-    def with_prefixlen(self):
-        return '%s/%d' % (str(self.ip), self._prefixlen)
-
-    @property
-    def with_netmask(self):
-        return '%s/%s' % (str(self.ip), str(self.netmask))
-
-    @property
-    def with_hostmask(self):
-        return '%s/%s' % (str(self.ip), str(self.hostmask))
-
-    @property
-    def numhosts(self):
-        """Number of hosts in the current subnet."""
-        return int(self.broadcast) - int(self.network) + 1
-
-    @property
-    def version(self):
-        raise NotImplementedError('BaseNet has no version')
-
-    @property
-    def prefixlen(self):
-        return self._prefixlen
-
-    def address_exclude(self, other):
-        """Remove an address from a larger block.
-
-        For example:
-
-            addr1 = IPNetwork('10.1.1.0/24')
-            addr2 = IPNetwork('10.1.1.0/26')
-            addr1.address_exclude(addr2) =
-                [IPNetwork('10.1.1.64/26'), IPNetwork('10.1.1.128/25')]
-
-        or IPv6:
-
-            addr1 = IPNetwork('::1/32')
-            addr2 = IPNetwork('::1/128')
-            addr1.address_exclude(addr2) = [IPNetwork('::0/128'),
-                IPNetwork('::2/127'),
-                IPNetwork('::4/126'),
-                IPNetwork('::8/125'),
-                ...
-                IPNetwork('0:0:8000::/33')]
-
-        Args:
-            other: An IPvXNetwork object of the same type.
-
-        Returns:
-            A sorted list of IPvXNetwork objects addresses which is self
-            minus other.
-
-        Raises:
-            TypeError: If self and other are of difffering address
-              versions, or if other is not a network object.
-            ValueError: If other is not completely contained by self.
-
-        """
-        if not self._version == other._version:
-            raise TypeError("%s and %s are not of the same version" % (
-                str(self), str(other)))
-
-        if not isinstance(other, _BaseNet):
-            raise TypeError("%s is not a network object" % str(other))
-
-        if other not in self:
-            raise ValueError('%s not contained in %s' % (str(other),
-                                                         str(self)))
-        if other == self:
-            return []
-
-        ret_addrs = []
-
-        # Make sure we're comparing the network of other.
-        other = IPNetwork('%s/%s' % (str(other.network), str(other.prefixlen)),
-                   version=other._version)
-
-        s1, s2 = self.subnet()
-        while s1 != other and s2 != other:
-            if other in s1:
-                ret_addrs.append(s2)
-                s1, s2 = s1.subnet()
-            elif other in s2:
-                ret_addrs.append(s1)
-                s1, s2 = s2.subnet()
-            else:
-                # If we got here, there's a bug somewhere.
-                assert True == False, ('Error performing exclusion: '
-                                       's1: %s s2: %s other: %s' %
-                                       (str(s1), str(s2), str(other)))
-        if s1 == other:
-            ret_addrs.append(s2)
-        elif s2 == other:
-            ret_addrs.append(s1)
-        else:
-            # If we got here, there's a bug somewhere.
-            assert True == False, ('Error performing exclusion: '
-                                   's1: %s s2: %s other: %s' %
-                                   (str(s1), str(s2), str(other)))
-
-        return sorted(ret_addrs, key=_BaseNet._get_networks_key)
-
-    def compare_networks(self, other):
-        """Compare two IP objects.
-
-        This is only concerned about the comparison of the integer
-        representation of the network addresses.  This means that the
-        host bits aren't considered at all in this method.  If you want
-        to compare host bits, you can easily enough do a
-        'HostA._ip < HostB._ip'
-
-        Args:
-            other: An IP object.
-
-        Returns:
-            If the IP versions of self and other are the same, returns:
-
-            -1 if self < other:
-              eg: IPv4('1.1.1.0/24') < IPv4('1.1.2.0/24')
-              IPv6('1080::200C:417A') < IPv6('1080::200B:417B')
-            0 if self == other
-              eg: IPv4('1.1.1.1/24') == IPv4('1.1.1.2/24')
-              IPv6('1080::200C:417A/96') == IPv6('1080::200C:417B/96')
-            1 if self > other
-              eg: IPv4('1.1.1.0/24') > IPv4('1.1.0.0/24')
-              IPv6('1080::1:200C:417A/112') >
-              IPv6('1080::0:200C:417A/112')
-
-            If the IP versions of self and other are different, returns:
-
-            -1 if self._version < other._version
-              eg: IPv4('10.0.0.1/24') < IPv6('::1/128')
-            1 if self._version > other._version
-              eg: IPv6('::1/128') > IPv4('255.255.255.0/24')
-
-        """
-        if self._version < other._version:
-            return -1
-        if self._version > other._version:
-            return 1
-        # self._version == other._version below here:
-        if self.network < other.network:
-            return -1
-        if self.network > other.network:
-            return 1
-        # self.network == other.network below here:
-        if self.netmask < other.netmask:
-            return -1
-        if self.netmask > other.netmask:
-            return 1
-        # self.network == other.network and self.netmask == other.netmask
-        return 0
-
-    def _get_networks_key(self):
-        """Network-only key function.
-
-        Returns an object that identifies this address' network and
-        netmask. This function is a suitable "key" argument for sorted()
-        and list.sort().
-
-        """
-        return (self._version, self.network, self.netmask)
-
-    def _ip_int_from_prefix(self, prefixlen=None):
-        """Turn the prefix length netmask into a int for comparison.
-
-        Args:
-            prefixlen: An integer, the prefix length.
-
-        Returns:
-            An integer.
-
-        """
-        if not prefixlen and prefixlen != 0:
-            prefixlen = self._prefixlen
-        return self._ALL_ONES ^ (self._ALL_ONES >> prefixlen)
-
-    def _prefix_from_ip_int(self, ip_int, mask=32):
-        """Return prefix length from the decimal netmask.
-
-        Args:
-            ip_int: An integer, the IP address.
-            mask: The netmask.  Defaults to 32.
-
-        Returns:
-            An integer, the prefix length.
-
-        """
-        while mask:
-            if ip_int & 1 == 1:
-                break
-            ip_int >>= 1
-            mask -= 1
-
-        return mask
-
-    def _ip_string_from_prefix(self, prefixlen=None):
-        """Turn a prefix length into a dotted decimal string.
-
-        Args:
-            prefixlen: An integer, the netmask prefix length.
-
-        Returns:
-            A string, the dotted decimal netmask string.
-
-        """
-        if not prefixlen:
-            prefixlen = self._prefixlen
-        return self._string_from_ip_int(self._ip_int_from_prefix(prefixlen))
-
-    def iter_subnets(self, prefixlen_diff=1, new_prefix=None):
-        """The subnets which join to make the current subnet.
-
-        In the case that self contains only one IP
-        (self._prefixlen == 32 for IPv4 or self._prefixlen == 128
-        for IPv6), return a list with just ourself.
-
-        Args:
-            prefixlen_diff: An integer, the amount the prefix length
-              should be increased by. This should not be set if
-              new_prefix is also set.
-            new_prefix: The desired new prefix length. This must be a
-              larger number (smaller prefix) than the existing prefix.
-              This should not be set if prefixlen_diff is also set.
-
-        Returns:
-            An iterator of IPv(4|6) objects.
-
-        Raises:
-            ValueError: The prefixlen_diff is too small or too large.
-                OR
-            prefixlen_diff and new_prefix are both set or new_prefix
-              is a smaller number than the current prefix (smaller
-              number means a larger network)
-
-        """
-        if self._prefixlen == self._max_prefixlen:
-            yield self
-            return
-
-        if new_prefix is not None:
-            if new_prefix < self._prefixlen:
-                raise ValueError('new prefix must be longer')
-            if prefixlen_diff != 1:
-                raise ValueError('cannot set prefixlen_diff and new_prefix')
-            prefixlen_diff = new_prefix - self._prefixlen
-
-        if prefixlen_diff < 0:
-            raise ValueError('prefix length diff must be > 0')
-        new_prefixlen = self._prefixlen + prefixlen_diff
-
-        if not self._is_valid_netmask(str(new_prefixlen)):
-            raise ValueError(
-                'prefix length diff %d is invalid for netblock %s' % (
-                    new_prefixlen, str(self)))
-
-        first = IPNetwork('%s/%s' % (str(self.network),
-                                     str(self._prefixlen + prefixlen_diff)),
-                         version=self._version)
-
-        yield first
-        current = first
-        while True:
-            broadcast = current.broadcast
-            if broadcast == self.broadcast:
-                return
-            new_addr = IPAddress(int(broadcast) + 1, version=self._version)
-            current = IPNetwork('%s/%s' % (str(new_addr), str(new_prefixlen)),
-                                version=self._version)
-
-            yield current
-
-    def masked(self):
-        """Return the network object with the host bits masked out."""
-        return IPNetwork('%s/%d' % (self.network, self._prefixlen),
-                         version=self._version)
-
-    def subnet(self, prefixlen_diff=1, new_prefix=None):
-        """Return a list of subnets, rather than an iterator."""
-        return list(self.iter_subnets(prefixlen_diff, new_prefix))
-
-    def supernet(self, prefixlen_diff=1, new_prefix=None):
-        """The supernet containing the current network.
-
-        Args:
-            prefixlen_diff: An integer, the amount the prefix length of
-              the network should be decreased by.  For example, given a
-              /24 network and a prefixlen_diff of 3, a supernet with a
-              /21 netmask is returned.
-
-        Returns:
-            An IPv4 network object.
-
-        Raises:
-            ValueError: If self.prefixlen - prefixlen_diff < 0. I.e., you have a
-              negative prefix length.
-                OR
-            If prefixlen_diff and new_prefix are both set or new_prefix is a
-              larger number than the current prefix (larger number means a
-              smaller network)
-
-        """
-        if self._prefixlen == 0:
-            return self
-
-        if new_prefix is not None:
-            if new_prefix > self._prefixlen:
-                raise ValueError('new prefix must be shorter')
-            if prefixlen_diff != 1:
-                raise ValueError('cannot set prefixlen_diff and new_prefix')
-            prefixlen_diff = self._prefixlen - new_prefix
-
-
-        if self.prefixlen - prefixlen_diff < 0:
-            raise ValueError(
-                'current prefixlen is %d, cannot have a prefixlen_diff of %d' %
-                (self.prefixlen, prefixlen_diff))
-        return IPNetwork('%s/%s' % (str(self.network),
-                                    str(self.prefixlen - prefixlen_diff)),
-                         version=self._version)
-
-    # backwards compatibility
-    Subnet = subnet
-    Supernet = supernet
-    AddressExclude = address_exclude
-    CompareNetworks = compare_networks
-    Contains = __contains__
-
-
-class _BaseV4(object):
-
-    """Base IPv4 object.
-
-    The following methods are used by IPv4 objects in both single IP
-    addresses and networks.
-
-    """
-
-    # Equivalent to 255.255.255.255 or 32 bits of 1's.
-    _ALL_ONES = (2**IPV4LENGTH) - 1
-    _DECIMAL_DIGITS = frozenset('0123456789')
-
-    def __init__(self, address):
-        self._version = 4
-        self._max_prefixlen = IPV4LENGTH
-
-    def _explode_shorthand_ip_string(self, ip_str=None):
-        if not ip_str:
-            ip_str = str(self)
-        return ip_str
-
-    def _ip_int_from_string(self, ip_str):
-        """Turn the given IP string into an integer for comparison.
-
-        Args:
-            ip_str: A string, the IP ip_str.
-
-        Returns:
-            The IP ip_str as an integer.
-
-        Raises:
-            AddressValueError: if ip_str isn't a valid IPv4 Address.
-
-        """
-        octets = ip_str.split('.')
-        if len(octets) != 4:
-            raise AddressValueError(ip_str)
-
-        packed_ip = 0
-        for oc in octets:
-            try:
-                packed_ip = (packed_ip << 8) | self._parse_octet(oc)
-            except ValueError:
-                raise AddressValueError(ip_str)
-        return packed_ip
-
-    def _parse_octet(self, octet_str):
-        """Convert a decimal octet into an integer.
-
-        Args:
-            octet_str: A string, the number to parse.
-
-        Returns:
-            The octet as an integer.
-
-        Raises:
-            ValueError: if the octet isn't strictly a decimal from [0..255].
-
-        """
-        # Whitelist the characters, since int() allows a lot of bizarre stuff.
-        if not self._DECIMAL_DIGITS.issuperset(octet_str):
-            raise ValueError
-        octet_int = int(octet_str, 10)
-        # Disallow leading zeroes, because no clear standard exists on
-        # whether these should be interpreted as decimal or octal.
-        if octet_int > 255 or (octet_str[0] == '0' and len(octet_str) > 1):
-            raise ValueError
-        return octet_int
-
-    def _string_from_ip_int(self, ip_int):
-        """Turns a 32-bit integer into dotted decimal notation.
-
-        Args:
-            ip_int: An integer, the IP address.
-
-        Returns:
-            The IP address as a string in dotted decimal notation.
-
-        """
-        octets = []
-        for _ in xrange(4):
-            octets.insert(0, str(ip_int & 0xFF))
-            ip_int >>= 8
-        return '.'.join(octets)
-
-    @property
-    def max_prefixlen(self):
-        return self._max_prefixlen
-
-    @property
-    def packed(self):
-        """The binary representation of this address."""
-        return v4_int_to_packed(self._ip)
-
-    @property
-    def version(self):
-        return self._version
-
-    @property
-    def is_reserved(self):
-       """Test if the address is otherwise IETF reserved.
-
-        Returns:
-            A boolean, True if the address is within the
-            reserved IPv4 Network range.
-
-       """
-       return self in IPv4Network('240.0.0.0/4')
-
-    @property
-    def is_private(self):
-        """Test if this address is allocated for private networks.
-
-        Returns:
-            A boolean, True if the address is reserved per RFC 1918.
-
-        """
-        return (self in IPv4Network('10.0.0.0/8') or
-                self in IPv4Network('172.16.0.0/12') or
-                self in IPv4Network('192.168.0.0/16'))
-
-    @property
-    def is_multicast(self):
-        """Test if the address is reserved for multicast use.
-
-        Returns:
-            A boolean, True if the address is multicast.
-            See RFC 3171 for details.
-
-        """
-        return self in IPv4Network('224.0.0.0/4')
-
-    @property
-    def is_unspecified(self):
-        """Test if the address is unspecified.
-
-        Returns:
-            A boolean, True if this is the unspecified address as defined in
-            RFC 5735 3.
-
-        """
-        return self in IPv4Network('0.0.0.0')
-
-    @property
-    def is_loopback(self):
-        """Test if the address is a loopback address.
-
-        Returns:
-            A boolean, True if the address is a loopback per RFC 3330.
-
-        """
-        return self in IPv4Network('127.0.0.0/8')
-
-    @property
-    def is_link_local(self):
-        """Test if the address is reserved for link-local.
-
-        Returns:
-            A boolean, True if the address is link-local per RFC 3927.
-
-        """
-        return self in IPv4Network('169.254.0.0/16')
-
-
-class IPv4Address(_BaseV4, _BaseIP):
-
-    """Represent and manipulate single IPv4 Addresses."""
-
-    def __init__(self, address):
-
-        """
-        Args:
-            address: A string or integer representing the IP
-              '192.168.1.1'
-
-              Additionally, an integer can be passed, so
-              IPv4Address('192.168.1.1') == IPv4Address(3232235777).
-              or, more generally
-              IPv4Address(int(IPv4Address('192.168.1.1'))) ==
-                IPv4Address('192.168.1.1')
-
-        Raises:
-            AddressValueError: If ipaddr isn't a valid IPv4 address.
-
-        """
-        _BaseIP.__init__(self, address)
-        _BaseV4.__init__(self, address)
-
-        # Efficient constructor from integer.
-        if isinstance(address, (int, long)):
-            self._ip = address
-            if address < 0 or address > self._ALL_ONES:
-                raise AddressValueError(address)
-            return
-
-        # Constructing from a packed address
-        if _compat_has_real_bytes:
-            if isinstance(address, bytes) and len(address) == 4:
-                self._ip = struct.unpack('!I', address)[0]
-                return
-
-        # Assume input argument to be string or any object representation
-        # which converts into a formatted IP string.
-        addr_str = str(address)
-        self._ip = self._ip_int_from_string(addr_str)
-
-
-class IPv4Network(_BaseV4, _BaseNet):
-
-    """This class represents and manipulates 32-bit IPv4 networks.
-
-    Attributes: [examples for IPv4Network('1.2.3.4/27')]
-        ._ip: 16909060
-        .ip: IPv4Address('1.2.3.4')
-        .network: IPv4Address('1.2.3.0')
-        .hostmask: IPv4Address('0.0.0.31')
-        .broadcast: IPv4Address('1.2.3.31')
-        .netmask: IPv4Address('255.255.255.224')
-        .prefixlen: 27
-
-    """
-
-    # the valid octets for host and netmasks. only useful for IPv4.
-    _valid_mask_octets = set((255, 254, 252, 248, 240, 224, 192, 128, 0))
-
-    def __init__(self, address, strict=False):
-        """Instantiate a new IPv4 network object.
-
-        Args:
-            address: A string or integer representing the IP [& network].
-              '192.168.1.1/24'
-              '192.168.1.1/255.255.255.0'
-              '192.168.1.1/0.0.0.255'
-              are all functionally the same in IPv4. Similarly,
-              '192.168.1.1'
-              '192.168.1.1/255.255.255.255'
-              '192.168.1.1/32'
-              are also functionaly equivalent. That is to say, failing to
-              provide a subnetmask will create an object with a mask of /32.
-
-              If the mask (portion after the / in the argument) is given in
-              dotted quad form, it is treated as a netmask if it starts with a
-              non-zero field (e.g. /255.0.0.0 == /8) and as a hostmask if it
-              starts with a zero field (e.g. 0.255.255.255 == /8), with the
-              single exception of an all-zero mask which is treated as a
-              netmask == /0. If no mask is given, a default of /32 is used.
-
-              Additionally, an integer can be passed, so
-              IPv4Network('192.168.1.1') == IPv4Network(3232235777).
-              or, more generally
-              IPv4Network(int(IPv4Network('192.168.1.1'))) ==
-                IPv4Network('192.168.1.1')
-
-            strict: A boolean. If true, ensure that we have been passed
-              A true network address, eg, 192.168.1.0/24 and not an
-              IP address on a network, eg, 192.168.1.1/24.
-
-        Raises:
-            AddressValueError: If ipaddr isn't a valid IPv4 address.
-            NetmaskValueError: If the netmask isn't valid for
-              an IPv4 address.
-            ValueError: If strict was True and a network address was not
-              supplied.
-
-        """
-        _BaseNet.__init__(self, address)
-        _BaseV4.__init__(self, address)
-
-        # Efficient constructor from integer.
-        if isinstance(address, (int, long)):
-            self._ip = address
-            self.ip = IPv4Address(self._ip)
-            self._prefixlen = self._max_prefixlen
-            self.netmask = IPv4Address(self._ALL_ONES)
-            if address < 0 or address > self._ALL_ONES:
-                raise AddressValueError(address)
-            return
-
-        # Constructing from a packed address
-        if _compat_has_real_bytes:
-            if isinstance(address, bytes) and len(address) == 4:
-                self._ip = struct.unpack('!I', address)[0]
-                self.ip = IPv4Address(self._ip)
-                self._prefixlen = self._max_prefixlen
-                self.netmask = IPv4Address(self._ALL_ONES)
-                return
-
-        # Assume input argument to be string or any object representation
-        # which converts into a formatted IP prefix string.
-        addr = str(address).split('/')
-
-        if len(addr) > 2:
-            raise AddressValueError(address)
-
-        self._ip = self._ip_int_from_string(addr[0])
-        self.ip = IPv4Address(self._ip)
-
-        if len(addr) == 2:
-            mask = addr[1].split('.')
-            if len(mask) == 4:
-                # We have dotted decimal netmask.
-                if self._is_valid_netmask(addr[1]):
-                    self.netmask = IPv4Address(self._ip_int_from_string(
-                            addr[1]))
-                elif self._is_hostmask(addr[1]):
-                    self.netmask = IPv4Address(
-                        self._ip_int_from_string(addr[1]) ^ self._ALL_ONES)
-                else:
-                    raise NetmaskValueError('%s is not a valid netmask'
-                                                     % addr[1])
-
-                self._prefixlen = self._prefix_from_ip_int(int(self.netmask))
-            else:
-                # We have a netmask in prefix length form.
-                if not self._is_valid_netmask(addr[1]):
-                    raise NetmaskValueError(addr[1])
-                self._prefixlen = int(addr[1])
-                self.netmask = IPv4Address(self._ip_int_from_prefix(
-                    self._prefixlen))
-        else:
-            self._prefixlen = self._max_prefixlen
-            self.netmask = IPv4Address(self._ip_int_from_prefix(
-                self._prefixlen))
-        if strict:
-            if self.ip != self.network:
-                raise ValueError('%s has host bits set' %
-                                 self.ip)
-
-    def _is_hostmask(self, ip_str):
-        """Test if the IP string is a hostmask (rather than a netmask).
-
-        Args:
-            ip_str: A string, the potential hostmask.
-
-        Returns:
-            A boolean, True if the IP string is a hostmask.
-
-        """
-        bits = ip_str.split('.')
-        try:
-            parts = [int(x) for x in bits if int(x) in self._valid_mask_octets]
-        except ValueError:
-            return False
-        if len(parts) != len(bits):
-            return False
-        if parts[0] < parts[-1]:
-            return True
-        return False
-
-    def _is_valid_netmask(self, netmask):
-        """Verify that the netmask is valid.
-
-        Args:
-            netmask: A string, either a prefix or dotted decimal
-              netmask.
-
-        Returns:
-            A boolean, True if the prefix represents a valid IPv4
-            netmask.
-
-        """
-        mask = netmask.split('.')
-        if len(mask) == 4:
-            if [x for x in mask if int(x) not in self._valid_mask_octets]:
-                return False
-            if [y for idx, y in enumerate(mask) if idx > 0 and
-                y > mask[idx - 1]]:
-                return False
-            return True
-        try:
-            netmask = int(netmask)
-        except ValueError:
-            return False
-        return 0 <= netmask <= self._max_prefixlen
-
-    # backwards compatibility
-    IsRFC1918 = lambda self: self.is_private
-    IsMulticast = lambda self: self.is_multicast
-    IsLoopback = lambda self: self.is_loopback
-    IsLinkLocal = lambda self: self.is_link_local
-
-
-class _BaseV6(object):
-
-    """Base IPv6 object.
-
-    The following methods are used by IPv6 objects in both single IP
-    addresses and networks.
-
-    """
-
-    _ALL_ONES = (2**IPV6LENGTH) - 1
-    _HEXTET_COUNT = 8
-    _HEX_DIGITS = frozenset('0123456789ABCDEFabcdef')
-
-    def __init__(self, address):
-        self._version = 6
-        self._max_prefixlen = IPV6LENGTH
-
-    def _ip_int_from_string(self, ip_str):
-        """Turn an IPv6 ip_str into an integer.
-
-        Args:
-            ip_str: A string, the IPv6 ip_str.
-
-        Returns:
-            A long, the IPv6 ip_str.
-
-        Raises:
-            AddressValueError: if ip_str isn't a valid IPv6 Address.
-
-        """
-        parts = ip_str.split(':')
-
-        # An IPv6 address needs at least 2 colons (3 parts).
-        if len(parts) < 3:
-            raise AddressValueError(ip_str)
-
-        # If the address has an IPv4-style suffix, convert it to hexadecimal.
-        if '.' in parts[-1]:
-            ipv4_int = IPv4Address(parts.pop())._ip
-            parts.append('%x' % ((ipv4_int >> 16) & 0xFFFF))
-            parts.append('%x' % (ipv4_int & 0xFFFF))
-
-        # An IPv6 address can't have more than 8 colons (9 parts).
-        if len(parts) > self._HEXTET_COUNT + 1:
-            raise AddressValueError(ip_str)
-
-        # Disregarding the endpoints, find '::' with nothing in between.
-        # This indicates that a run of zeroes has been skipped.
-        try:
-            skip_index, = (
-                [i for i in xrange(1, len(parts) - 1) if not parts[i]] or
-                [None])
-        except ValueError:
-            # Can't have more than one '::'
-            raise AddressValueError(ip_str)
-
-        # parts_hi is the number of parts to copy from above/before the '::'
-        # parts_lo is the number of parts to copy from below/after the '::'
-        if skip_index is not None:
-            # If we found a '::', then check if it also covers the endpoints.
-            parts_hi = skip_index
-            parts_lo = len(parts) - skip_index - 1
-            if not parts[0]:
-                parts_hi -= 1
-                if parts_hi:
-                    raise AddressValueError(ip_str)  # ^: requires ^::
-            if not parts[-1]:
-                parts_lo -= 1
-                if parts_lo:
-                    raise AddressValueError(ip_str)  # :$ requires ::$
-            parts_skipped = self._HEXTET_COUNT - (parts_hi + parts_lo)
-            if parts_skipped < 1:
-                raise AddressValueError(ip_str)
-        else:
-            # Otherwise, allocate the entire address to parts_hi.  The endpoints
-            # could still be empty, but _parse_hextet() will check for that.
-            if len(parts) != self._HEXTET_COUNT:
-                raise AddressValueError(ip_str)
-            parts_hi = len(parts)
-            parts_lo = 0
-            parts_skipped = 0
-
-        try:
-            # Now, parse the hextets into a 128-bit integer.
-            ip_int = 0L
-            for i in xrange(parts_hi):
-                ip_int <<= 16
-                ip_int |= self._parse_hextet(parts[i])
-            ip_int <<= 16 * parts_skipped
-            for i in xrange(-parts_lo, 0):
-                ip_int <<= 16
-                ip_int |= self._parse_hextet(parts[i])
-            return ip_int
-        except ValueError:
-            raise AddressValueError(ip_str)
-
-    def _parse_hextet(self, hextet_str):
-        """Convert an IPv6 hextet string into an integer.
-
-        Args:
-            hextet_str: A string, the number to parse.
-
-        Returns:
-            The hextet as an integer.
-
-        Raises:
-            ValueError: if the input isn't strictly a hex number from [0..FFFF].
-
-        """
-        # Whitelist the characters, since int() allows a lot of bizarre stuff.
-        if not self._HEX_DIGITS.issuperset(hextet_str):
-            raise ValueError
-        hextet_int = int(hextet_str, 16)
-        if hextet_int > 0xFFFF:
-            raise ValueError
-        return hextet_int
-
-    def _compress_hextets(self, hextets):
-        """Compresses a list of hextets.
-
-        Compresses a list of strings, replacing the longest continuous
-        sequence of "0" in the list with "" and adding empty strings at
-        the beginning or at the end of the string such that subsequently
-        calling ":".join(hextets) will produce the compressed version of
-        the IPv6 address.
-
-        Args:
-            hextets: A list of strings, the hextets to compress.
-
-        Returns:
-            A list of strings.
-
-        """
-        best_doublecolon_start = -1
-        best_doublecolon_len = 0
-        doublecolon_start = -1
-        doublecolon_len = 0
-        for index in range(len(hextets)):
-            if hextets[index] == '0':
-                doublecolon_len += 1
-                if doublecolon_start == -1:
-                    # Start of a sequence of zeros.
-                    doublecolon_start = index
-                if doublecolon_len > best_doublecolon_len:
-                    # This is the longest sequence of zeros so far.
-                    best_doublecolon_len = doublecolon_len
-                    best_doublecolon_start = doublecolon_start
-            else:
-                doublecolon_len = 0
-                doublecolon_start = -1
-
-        if best_doublecolon_len > 1:
-            best_doublecolon_end = (best_doublecolon_start +
-                                    best_doublecolon_len)
-            # For zeros at the end of the address.
-            if best_doublecolon_end == len(hextets):
-                hextets += ['']
-            hextets[best_doublecolon_start:best_doublecolon_end] = ['']
-            # For zeros at the beginning of the address.
-            if best_doublecolon_start == 0:
-                hextets = [''] + hextets
-
-        return hextets
-
-    def _string_from_ip_int(self, ip_int=None):
-        """Turns a 128-bit integer into hexadecimal notation.
-
-        Args:
-            ip_int: An integer, the IP address.
-
-        Returns:
-            A string, the hexadecimal representation of the address.
-
-        Raises:
-            ValueError: The address is bigger than 128 bits of all ones.
-
-        """
-        if not ip_int and ip_int != 0:
-            ip_int = int(self._ip)
-
-        if ip_int > self._ALL_ONES:
-            raise ValueError('IPv6 address is too large')
-
-        hex_str = '%032x' % ip_int
-        hextets = []
-        for x in range(0, 32, 4):
-            hextets.append('%x' % int(hex_str[x:x+4], 16))
-
-        hextets = self._compress_hextets(hextets)
-        return ':'.join(hextets)
-
-    def _explode_shorthand_ip_string(self, ip_str=None):
-        """Expand a shortened IPv6 address.
-
-        Args:
-            ip_str: A string, the IPv6 address.
-
-        Returns:
-            A string, the expanded IPv6 address.
-
-        """
-        if not ip_str:
-            ip_str = str(self)
-            if isinstance(self, _BaseNet):
-                ip_str = str(self.ip)
-
-        ip_int = self._ip_int_from_string(ip_str)
-        parts = []
-        for i in xrange(self._HEXTET_COUNT):
-            parts.append('%04x' % (ip_int & 0xFFFF))
-            ip_int >>= 16
-        parts.reverse()
-        return ':'.join(parts)
-
-    @property
-    def max_prefixlen(self):
-        return self._max_prefixlen
-
-    @property
-    def packed(self):
-        """The binary representation of this address."""
-        return v6_int_to_packed(self._ip)
-
-    @property
-    def version(self):
-        return self._version
-
-    @property
-    def is_multicast(self):
-        """Test if the address is reserved for multicast use.
-
-        Returns:
-            A boolean, True if the address is a multicast address.
-            See RFC 2373 2.7 for details.
-
-        """
-        return self in IPv6Network('ff00::/8')
-
-    @property
-    def is_reserved(self):
-        """Test if the address is otherwise IETF reserved.
-
-        Returns:
-            A boolean, True if the address is within one of the
-            reserved IPv6 Network ranges.
-
-        """
-        return (self in IPv6Network('::/8') or
-                self in IPv6Network('100::/8') or
-                self in IPv6Network('200::/7') or
-                self in IPv6Network('400::/6') or
-                self in IPv6Network('800::/5') or
-                self in IPv6Network('1000::/4') or
-                self in IPv6Network('4000::/3') or
-                self in IPv6Network('6000::/3') or
-                self in IPv6Network('8000::/3') or
-                self in IPv6Network('A000::/3') or
-                self in IPv6Network('C000::/3') or
-                self in IPv6Network('E000::/4') or
-                self in IPv6Network('F000::/5') or
-                self in IPv6Network('F800::/6') or
-                self in IPv6Network('FE00::/9'))
-
-    @property
-    def is_unspecified(self):
-        """Test if the address is unspecified.
-
-        Returns:
-            A boolean, True if this is the unspecified address as defined in
-            RFC 2373 2.5.2.
-
-        """
-        return self._ip == 0 and getattr(self, '_prefixlen', 128) == 128
-
-    @property
-    def is_loopback(self):
-        """Test if the address is a loopback address.
-
-        Returns:
-            A boolean, True if the address is a loopback address as defined in
-            RFC 2373 2.5.3.
-
-        """
-        return self._ip == 1 and getattr(self, '_prefixlen', 128) == 128
-
-    @property
-    def is_link_local(self):
-        """Test if the address is reserved for link-local.
-
-        Returns:
-            A boolean, True if the address is reserved per RFC 4291.
-
-        """
-        return self in IPv6Network('fe80::/10')
-
-    @property
-    def is_site_local(self):
-        """Test if the address is reserved for site-local.
-
-        Note that the site-local address space has been deprecated by RFC 3879.
-        Use is_private to test if this address is in the space of unique local
-        addresses as defined by RFC 4193.
-
-        Returns:
-            A boolean, True if the address is reserved per RFC 3513 2.5.6.
-
-        """
-        return self in IPv6Network('fec0::/10')
-
-    @property
-    def is_private(self):
-        """Test if this address is allocated for private networks.
-
-        Returns:
-            A boolean, True if the address is reserved per RFC 4193.
-
-        """
-        return self in IPv6Network('fc00::/7')
-
-    @property
-    def ipv4_mapped(self):
-        """Return the IPv4 mapped address.
-
-        Returns:
-            If the IPv6 address is a v4 mapped address, return the
-            IPv4 mapped address. Return None otherwise.
-
-        """
-        if (self._ip >> 32) != 0xFFFF:
-            return None
-        return IPv4Address(self._ip & 0xFFFFFFFF)
-
-    @property
-    def teredo(self):
-        """Tuple of embedded teredo IPs.
-
-        Returns:
-            Tuple of the (server, client) IPs or None if the address
-            doesn't appear to be a teredo address (doesn't start with
-            2001::/32)
-
-        """
-        if (self._ip >> 96) != 0x20010000:
-            return None
-        return (IPv4Address((self._ip >> 64) & 0xFFFFFFFF),
-                IPv4Address(~self._ip & 0xFFFFFFFF))
-
-    @property
-    def sixtofour(self):
-        """Return the IPv4 6to4 embedded address.
-
-        Returns:
-            The IPv4 6to4-embedded address if present or None if the
-            address doesn't appear to contain a 6to4 embedded address.
-
-        """
-        if (self._ip >> 112) != 0x2002:
-            return None
-        return IPv4Address((self._ip >> 80) & 0xFFFFFFFF)
-
-
-class IPv6Address(_BaseV6, _BaseIP):
-
-    """Represent and manipulate single IPv6 Addresses.
-    """
-
-    def __init__(self, address):
-        """Instantiate a new IPv6 address object.
-
-        Args:
-            address: A string or integer representing the IP
-
-              Additionally, an integer can be passed, so
-              IPv6Address('2001:4860::') ==
-                IPv6Address(42541956101370907050197289607612071936L).
-              or, more generally
-              IPv6Address(IPv6Address('2001:4860::')._ip) ==
-                IPv6Address('2001:4860::')
-
-        Raises:
-            AddressValueError: If address isn't a valid IPv6 address.
-
-        """
-        _BaseIP.__init__(self, address)
-        _BaseV6.__init__(self, address)
-
-        # Efficient constructor from integer.
-        if isinstance(address, (int, long)):
-            self._ip = address
-            if address < 0 or address > self._ALL_ONES:
-                raise AddressValueError(address)
-            return
-
-        # Constructing from a packed address
-        if _compat_has_real_bytes:
-            if isinstance(address, bytes) and len(address) == 16:
-                tmp = struct.unpack('!QQ', address)
-                self._ip = (tmp[0] << 64) | tmp[1]
-                return
-
-        # Assume input argument to be string or any object representation
-        # which converts into a formatted IP string.
-        addr_str = str(address)
-        if not addr_str:
-            raise AddressValueError('')
-
-        self._ip = self._ip_int_from_string(addr_str)
-
-
-class IPv6Network(_BaseV6, _BaseNet):
-
-    """This class represents and manipulates 128-bit IPv6 networks.
-
-    Attributes: [examples for IPv6('2001:658:22A:CAFE:200::1/64')]
-        .ip: IPv6Address('2001:658:22a:cafe:200::1')
-        .network: IPv6Address('2001:658:22a:cafe::')
-        .hostmask: IPv6Address('::ffff:ffff:ffff:ffff')
-        .broadcast: IPv6Address('2001:658:22a:cafe:ffff:ffff:ffff:ffff')
-        .netmask: IPv6Address('ffff:ffff:ffff:ffff::')
-        .prefixlen: 64
-
-    """
-
-
-    def __init__(self, address, strict=False):
-        """Instantiate a new IPv6 Network object.
-
-        Args:
-            address: A string or integer representing the IPv6 network or the IP
-              and prefix/netmask.
-              '2001:4860::/128'
-              '2001:4860:0000:0000:0000:0000:0000:0000/128'
-              '2001:4860::'
-              are all functionally the same in IPv6.  That is to say,
-              failing to provide a subnetmask will create an object with
-              a mask of /128.
-
-              Additionally, an integer can be passed, so
-              IPv6Network('2001:4860::') ==
-                IPv6Network(42541956101370907050197289607612071936L).
-              or, more generally
-              IPv6Network(IPv6Network('2001:4860::')._ip) ==
-                IPv6Network('2001:4860::')
-
-            strict: A boolean. If true, ensure that we have been passed
-              A true network address, eg, 192.168.1.0/24 and not an
-              IP address on a network, eg, 192.168.1.1/24.
-
-        Raises:
-            AddressValueError: If address isn't a valid IPv6 address.
-            NetmaskValueError: If the netmask isn't valid for
-              an IPv6 address.
-            ValueError: If strict was True and a network address was not
-              supplied.
-
-        """
-        _BaseNet.__init__(self, address)
-        _BaseV6.__init__(self, address)
-
-        # Efficient constructor from integer.
-        if isinstance(address, (int, long)):
-            self._ip = address
-            self.ip = IPv6Address(self._ip)
-            self._prefixlen = self._max_prefixlen
-            self.netmask = IPv6Address(self._ALL_ONES)
-            if address < 0 or address > self._ALL_ONES:
-                raise AddressValueError(address)
-            return
-
-        # Constructing from a packed address
-        if _compat_has_real_bytes:
-            if isinstance(address, bytes) and len(address) == 16:
-                tmp = struct.unpack('!QQ', address)
-                self._ip = (tmp[0] << 64) | tmp[1]
-                self.ip = IPv6Address(self._ip)
-                self._prefixlen = self._max_prefixlen
-                self.netmask = IPv6Address(self._ALL_ONES)
-                return
-
-        # Assume input argument to be string or any object representation
-        # which converts into a formatted IP prefix string.
-        addr = str(address).split('/')
-
-        if len(addr) > 2:
-            raise AddressValueError(address)
-
-        self._ip = self._ip_int_from_string(addr[0])
-        self.ip = IPv6Address(self._ip)
-
-        if len(addr) == 2:
-            if self._is_valid_netmask(addr[1]):
-                self._prefixlen = int(addr[1])
-            else:
-                raise NetmaskValueError(addr[1])
-        else:
-            self._prefixlen = self._max_prefixlen
-
-        self.netmask = IPv6Address(self._ip_int_from_prefix(self._prefixlen))
-
-        if strict:
-            if self.ip != self.network:
-                raise ValueError('%s has host bits set' %
-                                 self.ip)
-
-    def _is_valid_netmask(self, prefixlen):
-        """Verify that the netmask/prefixlen is valid.
-
-        Args:
-            prefixlen: A string, the netmask in prefix length format.
-
-        Returns:
-            A boolean, True if the prefix represents a valid IPv6
-            netmask.
-
-        """
-        try:
-            prefixlen = int(prefixlen)
-        except ValueError:
-            return False
-        return 0 <= prefixlen <= self._max_prefixlen
-
-    @property
-    def with_netmask(self):
-        return self.with_prefixlen
diff --git a/contrib/ipaddr-py/ipaddr_test.py b/contrib/ipaddr-py/ipaddr_test.py
deleted file mode 100755
index 09bece0e751..00000000000
--- a/contrib/ipaddr-py/ipaddr_test.py
+++ /dev/null
@@ -1,1099 +0,0 @@
-#!/usr/bin/python
-#
-# Copyright 2007 Google Inc.
-#  Licensed to PSF under a Contributor Agreement.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#      http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-"""Unittest for ipaddr module."""
-
-
-import unittest
-import time
-import ipaddr
-
-# Compatibility function to cast str to bytes objects
-if ipaddr._compat_has_real_bytes:
-    _cb = lambda bytestr: bytes(bytestr, 'charmap')
-else:
-    _cb = str
-
-class IpaddrUnitTest(unittest.TestCase):
-
-    def setUp(self):
-        self.ipv4 = ipaddr.IPv4Network('1.2.3.4/24')
-        self.ipv4_hostmask = ipaddr.IPv4Network('10.0.0.1/0.255.255.255')
-        self.ipv6 = ipaddr.IPv6Network('2001:658:22a:cafe:200:0:0:1/64')
-
-    def tearDown(self):
-        del(self.ipv4)
-        del(self.ipv4_hostmask)
-        del(self.ipv6)
-        del(self)
-
-    def testRepr(self):
-        self.assertEqual("IPv4Network('1.2.3.4/32')",
-                         repr(ipaddr.IPv4Network('1.2.3.4')))
-        self.assertEqual("IPv6Network('::1/128')",
-                         repr(ipaddr.IPv6Network('::1')))
-
-    def testAutoMasking(self):
-        addr1 = ipaddr.IPv4Network('1.1.1.255/24')
-        addr1_masked = ipaddr.IPv4Network('1.1.1.0/24')
-        self.assertEqual(addr1_masked, addr1.masked())
-
-        addr2 = ipaddr.IPv6Network('2000:cafe::efac:100/96')
-        addr2_masked = ipaddr.IPv6Network('2000:cafe::/96')
-        self.assertEqual(addr2_masked, addr2.masked())
-
-    # issue57
-    def testAddressIntMath(self):
-        self.assertEqual(ipaddr.IPv4Address('1.1.1.1') + 255,
-                         ipaddr.IPv4Address('1.1.2.0'))
-        self.assertEqual(ipaddr.IPv4Address('1.1.1.1') - 256,
-                         ipaddr.IPv4Address('1.1.0.1'))
-        self.assertEqual(ipaddr.IPv6Address('::1') + (2**16 - 2),
-                         ipaddr.IPv6Address('::ffff'))
-        self.assertEqual(ipaddr.IPv6Address('::ffff') - (2**16 - 2),
-                         ipaddr.IPv6Address('::1'))
-
-    def testInvalidStrings(self):
-        def AssertInvalidIP(ip_str):
-            self.assertRaises(ValueError, ipaddr.IPAddress, ip_str)
-        AssertInvalidIP("")
-        AssertInvalidIP("016.016.016.016")
-        AssertInvalidIP("016.016.016")
-        AssertInvalidIP("016.016")
-        AssertInvalidIP("016")
-        AssertInvalidIP("000.000.000.000")
-        AssertInvalidIP("000")
-        AssertInvalidIP("0x0a.0x0a.0x0a.0x0a")
-        AssertInvalidIP("0x0a.0x0a.0x0a")
-        AssertInvalidIP("0x0a.0x0a")
-        AssertInvalidIP("0x0a")
-        AssertInvalidIP("42.42.42.42.42")
-        AssertInvalidIP("42.42.42")
-        AssertInvalidIP("42.42")
-        AssertInvalidIP("42")
-        AssertInvalidIP("42..42.42")
-        AssertInvalidIP("42..42.42.42")
-        AssertInvalidIP("42.42.42.42.")
-        AssertInvalidIP("42.42.42.42...")
-        AssertInvalidIP(".42.42.42.42")
-        AssertInvalidIP("...42.42.42.42")
-        AssertInvalidIP("42.42.42.-0")
-        AssertInvalidIP("42.42.42.+0")
-        AssertInvalidIP(".")
-        AssertInvalidIP("...")
-        AssertInvalidIP("bogus")
-        AssertInvalidIP("bogus.com")
-        AssertInvalidIP("192.168.0.1.com")
-        AssertInvalidIP("12345.67899.-54321.-98765")
-        AssertInvalidIP("257.0.0.0")
-        AssertInvalidIP("42.42.42.-42")
-        AssertInvalidIP("3ffe::1.net")
-        AssertInvalidIP("3ffe::1::1")
-        AssertInvalidIP("1::2::3::4:5")
-        AssertInvalidIP("::7:6:5:4:3:2:")
-        AssertInvalidIP(":6:5:4:3:2:1::")
-        AssertInvalidIP("2001::db:::1")
-        AssertInvalidIP("FEDC:9878")
-        AssertInvalidIP("+1.+2.+3.4")
-        AssertInvalidIP("1.2.3.4e0")
-        AssertInvalidIP("::7:6:5:4:3:2:1:0")
-        AssertInvalidIP("7:6:5:4:3:2:1:0::")
-        AssertInvalidIP("9:8:7:6:5:4:3::2:1")
-        AssertInvalidIP("0:1:2:3::4:5:6:7")
-        AssertInvalidIP("3ffe:0:0:0:0:0:0:0:1")
-        AssertInvalidIP("3ffe::10000")
-        AssertInvalidIP("3ffe::goog")
-        AssertInvalidIP("3ffe::-0")
-        AssertInvalidIP("3ffe::+0")
-        AssertInvalidIP("3ffe::-1")
-        AssertInvalidIP(":")
-        AssertInvalidIP(":::")
-        AssertInvalidIP("::1.2.3")
-        AssertInvalidIP("::1.2.3.4.5")
-        AssertInvalidIP("::1.2.3.4:")
-        AssertInvalidIP("1.2.3.4::")
-        AssertInvalidIP("2001:db8::1:")
-        AssertInvalidIP(":2001:db8::1")
-        AssertInvalidIP(":1:2:3:4:5:6:7")
-        AssertInvalidIP("1:2:3:4:5:6:7:")
-        AssertInvalidIP(":1:2:3:4:5:6:")
-
-        self.assertRaises(ipaddr.AddressValueError, ipaddr.IPv4Network, '')
-        self.assertRaises(ipaddr.AddressValueError, ipaddr.IPv4Network,
-                          'google.com')
-        self.assertRaises(ipaddr.AddressValueError, ipaddr.IPv4Network,
-                          '::1.2.3.4')
-        self.assertRaises(ipaddr.AddressValueError, ipaddr.IPv6Network, '')
-        self.assertRaises(ipaddr.AddressValueError, ipaddr.IPv6Network,
-                          'google.com')
-        self.assertRaises(ipaddr.AddressValueError, ipaddr.IPv6Network,
-                          '1.2.3.4')
-        self.assertRaises(ipaddr.AddressValueError, ipaddr.IPv6Network,
-                          'cafe:cafe::/128/190')
-        self.assertRaises(ipaddr.AddressValueError, ipaddr.IPv6Network,
-                          '1234:axy::b')
-        self.assertRaises(ipaddr.AddressValueError, ipaddr.IPv6Address,
-                          '1234:axy::b')
-        self.assertRaises(ipaddr.AddressValueError, ipaddr.IPv6Address,
-                          '2001:db8:::1')
-        self.assertRaises(ipaddr.AddressValueError, ipaddr.IPv6Address,
-                          '2001:888888::1')
-        self.assertRaises(ipaddr.AddressValueError,
-                          ipaddr.IPv4Address(1)._ip_int_from_string,
-                          '1.a.2.3')
-        self.assertEqual(False, ipaddr.IPv4Network(1)._is_hostmask('1.a.2.3'))
-
-    def testGetNetwork(self):
-        self.assertEqual(int(self.ipv4.network), 16909056)
-        self.assertEqual(str(self.ipv4.network), '1.2.3.0')
-        self.assertEqual(str(self.ipv4_hostmask.network), '10.0.0.0')
-
-        self.assertEqual(int(self.ipv6.network),
-                         42540616829182469433403647294022090752)
-        self.assertEqual(str(self.ipv6.network),
-                         '2001:658:22a:cafe::')
-        self.assertEqual(str(self.ipv6.hostmask),
-                         '::ffff:ffff:ffff:ffff')
-
-    def testBadVersionComparison(self):
-        # These should always raise TypeError
-        v4addr = ipaddr.IPAddress('1.1.1.1')
-        v4net = ipaddr.IPNetwork('1.1.1.1')
-        v6addr = ipaddr.IPAddress('::1')
-        v6net = ipaddr.IPAddress('::1')
-
-        self.assertRaises(TypeError, v4addr.__lt__, v6addr)
-        self.assertRaises(TypeError, v4addr.__gt__, v6addr)
-        self.assertRaises(TypeError, v4net.__lt__, v6net)
-        self.assertRaises(TypeError, v4net.__gt__, v6net)
-
-        self.assertRaises(TypeError, v6addr.__lt__, v4addr)
-        self.assertRaises(TypeError, v6addr.__gt__, v4addr)
-        self.assertRaises(TypeError, v6net.__lt__, v4net)
-        self.assertRaises(TypeError, v6net.__gt__, v4net)
-
-    def testMixedTypeComparison(self):
-        v4addr = ipaddr.IPAddress('1.1.1.1')
-        v4net = ipaddr.IPNetwork('1.1.1.1/32')
-        v6addr = ipaddr.IPAddress('::1')
-        v6net = ipaddr.IPNetwork('::1/128')
-
-        self.assertFalse(v4net.__contains__(v6net))
-        self.assertFalse(v6net.__contains__(v4net))
-
-        self.assertRaises(TypeError, lambda: v4addr < v4net)
-        self.assertRaises(TypeError, lambda: v4addr > v4net)
-        self.assertRaises(TypeError, lambda: v4net < v4addr)
-        self.assertRaises(TypeError, lambda: v4net > v4addr)
-
-        self.assertRaises(TypeError, lambda: v6addr < v6net)
-        self.assertRaises(TypeError, lambda: v6addr > v6net)
-        self.assertRaises(TypeError, lambda: v6net < v6addr)
-        self.assertRaises(TypeError, lambda: v6net > v6addr)
-
-        # with get_mixed_type_key, you can sort addresses and network.
-        self.assertEqual([v4addr, v4net], sorted([v4net, v4addr],
-                                                 key=ipaddr.get_mixed_type_key))
-        self.assertEqual([v6addr, v6net], sorted([v6net, v6addr],
-                                                 key=ipaddr.get_mixed_type_key))
-
-    def testIpFromInt(self):
-        self.assertEqual(self.ipv4.ip, ipaddr.IPv4Network(16909060).ip)
-        self.assertRaises(ipaddr.AddressValueError,
-                          ipaddr.IPv4Network, 2**32)
-        self.assertRaises(ipaddr.AddressValueError,
-                          ipaddr.IPv4Network, -1)
-
-        ipv4 = ipaddr.IPNetwork('1.2.3.4')
-        ipv6 = ipaddr.IPNetwork('2001:658:22a:cafe:200:0:0:1')
-        self.assertEqual(ipv4, ipaddr.IPNetwork(int(ipv4)))
-        self.assertEqual(ipv6, ipaddr.IPNetwork(int(ipv6)))
-
-        v6_int = 42540616829182469433547762482097946625
-        self.assertEqual(self.ipv6.ip, ipaddr.IPv6Network(v6_int).ip)
-        self.assertRaises(ipaddr.AddressValueError,
-                          ipaddr.IPv6Network, 2**128)
-        self.assertRaises(ipaddr.AddressValueError,
-                          ipaddr.IPv6Network, -1)
-
-        self.assertEqual(ipaddr.IPNetwork(self.ipv4.ip).version, 4)
-        self.assertEqual(ipaddr.IPNetwork(self.ipv6.ip).version, 6)
-
-    if ipaddr._compat_has_real_bytes: # on python3+
-        def testIpFromPacked(self):
-            ip = ipaddr.IPNetwork
-
-            self.assertEqual(self.ipv4.ip,
-                             ip(_cb('\x01\x02\x03\x04')).ip)
-            self.assertEqual(ip('255.254.253.252'),
-                             ip(_cb('\xff\xfe\xfd\xfc')))
-            self.assertRaises(ValueError, ipaddr.IPNetwork, _cb('\x00' * 3))
-            self.assertRaises(ValueError, ipaddr.IPNetwork, _cb('\x00' * 5))
-            self.assertEqual(self.ipv6.ip,
-                             ip(_cb('\x20\x01\x06\x58\x02\x2a\xca\xfe'
-                               '\x02\x00\x00\x00\x00\x00\x00\x01')).ip)
-            self.assertEqual(ip('ffff:2:3:4:ffff::'),
-                             ip(_cb('\xff\xff\x00\x02\x00\x03\x00\x04' +
-                                   '\xff\xff' + '\x00' * 6)))
-            self.assertEqual(ip('::'),
-                             ip(_cb('\x00' * 16)))
-            self.assertRaises(ValueError, ip, _cb('\x00' * 15))
-            self.assertRaises(ValueError, ip, _cb('\x00' * 17))
-
-    def testGetIp(self):
-        self.assertEqual(int(self.ipv4.ip), 16909060)
-        self.assertEqual(str(self.ipv4.ip), '1.2.3.4')
-        self.assertEqual(str(self.ipv4_hostmask.ip), '10.0.0.1')
-
-        self.assertEqual(int(self.ipv6.ip),
-                         42540616829182469433547762482097946625)
-        self.assertEqual(str(self.ipv6.ip),
-                         '2001:658:22a:cafe:200::1')
-
-    def testGetNetmask(self):
-        self.assertEqual(int(self.ipv4.netmask), 4294967040L)
-        self.assertEqual(str(self.ipv4.netmask), '255.255.255.0')
-        self.assertEqual(str(self.ipv4_hostmask.netmask), '255.0.0.0')
-        self.assertEqual(int(self.ipv6.netmask),
-                         340282366920938463444927863358058659840)
-        self.assertEqual(self.ipv6.prefixlen, 64)
-
-    def testZeroNetmask(self):
-        ipv4_zero_netmask = ipaddr.IPv4Network('1.2.3.4/0')
-        self.assertEqual(int(ipv4_zero_netmask.netmask), 0)
-        self.assertTrue(ipv4_zero_netmask._is_valid_netmask(str(0)))
-
-        ipv6_zero_netmask = ipaddr.IPv6Network('::1/0')
-        self.assertEqual(int(ipv6_zero_netmask.netmask), 0)
-        self.assertTrue(ipv6_zero_netmask._is_valid_netmask(str(0)))
-
-    def testGetBroadcast(self):
-        self.assertEqual(int(self.ipv4.broadcast), 16909311L)
-        self.assertEqual(str(self.ipv4.broadcast), '1.2.3.255')
-
-        self.assertEqual(int(self.ipv6.broadcast),
-                         42540616829182469451850391367731642367)
-        self.assertEqual(str(self.ipv6.broadcast),
-                         '2001:658:22a:cafe:ffff:ffff:ffff:ffff')
-
-    def testGetPrefixlen(self):
-        self.assertEqual(self.ipv4.prefixlen, 24)
-
-        self.assertEqual(self.ipv6.prefixlen, 64)
-
-    def testGetSupernet(self):
-        self.assertEqual(self.ipv4.supernet().prefixlen, 23)
-        self.assertEqual(str(self.ipv4.supernet().network), '1.2.2.0')
-        self.assertEqual(ipaddr.IPv4Network('0.0.0.0/0').supernet(),
-                         ipaddr.IPv4Network('0.0.0.0/0'))
-
-        self.assertEqual(self.ipv6.supernet().prefixlen, 63)
-        self.assertEqual(str(self.ipv6.supernet().network),
-                         '2001:658:22a:cafe::')
-        self.assertEqual(ipaddr.IPv6Network('::0/0').supernet(),
-                         ipaddr.IPv6Network('::0/0'))
-
-    def testGetSupernet3(self):
-        self.assertEqual(self.ipv4.supernet(3).prefixlen, 21)
-        self.assertEqual(str(self.ipv4.supernet(3).network), '1.2.0.0')
-
-        self.assertEqual(self.ipv6.supernet(3).prefixlen, 61)
-        self.assertEqual(str(self.ipv6.supernet(3).network),
-                         '2001:658:22a:caf8::')
-
-    def testGetSupernet4(self):
-        self.assertRaises(ValueError, self.ipv4.supernet, prefixlen_diff=2,
-                          new_prefix=1)
-        self.assertRaises(ValueError, self.ipv4.supernet, new_prefix=25)
-        self.assertEqual(self.ipv4.supernet(prefixlen_diff=2),
-                         self.ipv4.supernet(new_prefix=22))
-
-        self.assertRaises(ValueError, self.ipv6.supernet, prefixlen_diff=2,
-                          new_prefix=1)
-        self.assertRaises(ValueError, self.ipv6.supernet, new_prefix=65)
-        self.assertEqual(self.ipv6.supernet(prefixlen_diff=2),
-                         self.ipv6.supernet(new_prefix=62))
-
-    def testIterSubnets(self):
-        self.assertEqual(self.ipv4.subnet(), list(self.ipv4.iter_subnets()))
-        self.assertEqual(self.ipv6.subnet(), list(self.ipv6.iter_subnets()))
-
-    def testFancySubnetting(self):
-        self.assertEqual(sorted(self.ipv4.subnet(prefixlen_diff=3)),
-                         sorted(self.ipv4.subnet(new_prefix=27)))
-        self.assertRaises(ValueError, self.ipv4.subnet, new_prefix=23)
-        self.assertRaises(ValueError, self.ipv4.subnet,
-                          prefixlen_diff=3, new_prefix=27)
-        self.assertEqual(sorted(self.ipv6.subnet(prefixlen_diff=4)),
-                         sorted(self.ipv6.subnet(new_prefix=68)))
-        self.assertRaises(ValueError, self.ipv6.subnet, new_prefix=63)
-        self.assertRaises(ValueError, self.ipv6.subnet,
-                          prefixlen_diff=4, new_prefix=68)
-
-    def testGetSubnet(self):
-        self.assertEqual(self.ipv4.subnet()[0].prefixlen, 25)
-        self.assertEqual(str(self.ipv4.subnet()[0].network), '1.2.3.0')
-        self.assertEqual(str(self.ipv4.subnet()[1].network), '1.2.3.128')
-
-        self.assertEqual(self.ipv6.subnet()[0].prefixlen, 65)
-
-    def testGetSubnetForSingle32(self):
-        ip = ipaddr.IPv4Network('1.2.3.4/32')
-        subnets1 = [str(x) for x in ip.subnet()]
-        subnets2 = [str(x) for x in ip.subnet(2)]
-        self.assertEqual(subnets1, ['1.2.3.4/32'])
-        self.assertEqual(subnets1, subnets2)
-
-    def testGetSubnetForSingle128(self):
-        ip = ipaddr.IPv6Network('::1/128')
-        subnets1 = [str(x) for x in ip.subnet()]
-        subnets2 = [str(x) for x in ip.subnet(2)]
-        self.assertEqual(subnets1, ['::1/128'])
-        self.assertEqual(subnets1, subnets2)
-
-    def testSubnet2(self):
-        ips = [str(x) for x in self.ipv4.subnet(2)]
-        self.assertEqual(
-            ips,
-            ['1.2.3.0/26', '1.2.3.64/26', '1.2.3.128/26', '1.2.3.192/26'])
-
-        ipsv6 = [str(x) for x in self.ipv6.subnet(2)]
-        self.assertEqual(
-            ipsv6,
-            ['2001:658:22a:cafe::/66',
-             '2001:658:22a:cafe:4000::/66',
-             '2001:658:22a:cafe:8000::/66',
-             '2001:658:22a:cafe:c000::/66'])
-
-    def testSubnetFailsForLargeCidrDiff(self):
-        self.assertRaises(ValueError, self.ipv4.subnet, 9)
-        self.assertRaises(ValueError, self.ipv6.subnet, 65)
-
-    def testSupernetFailsForLargeCidrDiff(self):
-        self.assertRaises(ValueError, self.ipv4.supernet, 25)
-        self.assertRaises(ValueError, self.ipv6.supernet, 65)
-
-    def testSubnetFailsForNegativeCidrDiff(self):
-        self.assertRaises(ValueError, self.ipv4.subnet, -1)
-        self.assertRaises(ValueError, self.ipv6.subnet, -1)
-
-    def testGetNumHosts(self):
-        self.assertEqual(self.ipv4.numhosts, 256)
-        self.assertEqual(self.ipv4.subnet()[0].numhosts, 128)
-        self.assertEqual(self.ipv4.supernet().numhosts, 512)
-
-        self.assertEqual(self.ipv6.numhosts, 18446744073709551616)
-        self.assertEqual(self.ipv6.subnet()[0].numhosts, 9223372036854775808)
-        self.assertEqual(self.ipv6.supernet().numhosts, 36893488147419103232)
-
-    def testContains(self):
-        self.assertTrue(ipaddr.IPv4Network('1.2.3.128/25') in self.ipv4)
-        self.assertFalse(ipaddr.IPv4Network('1.2.4.1/24') in self.ipv4)
-        self.assertTrue(self.ipv4 in self.ipv4)
-        self.assertTrue(self.ipv6 in self.ipv6)
-        # We can test addresses and string as well.
-        addr1 = ipaddr.IPv4Address('1.2.3.37')
-        self.assertTrue(addr1 in self.ipv4)
-        # issue 61, bad network comparison on like-ip'd network objects
-        # with identical broadcast addresses.
-        self.assertFalse(ipaddr.IPv4Network('1.1.0.0/16').__contains__(
-                ipaddr.IPv4Network('1.0.0.0/15')))
-
-    def testBadAddress(self):
-        self.assertRaises(ipaddr.AddressValueError, ipaddr.IPv4Network,
-                          'poop')
-        self.assertRaises(ipaddr.AddressValueError,
-                          ipaddr.IPv4Network, '1.2.3.256')
-
-        self.assertRaises(ipaddr.AddressValueError, ipaddr.IPv6Network,
-                          'poopv6')
-        self.assertRaises(ipaddr.AddressValueError,
-                          ipaddr.IPv4Network, '1.2.3.4/32/24')
-        self.assertRaises(ipaddr.AddressValueError,
-                          ipaddr.IPv4Network, '10/8')
-        self.assertRaises(ipaddr.AddressValueError,
-                          ipaddr.IPv6Network, '10/8')
-
-
-    def testBadNetMask(self):
-        self.assertRaises(ipaddr.NetmaskValueError,
-                          ipaddr.IPv4Network, '1.2.3.4/')
-        self.assertRaises(ipaddr.NetmaskValueError,
-                          ipaddr.IPv4Network, '1.2.3.4/33')
-        self.assertRaises(ipaddr.NetmaskValueError,
-                          ipaddr.IPv4Network, '1.2.3.4/254.254.255.256')
-        self.assertRaises(ipaddr.NetmaskValueError,
-                          ipaddr.IPv4Network, '1.1.1.1/240.255.0.0')
-        self.assertRaises(ipaddr.NetmaskValueError,
-                          ipaddr.IPv6Network, '::1/')
-        self.assertRaises(ipaddr.NetmaskValueError,
-                          ipaddr.IPv6Network, '::1/129')
-
-    def testNth(self):
-        self.assertEqual(str(self.ipv4[5]), '1.2.3.5')
-        self.assertRaises(IndexError, self.ipv4.__getitem__, 256)
-
-        self.assertEqual(str(self.ipv6[5]),
-                         '2001:658:22a:cafe::5')
-
-    def testGetitem(self):
-        # http://code.google.com/p/ipaddr-py/issues/detail?id=15
-        addr = ipaddr.IPv4Network('172.31.255.128/255.255.255.240')
-        self.assertEqual(28, addr.prefixlen)
-        addr_list = list(addr)
-        self.assertEqual('172.31.255.128', str(addr_list[0]))
-        self.assertEqual('172.31.255.128', str(addr[0]))
-        self.assertEqual('172.31.255.143', str(addr_list[-1]))
-        self.assertEqual('172.31.255.143', str(addr[-1]))
-        self.assertEqual(addr_list[-1], addr[-1])
-
-    def testEqual(self):
-        self.assertTrue(self.ipv4 == ipaddr.IPv4Network('1.2.3.4/24'))
-        self.assertFalse(self.ipv4 == ipaddr.IPv4Network('1.2.3.4/23'))
-        self.assertFalse(self.ipv4 == ipaddr.IPv6Network('::1.2.3.4/24'))
-        self.assertFalse(self.ipv4 == '')
-        self.assertFalse(self.ipv4 == [])
-        self.assertFalse(self.ipv4 == 2)
-        self.assertTrue(ipaddr.IPNetwork('1.1.1.1/32') ==
-                        ipaddr.IPAddress('1.1.1.1'))
-        self.assertTrue(ipaddr.IPNetwork('1.1.1.1/24') ==
-                        ipaddr.IPAddress('1.1.1.1'))
-        self.assertFalse(ipaddr.IPNetwork('1.1.1.0/24') ==
-                         ipaddr.IPAddress('1.1.1.1'))
-
-        self.assertTrue(self.ipv6 ==
-            ipaddr.IPv6Network('2001:658:22a:cafe:200::1/64'))
-        self.assertTrue(ipaddr.IPNetwork('::1/128') ==
-                        ipaddr.IPAddress('::1'))
-        self.assertTrue(ipaddr.IPNetwork('::1/127') ==
-                        ipaddr.IPAddress('::1'))
-        self.assertFalse(ipaddr.IPNetwork('::0/127') ==
-                         ipaddr.IPAddress('::1'))
-        self.assertFalse(self.ipv6 ==
-            ipaddr.IPv6Network('2001:658:22a:cafe:200::1/63'))
-        self.assertFalse(self.ipv6 == ipaddr.IPv4Network('1.2.3.4/23'))
-        self.assertFalse(self.ipv6 == '')
-        self.assertFalse(self.ipv6 == [])
-        self.assertFalse(self.ipv6 == 2)
-
-    def testNotEqual(self):
-        self.assertFalse(self.ipv4 != ipaddr.IPv4Network('1.2.3.4/24'))
-        self.assertTrue(self.ipv4 != ipaddr.IPv4Network('1.2.3.4/23'))
-        self.assertTrue(self.ipv4 != ipaddr.IPv6Network('::1.2.3.4/24'))
-        self.assertTrue(self.ipv4 != '')
-        self.assertTrue(self.ipv4 != [])
-        self.assertTrue(self.ipv4 != 2)
-
-        addr2 = ipaddr.IPAddress('2001:658:22a:cafe:200::1')
-        self.assertFalse(self.ipv6 !=
-            ipaddr.IPv6Network('2001:658:22a:cafe:200::1/64'))
-        self.assertTrue(self.ipv6 !=
-            ipaddr.IPv6Network('2001:658:22a:cafe:200::1/63'))
-        self.assertTrue(self.ipv6 != ipaddr.IPv4Network('1.2.3.4/23'))
-        self.assertTrue(self.ipv6 != '')
-        self.assertTrue(self.ipv6 != [])
-        self.assertTrue(self.ipv6 != 2)
-
-    def testSlash32Constructor(self):
-        self.assertEqual(str(ipaddr.IPv4Network('1.2.3.4/255.255.255.255')),
-                          '1.2.3.4/32')
-
-    def testSlash128Constructor(self):
-        self.assertEqual(str(ipaddr.IPv6Network('::1/128')),
-                                  '::1/128')
-
-    def testSlash0Constructor(self):
-        self.assertEqual(str(ipaddr.IPv4Network('1.2.3.4/0.0.0.0')),
-                          '1.2.3.4/0')
-
-    def testCollapsing(self):
-        # test only IP addresses including some duplicates
-        ip1 = ipaddr.IPv4Address('1.1.1.0')
-        ip2 = ipaddr.IPv4Address('1.1.1.1')
-        ip3 = ipaddr.IPv4Address('1.1.1.2')
-        ip4 = ipaddr.IPv4Address('1.1.1.3')
-        ip5 = ipaddr.IPv4Address('1.1.1.4')
-        ip6 = ipaddr.IPv4Address('1.1.1.0')
-        # check that addreses are subsumed properly.
-        collapsed = ipaddr.collapse_address_list([ip1, ip2, ip3, ip4, ip5, ip6])
-        self.assertEqual(collapsed, [ipaddr.IPv4Network('1.1.1.0/30'),
-                                     ipaddr.IPv4Network('1.1.1.4/32')])
-
-        # test a mix of IP addresses and networks including some duplicates
-        ip1 = ipaddr.IPv4Address('1.1.1.0')
-        ip2 = ipaddr.IPv4Address('1.1.1.1')
-        ip3 = ipaddr.IPv4Address('1.1.1.2')
-        ip4 = ipaddr.IPv4Address('1.1.1.3')
-        ip5 = ipaddr.IPv4Network('1.1.1.4/30')
-        ip6 = ipaddr.IPv4Network('1.1.1.4/30')
-        # check that addreses are subsumed properly.
-        collapsed = ipaddr.collapse_address_list([ip5, ip1, ip2, ip3, ip4, ip6])
-        self.assertEqual(collapsed, [ipaddr.IPv4Network('1.1.1.0/29')])
-
-        # test only IP networks
-        ip1 = ipaddr.IPv4Network('1.1.0.0/24')
-        ip2 = ipaddr.IPv4Network('1.1.1.0/24')
-        ip3 = ipaddr.IPv4Network('1.1.2.0/24')
-        ip4 = ipaddr.IPv4Network('1.1.3.0/24')
-        ip5 = ipaddr.IPv4Network('1.1.4.0/24')
-        # stored in no particular order b/c we want CollapseAddr to call [].sort
-        ip6 = ipaddr.IPv4Network('1.1.0.0/22')
-        # check that addreses are subsumed properly.
-        collapsed = ipaddr.collapse_address_list([ip1, ip2, ip3, ip4, ip5, ip6])
-        self.assertEqual(collapsed, [ipaddr.IPv4Network('1.1.0.0/22'),
-                                     ipaddr.IPv4Network('1.1.4.0/24')])
-
-        # test that two addresses are supernet'ed properly
-        collapsed = ipaddr.collapse_address_list([ip1, ip2])
-        self.assertEqual(collapsed, [ipaddr.IPv4Network('1.1.0.0/23')])
-
-        # test same IP networks
-        ip_same1 = ip_same2 = ipaddr.IPv4Network('1.1.1.1/32')
-        self.assertEqual(ipaddr.collapse_address_list([ip_same1, ip_same2]),
-                         [ip_same1])
-
-        # test same IP addresses
-        ip_same1 = ip_same2 = ipaddr.IPv4Address('1.1.1.1')
-        self.assertEqual(ipaddr.collapse_address_list([ip_same1, ip_same2]),
-                         [ipaddr.IPNetwork('1.1.1.1/32')])
-        ip1 = ipaddr.IPv6Network('::2001:1/100')
-        ip2 = ipaddr.IPv6Network('::2002:1/120')
-        ip3 = ipaddr.IPv6Network('::2001:1/96')
-        # test that ipv6 addresses are subsumed properly.
-        collapsed = ipaddr.collapse_address_list([ip1, ip2, ip3])
-        self.assertEqual(collapsed, [ip3])
-
-        # the toejam test
-        ip1 = ipaddr.IPAddress('1.1.1.1')
-        ip2 = ipaddr.IPAddress('::1')
-        self.assertRaises(TypeError, ipaddr.collapse_address_list,
-                          [ip1, ip2])
-
-    def testSummarizing(self):
-        #ip = ipaddr.IPAddress
-        #ipnet = ipaddr.IPNetwork
-        summarize = ipaddr.summarize_address_range
-        ip1 = ipaddr.IPAddress('1.1.1.0')
-        ip2 = ipaddr.IPAddress('1.1.1.255')
-        # test a /24 is sumamrized properly
-        self.assertEqual(summarize(ip1, ip2)[0], ipaddr.IPNetwork('1.1.1.0/24'))
-        # test an  IPv4 range that isn't on a network byte boundary
-        ip2 = ipaddr.IPAddress('1.1.1.8')
-        self.assertEqual(summarize(ip1, ip2), [ipaddr.IPNetwork('1.1.1.0/29'),
-                                               ipaddr.IPNetwork('1.1.1.8')])
-
-        ip1 = ipaddr.IPAddress('1::')
-        ip2 = ipaddr.IPAddress('1:ffff:ffff:ffff:ffff:ffff:ffff:ffff')
-        # test a IPv6 is sumamrized properly
-        self.assertEqual(summarize(ip1, ip2)[0], ipaddr.IPNetwork('1::/16'))
-        # test an IPv6 range that isn't on a network byte boundary
-        ip2 = ipaddr.IPAddress('2::')
-        self.assertEqual(summarize(ip1, ip2), [ipaddr.IPNetwork('1::/16'),
-                                               ipaddr.IPNetwork('2::/128')])
-
-        # test exception raised when first is greater than last
-        self.assertRaises(ValueError, summarize, ipaddr.IPAddress('1.1.1.0'),
-            ipaddr.IPAddress('1.1.0.0'))
-        # test exception raised when first and last aren't IP addresses
-        self.assertRaises(TypeError, summarize,
-                          ipaddr.IPNetwork('1.1.1.0'),
-                          ipaddr.IPNetwork('1.1.0.0'))
-        self.assertRaises(TypeError, summarize,
-            ipaddr.IPNetwork('1.1.1.0'), ipaddr.IPNetwork('1.1.0.0'))
-        # test exception raised when first and last are not same version
-        self.assertRaises(TypeError, summarize, ipaddr.IPAddress('::'),
-            ipaddr.IPNetwork('1.1.0.0'))
-
-    def testAddressComparison(self):
-        self.assertTrue(ipaddr.IPAddress('1.1.1.1') <=
-                        ipaddr.IPAddress('1.1.1.1'))
-        self.assertTrue(ipaddr.IPAddress('1.1.1.1') <=
-                        ipaddr.IPAddress('1.1.1.2'))
-        self.assertTrue(ipaddr.IPAddress('::1') <= ipaddr.IPAddress('::1'))
-        self.assertTrue(ipaddr.IPAddress('::1') <= ipaddr.IPAddress('::2'))
-
-    def testNetworkComparison(self):
-        # ip1 and ip2 have the same network address
-        ip1 = ipaddr.IPv4Network('1.1.1.0/24')
-        ip2 = ipaddr.IPv4Network('1.1.1.1/24')
-        ip3 = ipaddr.IPv4Network('1.1.2.0/24')
-
-        self.assertTrue(ip1 < ip3)
-        self.assertTrue(ip3 > ip2)
-
-        self.assertEqual(ip1.compare_networks(ip2), 0)
-        self.assertTrue(ip1._get_networks_key() == ip2._get_networks_key())
-        self.assertEqual(ip1.compare_networks(ip3), -1)
-        self.assertTrue(ip1._get_networks_key() < ip3._get_networks_key())
-
-        ip1 = ipaddr.IPv6Network('2001::2000/96')
-        ip2 = ipaddr.IPv6Network('2001::2001/96')
-        ip3 = ipaddr.IPv6Network('2001:ffff::2000/96')
-
-        self.assertTrue(ip1 < ip3)
-        self.assertTrue(ip3 > ip2)
-        self.assertEqual(ip1.compare_networks(ip2), 0)
-        self.assertTrue(ip1._get_networks_key() == ip2._get_networks_key())
-        self.assertEqual(ip1.compare_networks(ip3), -1)
-        self.assertTrue(ip1._get_networks_key() < ip3._get_networks_key())
-
-        # Test comparing different protocols.
-        # Should always raise a TypeError.
-        ipv6 = ipaddr.IPv6Network('::/0')
-        ipv4 = ipaddr.IPv4Network('0.0.0.0/0')
-        self.assertRaises(TypeError, ipv4.__lt__, ipv6)
-        self.assertRaises(TypeError, ipv4.__gt__, ipv6)
-        self.assertRaises(TypeError, ipv6.__lt__, ipv4)
-        self.assertRaises(TypeError, ipv6.__gt__, ipv4)
-
-        # Regression test for issue 19.
-        ip1 = ipaddr.IPNetwork('10.1.2.128/25')
-        self.assertFalse(ip1 < ip1)
-        self.assertFalse(ip1 > ip1)
-        ip2 = ipaddr.IPNetwork('10.1.3.0/24')
-        self.assertTrue(ip1 < ip2)
-        self.assertFalse(ip2 < ip1)
-        self.assertFalse(ip1 > ip2)
-        self.assertTrue(ip2 > ip1)
-        ip3 = ipaddr.IPNetwork('10.1.3.0/25')
-        self.assertTrue(ip2 < ip3)
-        self.assertFalse(ip3 < ip2)
-        self.assertFalse(ip2 > ip3)
-        self.assertTrue(ip3 > ip2)
-
-        # Regression test for issue 28.
-        ip1 = ipaddr.IPNetwork('10.10.10.0/31')
-        ip2 = ipaddr.IPNetwork('10.10.10.0')
-        ip3 = ipaddr.IPNetwork('10.10.10.2/31')
-        ip4 = ipaddr.IPNetwork('10.10.10.2')
-        sorted = [ip1, ip2, ip3, ip4]
-        unsorted = [ip2, ip4, ip1, ip3]
-        unsorted.sort()
-        self.assertEqual(sorted, unsorted)
-        unsorted = [ip4, ip1, ip3, ip2]
-        unsorted.sort()
-        self.assertEqual(sorted, unsorted)
-        self.assertRaises(TypeError, ip1.__lt__, ipaddr.IPAddress('10.10.10.0'))
-        self.assertRaises(TypeError, ip2.__lt__, ipaddr.IPAddress('10.10.10.0'))
-
-        # <=, >=
-        self.assertTrue(ipaddr.IPNetwork('1.1.1.1') <=
-                        ipaddr.IPNetwork('1.1.1.1'))
-        self.assertTrue(ipaddr.IPNetwork('1.1.1.1') <=
-                        ipaddr.IPNetwork('1.1.1.2'))
-        self.assertFalse(ipaddr.IPNetwork('1.1.1.2') <=
-                        ipaddr.IPNetwork('1.1.1.1'))
-        self.assertTrue(ipaddr.IPNetwork('::1') <= ipaddr.IPNetwork('::1'))
-        self.assertTrue(ipaddr.IPNetwork('::1') <= ipaddr.IPNetwork('::2'))
-        self.assertFalse(ipaddr.IPNetwork('::2') <= ipaddr.IPNetwork('::1'))
-
-    def testStrictNetworks(self):
-        self.assertRaises(ValueError, ipaddr.IPNetwork, '192.168.1.1/24',
-                          strict=True)
-        self.assertRaises(ValueError, ipaddr.IPNetwork, '::1/120', strict=True)
-
-    def testOverlaps(self):
-        other = ipaddr.IPv4Network('1.2.3.0/30')
-        other2 = ipaddr.IPv4Network('1.2.2.0/24')
-        other3 = ipaddr.IPv4Network('1.2.2.64/26')
-        self.assertTrue(self.ipv4.overlaps(other))
-        self.assertFalse(self.ipv4.overlaps(other2))
-        self.assertTrue(other2.overlaps(other3))
-
-    def testEmbeddedIpv4(self):
-        ipv4_string = '192.168.0.1'
-        ipv4 = ipaddr.IPv4Network(ipv4_string)
-        v4compat_ipv6 = ipaddr.IPv6Network('::%s' % ipv4_string)
-        self.assertEqual(int(v4compat_ipv6.ip), int(ipv4.ip))
-        v4mapped_ipv6 = ipaddr.IPv6Network('::ffff:%s' % ipv4_string)
-        self.assertNotEqual(v4mapped_ipv6.ip, ipv4.ip)
-        self.assertRaises(ipaddr.AddressValueError, ipaddr.IPv6Network,
-                          '2001:1.1.1.1:1.1.1.1')
-
-    # Issue 67: IPv6 with embedded IPv4 address not recognized.
-    def testIPv6AddressTooLarge(self):
-        # RFC4291 2.5.5.2
-        self.assertEqual(ipaddr.IPAddress('::FFFF:192.0.2.1'),
-                          ipaddr.IPAddress('::FFFF:c000:201'))
-        # RFC4291 2.2 (part 3) x::d.d.d.d 
-        self.assertEqual(ipaddr.IPAddress('FFFF::192.0.2.1'),
-                          ipaddr.IPAddress('FFFF::c000:201'))
-
-    def testIPVersion(self):
-        self.assertEqual(self.ipv4.version, 4)
-        self.assertEqual(self.ipv6.version, 6)
-
-    def testMaxPrefixLength(self):
-        self.assertEqual(self.ipv4.max_prefixlen, 32)
-        self.assertEqual(self.ipv6.max_prefixlen, 128)
-
-    def testPacked(self):
-        self.assertEqual(self.ipv4.packed,
-                         _cb('\x01\x02\x03\x04'))
-        self.assertEqual(ipaddr.IPv4Network('255.254.253.252').packed,
-                         _cb('\xff\xfe\xfd\xfc'))
-        self.assertEqual(self.ipv6.packed,
-                         _cb('\x20\x01\x06\x58\x02\x2a\xca\xfe'
-                             '\x02\x00\x00\x00\x00\x00\x00\x01'))
-        self.assertEqual(ipaddr.IPv6Network('ffff:2:3:4:ffff::').packed,
-                         _cb('\xff\xff\x00\x02\x00\x03\x00\x04\xff\xff'
-                            + '\x00' * 6))
-        self.assertEqual(ipaddr.IPv6Network('::1:0:0:0:0').packed,
-                         _cb('\x00' * 6 + '\x00\x01' + '\x00' * 8))
-
-    def testIpStrFromPrefixlen(self):
-        ipv4 = ipaddr.IPv4Network('1.2.3.4/24')
-        self.assertEqual(ipv4._ip_string_from_prefix(), '255.255.255.0')
-        self.assertEqual(ipv4._ip_string_from_prefix(28), '255.255.255.240')
-
-    def testIpType(self):
-        ipv4net = ipaddr.IPNetwork('1.2.3.4')
-        ipv4addr = ipaddr.IPAddress('1.2.3.4')
-        ipv6net = ipaddr.IPNetwork('::1.2.3.4')
-        ipv6addr = ipaddr.IPAddress('::1.2.3.4')
-        self.assertEqual(ipaddr.IPv4Network, type(ipv4net))
-        self.assertEqual(ipaddr.IPv4Address, type(ipv4addr))
-        self.assertEqual(ipaddr.IPv6Network, type(ipv6net))
-        self.assertEqual(ipaddr.IPv6Address, type(ipv6addr))
-
-    def testReservedIpv4(self):
-        # test networks
-        self.assertEqual(True, ipaddr.IPNetwork('224.1.1.1/31').is_multicast)
-        self.assertEqual(False, ipaddr.IPNetwork('240.0.0.0').is_multicast)
-
-        self.assertEqual(True, ipaddr.IPNetwork('192.168.1.1/17').is_private)
-        self.assertEqual(False, ipaddr.IPNetwork('192.169.0.0').is_private)
-        self.assertEqual(True, ipaddr.IPNetwork('10.255.255.255').is_private)
-        self.assertEqual(False, ipaddr.IPNetwork('11.0.0.0').is_private)
-        self.assertEqual(True, ipaddr.IPNetwork('172.31.255.255').is_private)
-        self.assertEqual(False, ipaddr.IPNetwork('172.32.0.0').is_private)
-
-        self.assertEqual(True,
-                          ipaddr.IPNetwork('169.254.100.200/24').is_link_local)
-        self.assertEqual(False,
-                          ipaddr.IPNetwork('169.255.100.200/24').is_link_local)
-
-        self.assertEqual(True,
-                          ipaddr.IPNetwork('127.100.200.254/32').is_loopback)
-        self.assertEqual(True, ipaddr.IPNetwork('127.42.0.0/16').is_loopback)
-        self.assertEqual(False, ipaddr.IPNetwork('128.0.0.0').is_loopback)
-
-        # test addresses
-        self.assertEqual(True, ipaddr.IPAddress('224.1.1.1').is_multicast)
-        self.assertEqual(False, ipaddr.IPAddress('240.0.0.0').is_multicast)
-
-        self.assertEqual(True, ipaddr.IPAddress('192.168.1.1').is_private)
-        self.assertEqual(False, ipaddr.IPAddress('192.169.0.0').is_private)
-        self.assertEqual(True, ipaddr.IPAddress('10.255.255.255').is_private)
-        self.assertEqual(False, ipaddr.IPAddress('11.0.0.0').is_private)
-        self.assertEqual(True, ipaddr.IPAddress('172.31.255.255').is_private)
-        self.assertEqual(False, ipaddr.IPAddress('172.32.0.0').is_private)
-
-        self.assertEqual(True,
-                          ipaddr.IPAddress('169.254.100.200').is_link_local)
-        self.assertEqual(False,
-                          ipaddr.IPAddress('169.255.100.200').is_link_local)
-
-        self.assertEqual(True,
-                          ipaddr.IPAddress('127.100.200.254').is_loopback)
-        self.assertEqual(True, ipaddr.IPAddress('127.42.0.0').is_loopback)
-        self.assertEqual(False, ipaddr.IPAddress('128.0.0.0').is_loopback)
-        self.assertEqual(True, ipaddr.IPNetwork('0.0.0.0').is_unspecified)
-
-    def testReservedIpv6(self):
-
-        self.assertEqual(True, ipaddr.IPNetwork('ffff::').is_multicast)
-        self.assertEqual(True, ipaddr.IPNetwork(2**128-1).is_multicast)
-        self.assertEqual(True, ipaddr.IPNetwork('ff00::').is_multicast)
-        self.assertEqual(False, ipaddr.IPNetwork('fdff::').is_multicast)
-
-        self.assertEqual(True, ipaddr.IPNetwork('fecf::').is_site_local)
-        self.assertEqual(True, ipaddr.IPNetwork(
-                'feff:ffff:ffff:ffff::').is_site_local)
-        self.assertEqual(False, ipaddr.IPNetwork('fbf:ffff::').is_site_local)
-        self.assertEqual(False, ipaddr.IPNetwork('ff00::').is_site_local)
-
-        self.assertEqual(True, ipaddr.IPNetwork('fc00::').is_private)
-        self.assertEqual(True, ipaddr.IPNetwork(
-                'fc00:ffff:ffff:ffff::').is_private)
-        self.assertEqual(False, ipaddr.IPNetwork('fbff:ffff::').is_private)
-        self.assertEqual(False, ipaddr.IPNetwork('fe00::').is_private)
-
-        self.assertEqual(True, ipaddr.IPNetwork('fea0::').is_link_local)
-        self.assertEqual(True, ipaddr.IPNetwork('febf:ffff::').is_link_local)
-        self.assertEqual(False, ipaddr.IPNetwork('fe7f:ffff::').is_link_local)
-        self.assertEqual(False, ipaddr.IPNetwork('fec0::').is_link_local)
-
-        self.assertEqual(True, ipaddr.IPNetwork('0:0::0:01').is_loopback)
-        self.assertEqual(False, ipaddr.IPNetwork('::1/127').is_loopback)
-        self.assertEqual(False, ipaddr.IPNetwork('::').is_loopback)
-        self.assertEqual(False, ipaddr.IPNetwork('::2').is_loopback)
-
-        self.assertEqual(True, ipaddr.IPNetwork('0::0').is_unspecified)
-        self.assertEqual(False, ipaddr.IPNetwork('::1').is_unspecified)
-        self.assertEqual(False, ipaddr.IPNetwork('::/127').is_unspecified)
-
-        # test addresses
-        self.assertEqual(True, ipaddr.IPAddress('ffff::').is_multicast)
-        self.assertEqual(True, ipaddr.IPAddress(2**128-1).is_multicast)
-        self.assertEqual(True, ipaddr.IPAddress('ff00::').is_multicast)
-        self.assertEqual(False, ipaddr.IPAddress('fdff::').is_multicast)
-
-        self.assertEqual(True, ipaddr.IPAddress('fecf::').is_site_local)
-        self.assertEqual(True, ipaddr.IPAddress(
-                'feff:ffff:ffff:ffff::').is_site_local)
-        self.assertEqual(False, ipaddr.IPAddress('fbf:ffff::').is_site_local)
-        self.assertEqual(False, ipaddr.IPAddress('ff00::').is_site_local)
-
-        self.assertEqual(True, ipaddr.IPAddress('fc00::').is_private)
-        self.assertEqual(True, ipaddr.IPAddress(
-                'fc00:ffff:ffff:ffff::').is_private)
-        self.assertEqual(False, ipaddr.IPAddress('fbff:ffff::').is_private)
-        self.assertEqual(False, ipaddr.IPAddress('fe00::').is_private)
-
-        self.assertEqual(True, ipaddr.IPAddress('fea0::').is_link_local)
-        self.assertEqual(True, ipaddr.IPAddress('febf:ffff::').is_link_local)
-        self.assertEqual(False, ipaddr.IPAddress('fe7f:ffff::').is_link_local)
-        self.assertEqual(False, ipaddr.IPAddress('fec0::').is_link_local)
-
-        self.assertEqual(True, ipaddr.IPAddress('0:0::0:01').is_loopback)
-        self.assertEqual(True, ipaddr.IPAddress('::1').is_loopback)
-        self.assertEqual(False, ipaddr.IPAddress('::2').is_loopback)
-
-        self.assertEqual(True, ipaddr.IPAddress('0::0').is_unspecified)
-        self.assertEqual(False, ipaddr.IPAddress('::1').is_unspecified)
-
-        # some generic IETF reserved addresses
-        self.assertEqual(True, ipaddr.IPAddress('100::').is_reserved)
-        self.assertEqual(True, ipaddr.IPNetwork('4000::1/128').is_reserved)
-
-    def testIpv4Mapped(self):
-        self.assertEqual(ipaddr.IPAddress('::ffff:192.168.1.1').ipv4_mapped,
-                         ipaddr.IPAddress('192.168.1.1'))
-        self.assertEqual(ipaddr.IPAddress('::c0a8:101').ipv4_mapped, None)
-        self.assertEqual(ipaddr.IPAddress('::ffff:c0a8:101').ipv4_mapped,
-                         ipaddr.IPAddress('192.168.1.1'))
-
-    def testAddrExclude(self):
-        addr1 = ipaddr.IPNetwork('10.1.1.0/24')
-        addr2 = ipaddr.IPNetwork('10.1.1.0/26')
-        addr3 = ipaddr.IPNetwork('10.2.1.0/24')
-        addr4 = ipaddr.IPAddress('10.1.1.0')
-        self.assertEqual(addr1.address_exclude(addr2),
-                         [ipaddr.IPNetwork('10.1.1.64/26'),
-                          ipaddr.IPNetwork('10.1.1.128/25')])
-        self.assertRaises(ValueError, addr1.address_exclude, addr3)
-        self.assertRaises(TypeError, addr1.address_exclude, addr4)
-        self.assertEqual(addr1.address_exclude(addr1), [])
-
-    def testHash(self):
-        self.assertEqual(hash(ipaddr.IPNetwork('10.1.1.0/24')),
-                          hash(ipaddr.IPNetwork('10.1.1.0/24')))
-        self.assertEqual(hash(ipaddr.IPAddress('10.1.1.0')),
-                          hash(ipaddr.IPAddress('10.1.1.0')))
-        # i70
-        self.assertEqual(hash(ipaddr.IPAddress('1.2.3.4')),
-                          hash(ipaddr.IPAddress(
-                    long(ipaddr.IPAddress('1.2.3.4')._ip))))
-        ip1 = ipaddr.IPAddress('10.1.1.0')
-        ip2 = ipaddr.IPAddress('1::')
-        dummy = {}
-        dummy[self.ipv4] = None
-        dummy[self.ipv6] = None
-        dummy[ip1] = None
-        dummy[ip2] = None
-        self.assertTrue(self.ipv4 in dummy)
-        self.assertTrue(ip2 in dummy)
-
-    def testCopyConstructor(self):
-        addr1 = ipaddr.IPNetwork('10.1.1.0/24')
-        addr2 = ipaddr.IPNetwork(addr1)
-        addr3 = ipaddr.IPNetwork('2001:658:22a:cafe:200::1/64')
-        addr4 = ipaddr.IPNetwork(addr3)
-        addr5 = ipaddr.IPv4Address('1.1.1.1')
-        addr6 = ipaddr.IPv6Address('2001:658:22a:cafe:200::1')
-
-        self.assertEqual(addr1, addr2)
-        self.assertEqual(addr3, addr4)
-        self.assertEqual(addr5, ipaddr.IPv4Address(addr5))
-        self.assertEqual(addr6, ipaddr.IPv6Address(addr6))
-
-    def testCompressIPv6Address(self):
-        test_addresses = {
-            '1:2:3:4:5:6:7:8': '1:2:3:4:5:6:7:8/128',
-            '2001:0:0:4:0:0:0:8': '2001:0:0:4::8/128',
-            '2001:0:0:4:5:6:7:8': '2001::4:5:6:7:8/128',
-            '2001:0:3:4:5:6:7:8': '2001:0:3:4:5:6:7:8/128',
-            '2001:0:3:4:5:6:7:8': '2001:0:3:4:5:6:7:8/128',
-            '0:0:3:0:0:0:0:ffff': '0:0:3::ffff/128',
-            '0:0:0:4:0:0:0:ffff': '::4:0:0:0:ffff/128',
-            '0:0:0:0:5:0:0:ffff': '::5:0:0:ffff/128',
-            '1:0:0:4:0:0:7:8': '1::4:0:0:7:8/128',
-            '0:0:0:0:0:0:0:0': '::/128',
-            '0:0:0:0:0:0:0:0/0': '::/0',
-            '0:0:0:0:0:0:0:1': '::1/128',
-            '2001:0658:022a:cafe:0000:0000:0000:0000/66':
-            '2001:658:22a:cafe::/66',
-            '::1.2.3.4': '::102:304/128',
-            '1:2:3:4:5:ffff:1.2.3.4': '1:2:3:4:5:ffff:102:304/128',
-            '::7:6:5:4:3:2:1': '0:7:6:5:4:3:2:1/128',
-            '::7:6:5:4:3:2:0': '0:7:6:5:4:3:2:0/128',
-            '7:6:5:4:3:2:1::': '7:6:5:4:3:2:1:0/128',
-            '0:6:5:4:3:2:1::': '0:6:5:4:3:2:1:0/128',
-            }
-        for uncompressed, compressed in test_addresses.items():
-            self.assertEqual(compressed, str(ipaddr.IPv6Network(uncompressed)))
-
-    def testExplodeShortHandIpStr(self):
-        addr1 = ipaddr.IPv6Network('2001::1')
-        addr2 = ipaddr.IPv6Address('2001:0:5ef5:79fd:0:59d:a0e5:ba1')
-        self.assertEqual('2001:0000:0000:0000:0000:0000:0000:0001',
-                         addr1._explode_shorthand_ip_string(str(addr1.ip)))
-        self.assertEqual('0000:0000:0000:0000:0000:0000:0000:0001',
-                         ipaddr.IPv6Network('::1/128').exploded)
-        # issue 77
-        self.assertEqual('2001:0000:5ef5:79fd:0000:059d:a0e5:0ba1',
-                         addr2.exploded)
-
-    def testIntRepresentation(self):
-        self.assertEqual(16909060, int(self.ipv4))
-        self.assertEqual(42540616829182469433547762482097946625, int(self.ipv6))
-
-    def testHexRepresentation(self):
-        self.assertEqual(hex(0x1020304),
-                         hex(self.ipv4))
-
-        self.assertEqual(hex(0x20010658022ACAFE0200000000000001),
-                         hex(self.ipv6))
-
-    # backwards compatibility
-    def testBackwardsCompability(self):
-        self.assertEqual(ipaddr.CollapseAddrList(
-            [ipaddr.IPNetwork('1.1.0.0/24'), ipaddr.IPNetwork('1.1.1.0/24')]),
-                         [ipaddr.IPNetwork('1.1.0.0/23')])
-
-        self.assertEqual(ipaddr.IPNetwork('::42:0/112').AddressExclude(
-            ipaddr.IPNetwork('::42:8000/113')),
-                         [ipaddr.IPNetwork('::42:0/113')])
-
-        self.assertTrue(ipaddr.IPNetwork('1::/8').CompareNetworks(
-            ipaddr.IPNetwork('2::/9')) < 0)
-
-        self.assertEqual(ipaddr.IPNetwork('1::/16').Contains(
-            ipaddr.IPNetwork('2::/16')), False)
-
-        self.assertEqual(ipaddr.IPNetwork('0.0.0.0/0').Subnet(),
-                         [ipaddr.IPNetwork('0.0.0.0/1'),
-                          ipaddr.IPNetwork('128.0.0.0/1')])
-        self.assertEqual(ipaddr.IPNetwork('::/127').Subnet(),
-                         [ipaddr.IPNetwork('::/128'),
-                          ipaddr.IPNetwork('::1/128')])
-
-        self.assertEqual(ipaddr.IPNetwork('1.0.0.0/32').Supernet(),
-                         ipaddr.IPNetwork('1.0.0.0/31'))
-        self.assertEqual(ipaddr.IPNetwork('::/121').Supernet(),
-                         ipaddr.IPNetwork('::/120'))
-
-        self.assertEqual(ipaddr.IPNetwork('10.0.0.2').IsRFC1918(), True)
-        self.assertEqual(ipaddr.IPNetwork('10.0.0.0').IsMulticast(), False)
-        self.assertEqual(ipaddr.IPNetwork('127.255.255.255').IsLoopback(), True)
-        self.assertEqual(ipaddr.IPNetwork('169.255.255.255').IsLinkLocal(),
-                         False)
-
-    def testForceVersion(self):
-        self.assertEqual(ipaddr.IPNetwork(1).version, 4)
-        self.assertEqual(ipaddr.IPNetwork(1, version=6).version, 6)
-
-    def testWithStar(self):
-        self.assertEqual(str(self.ipv4.with_prefixlen), "1.2.3.4/24")
-        self.assertEqual(str(self.ipv4.with_netmask), "1.2.3.4/255.255.255.0")
-        self.assertEqual(str(self.ipv4.with_hostmask), "1.2.3.4/0.0.0.255")
-
-        self.assertEqual(str(self.ipv6.with_prefixlen),
-                         '2001:658:22a:cafe:200::1/64')
-        # rfc3513 sec 2.3 says that ipv6 only uses cidr notation for
-        # subnets
-        self.assertEqual(str(self.ipv6.with_netmask),
-                         '2001:658:22a:cafe:200::1/64')
-        # this probably don't make much sense, but it's included for
-        # compatibility with ipv4
-        self.assertEqual(str(self.ipv6.with_hostmask),
-                         '2001:658:22a:cafe:200::1/::ffff:ffff:ffff:ffff')
-
-    def testNetworkElementCaching(self):
-        # V4 - make sure we're empty
-        self.assertFalse(self.ipv4._cache.has_key('network'))
-        self.assertFalse(self.ipv4._cache.has_key('broadcast'))
-        self.assertFalse(self.ipv4._cache.has_key('hostmask'))
-
-        # V4 - populate and test
-        self.assertEqual(self.ipv4.network, ipaddr.IPv4Address('1.2.3.0'))
-        self.assertEqual(self.ipv4.broadcast, ipaddr.IPv4Address('1.2.3.255'))
-        self.assertEqual(self.ipv4.hostmask, ipaddr.IPv4Address('0.0.0.255'))
-
-        # V4 - check we're cached
-        self.assertTrue(self.ipv4._cache.has_key('network'))
-        self.assertTrue(self.ipv4._cache.has_key('broadcast'))
-        self.assertTrue(self.ipv4._cache.has_key('hostmask'))
-
-        # V6 - make sure we're empty
-        self.assertFalse(self.ipv6._cache.has_key('network'))
-        self.assertFalse(self.ipv6._cache.has_key('broadcast'))
-        self.assertFalse(self.ipv6._cache.has_key('hostmask'))
-
-        # V6 - populate and test
-        self.assertEqual(self.ipv6.network,
-                         ipaddr.IPv6Address('2001:658:22a:cafe::'))
-        self.assertEqual(self.ipv6.broadcast, ipaddr.IPv6Address(
-            '2001:658:22a:cafe:ffff:ffff:ffff:ffff'))
-        self.assertEqual(self.ipv6.hostmask,
-                         ipaddr.IPv6Address('::ffff:ffff:ffff:ffff'))
-
-        # V6 - check we're cached
-        self.assertTrue(self.ipv6._cache.has_key('network'))
-        self.assertTrue(self.ipv6._cache.has_key('broadcast'))
-        self.assertTrue(self.ipv6._cache.has_key('hostmask'))
-
-    def testTeredo(self):
-        # stolen from wikipedia
-        server = ipaddr.IPv4Address('65.54.227.120')
-        client = ipaddr.IPv4Address('192.0.2.45')
-        teredo_addr = '2001:0000:4136:e378:8000:63bf:3fff:fdd2'
-        self.assertEqual((server, client),
-                         ipaddr.IPAddress(teredo_addr).teredo)
-        bad_addr = '2000::4136:e378:8000:63bf:3fff:fdd2'
-        self.assertFalse(ipaddr.IPAddress(bad_addr).teredo)
-        bad_addr = '2001:0001:4136:e378:8000:63bf:3fff:fdd2'
-        self.assertFalse(ipaddr.IPAddress(bad_addr).teredo)
-
-        # i77
-        teredo_addr = ipaddr.IPv6Address('2001:0:5ef5:79fd:0:59d:a0e5:ba1')
-        self.assertEqual((ipaddr.IPv4Address('94.245.121.253'),
-                          ipaddr.IPv4Address('95.26.244.94')),
-                         teredo_addr.teredo)
-
-
-    def testsixtofour(self):
-        sixtofouraddr = ipaddr.IPAddress('2002:ac1d:2d64::1')
-        bad_addr = ipaddr.IPAddress('2000:ac1d:2d64::1')
-        self.assertEqual(ipaddr.IPv4Address('172.29.45.100'),
-                         sixtofouraddr.sixtofour)
-        self.assertFalse(bad_addr.sixtofour)
-
-
-if __name__ == '__main__':
-    unittest.main()
diff --git a/contrib/ipaddr-py/setup.py b/contrib/ipaddr-py/setup.py
deleted file mode 100755
index 33564320e45..00000000000
--- a/contrib/ipaddr-py/setup.py
+++ /dev/null
@@ -1,36 +0,0 @@
-#!/usr/bin/python
-#
-# Copyright 2008 Google Inc.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#      http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-from distutils.core import setup
-
-import ipaddr
-
-
-setup(name='ipaddr',
-      maintainer='Google',
-      maintainer_email='ipaddr-py-dev@googlegroups.com',
-      version=ipaddr.__version__,
-      url='http://code.google.com/p/ipaddr-py/',
-      license='Apache License, Version 2.0',
-      classifiers=[
-          'Development Status :: 5 - Production/Stable',
-          'Intended Audience :: Developers',
-          'License :: OSI Approved :: Apache Software License',
-          'Operating System :: OS Independent',
-          'Topic :: Internet',
-          'Topic :: Software Development :: Libraries',
-          'Topic :: System :: Networking'],
-      py_modules=['ipaddr'])
diff --git a/contrib/ipaddr-py/test-2to3.sh b/contrib/ipaddr-py/test-2to3.sh
deleted file mode 100755
index 408d665bcc2..00000000000
--- a/contrib/ipaddr-py/test-2to3.sh
+++ /dev/null
@@ -1,15 +0,0 @@
-#!/bin/sh
-
-# Converts the python2 ipaddr files to python3 and runs the unit tests
-# with both python versions.
-
-mkdir -p 2to3output && \
-cp -f *.py 2to3output && \
-( cd 2to3output && 2to3 . | patch -p0 ) && \
-py3version=$(python3 --version 2>&1) && \
-echo -e "\nTesting with ${py3version}" && \
-python3 2to3output/ipaddr_test.py && \
-rm -r 2to3output && \
-pyversion=$(python --version 2>&1) && \
-echo -e "\nTesting with ${pyversion}" && \
-./ipaddr_test.py
diff --git a/contrib/macfuse/mount_darwin.c b/contrib/macfuse/mount_darwin.c
index 10eff204bc6..d1d1c34e761 100644
--- a/contrib/macfuse/mount_darwin.c
+++ b/contrib/macfuse/mount_darwin.c
@@ -34,16 +34,15 @@
 #include "fuse_param.h"
 #include "fuse_ioctl.h"
 
-#include "glusterfs.h"
-#include "logging.h"
-#include "common-utils.h"
+#include "glusterfs/glusterfs.h"
+#include "glusterfs/logging.h"
+#include "glusterfs/common-utils.h"
 
 #define GFFUSE_LOGERR(...) \
         gf_log ("glusterfs-fuse", GF_LOG_ERROR, ## __VA_ARGS__)
 
 int
-gf_fuse_mount (const char *mountpoint, char *fsname,
-               unsigned long mountflags, char *mnt_param,
+gf_fuse_mount (const char *mountpoint, char *fsname, char *mnt_param,
                pid_t *mnt_pid, int status_fd) /* Not used on OS X */
 {
         int fd       = 0;
@@ -95,7 +94,7 @@ gf_fuse_mount (const char *mountpoint, char *fsname,
         }
 
         /* sysctlbyname() includes the trailing '\0' in version_len */
-        version_len_desired = strlen("2.x.y") + 1;
+        version_len_desired = sizeof ("2.x.y");
 
         if (version_len != version_len_desired) {
                 gf_log ("glusterfs-fuse", GF_LOG_ERROR,
diff --git a/contrib/mount/mntent.c b/contrib/mount/mntent.c
index e9b448845a7..9a7e5f39bdb 100644
--- a/contrib/mount/mntent.c
+++ b/contrib/mount/mntent.c
@@ -36,6 +36,7 @@
  */
 
 #if !defined(GF_LINUX_HOST_OS)
+
 #include <stdlib.h>
 #include <string.h>
 #include <sys/param.h>
@@ -49,166 +50,209 @@ typedef struct statvfs gf_statfs_t;
 typedef struct statfs gf_statfs_t;
 #endif
 
-static int pos = -1;
-static int mntsize = -1;
-static struct mntent _mntent;
+typedef struct _mntent_state {
+        struct mntent  mntent;
+        gf_statfs_t   *statfs;
+        int            count;
+        int            pos;
+        /* A buffer big enough to store all defined flags as a string.
+         * Increase it if necessary when more flags are defined. */
+        char           buf[256];
+} mntent_state_t;
+
+typedef struct _mntflag {
+        unsigned long  value;
+        const char    *on;
+        const char    *off;
+} mntflag_t;
+
+static mntflag_t mntflags[] = {
+        { MNT_RDONLY,      "ro",          "rw" },
+        { MNT_SYNCHRONOUS, "sync",        NULL },
+        { MNT_NOEXEC,      "noexec",      NULL },
+        { MNT_NOSUID,      "nosuid",      NULL },
+#if !defined(__FreeBSD__)
+        { MNT_NODEV,       "nodev",       NULL },
+#endif /* __FreeBSD__ */
+        { MNT_UNION,       "union",       NULL },
+        { MNT_ASYNC,       "async",       NULL },
+#if !defined(GF_DARWIN_HOST_OS)
+        { MNT_NOATIME,     "noatime",     NULL },
+#if !defined(__NetBSD__)
+        { MNT_NOCLUSTERR,  "noclusterr",  NULL },
+        { MNT_NOCLUSTERW,  "noclusterw",  NULL },
+        { MNT_NOSYMFOLLOW, "nosymfollow", NULL },
+        { MNT_SUIDDIR,     "suiddir",     NULL },
+#endif /* !__NetBSD__ */
+#endif /* !GF_DARWIN_HOST_OS */
+        { 0,               NULL,          NULL }
+};
 
 char *
 hasmntopt (const struct mntent *mnt, const char *option)
 {
-        int found;
         char *opt, *optbuf;
+        int len;
 
         optbuf = strdup(mnt->mnt_opts);
-        found = 0;
-        for (opt = optbuf; (opt = strtok(opt, " ")) != NULL; opt = NULL) {
-                if (!strcasecmp(opt, option)) {
-                        opt = opt - optbuf + mnt->mnt_opts;
-                        free (optbuf);
-                        return (opt);
+        if (optbuf == NULL) {
+                return NULL;
+        }
+
+        opt = optbuf;
+        len = 0;
+        while (*opt) {
+                while (opt[len] != 0) {
+                        if (opt[len] == ' ') {
+                                opt[len++] = 0;
+                                break;
+                        }
+                        len++;
+                }
+                if ((*opt != 0) && (strcasecmp(opt, option) == 0)) {
+                        break;
                 }
+                opt += len;
+                len = 0;
         }
-        free (optbuf);
-        return (NULL);
+        free(optbuf);
+        if (len == 0) {
+                return NULL;
+        }
+
+        return opt - optbuf + mnt->mnt_opts;
 }
 
-static char *
-concatopt (char *s0, const char *s1)
+static int
+writeopt(const char *text, char *buf, int buflen, int pos)
 {
-        size_t i;
-        char *cp;
-
-        if (s1 == NULL || *s1 == '\0')
-                return s0;
-        if (s0 && *s0) {
-                i = strlen(s0) + strlen(s1) + 1 + 1;
-                if ((cp = (char *)malloc(i)) == NULL)
-                        return (NULL);
-                (void)snprintf(cp, i, "%s %s", s0, s1);
-        } else
-                cp = strdup(s1);
-
-        if (s0)
-                free(s0);
-        return (cp);
-}
+        int len;
+
+        /* buflen must be > 0 */
+
+        if (text == NULL) {
+                return pos;
+        }
+
+        buf += pos;
+        if (pos > 0) {
+                /* We are sure we have at least one byte to store the space.
+                 * We don't need to check buflen here. */
+                *buf++ = ' ';
+                pos++;
+        }
+        len = strlen(text) + 1;
+        pos += len;
+        if (pos >= buflen) {
+                /* There won't be enough space for the text and the
+                 * terminating null character. We copy as much as we can
+                 * of the text and mark the end of the string with '...' */
+                memcpy(buf, text, buflen - pos + len);
+                if (buflen > 3) {
+                        strcpy(buf + buflen - 4, "...");
+                } else {
+                        strncpy(buf, "...", buflen - 1);
+                        buf[buflen - 1] = 0;
+                }
+                pos = buflen;
+        } else {
+                memcpy(buf, text, len);
+        }
 
+        return pos;
+}
 
 static char *
-flags2opts (int flags)
+flags2opts (int flags, char *buf, int buflen)
 {
-        char *res;
-        res = NULL;
-        res = concatopt(res, (flags & MNT_RDONLY) ? "ro" : "rw");
-        if (flags & MNT_SYNCHRONOUS)    res = concatopt(res, "sync");
-        if (flags & MNT_NOEXEC)         res = concatopt(res, "noexec");
-        if (flags & MNT_NOSUID)         res = concatopt(res, "nosuid");
-#if !defined(__FreeBSD__)
-        if (flags & MNT_NODEV)          res = concatopt(res, "nodev");
-#endif /* __FreeBSD__ */
-        if (flags & MNT_UNION)          res = concatopt(res, "union");
-        if (flags & MNT_ASYNC)          res = concatopt(res, "async");
-#if !defined(GF_DARWIN_HOST_OS)
-        if (flags & MNT_NOATIME)        res = concatopt(res, "noatime");
-#if !defined(__NetBSD__)
-        if (flags & MNT_NOCLUSTERR)     res = concatopt(res, "noclusterr");
-        if (flags & MNT_NOCLUSTERW)     res = concatopt(res, "noclusterw");
-        if (flags & MNT_NOSYMFOLLOW)    res = concatopt(res, "nosymfollow");
-        if (flags & MNT_SUIDDIR)        res = concatopt(res, "suiddir");
-#endif /* !__NetBSD__ */
-#endif /* !GF_DARWIN_HOS_OS */
-        return res;
+        char other[16];
+        mntflag_t *flg;
+        int pos;
+
+        if (buflen == 0) {
+                return NULL;
+        }
+
+        pos = 0;
+        for (flg = mntflags; flg->value != 0; flg++) {
+                pos = writeopt((flags & flg->value) == 0 ? flg->off : flg->on,
+                               buf, buflen, pos);
+                flags &= ~flg->value;
+        }
+
+        if (flags != 0) {
+                sprintf(other, "[0x%x]", flags);
+                writeopt(other, buf, buflen, pos);
+        }
+
+        return buf;
 }
 
-static struct mntent *
-statfs_to_mntent (gf_statfs_t *mntbuf)
+static void
+statfs_to_mntent (struct mntent *mntent, gf_statfs_t *mntbuf, char *buf,
+                  int buflen)
 {
-        static char opts_buf[40], *tmp;
         int f_flags;
 
-        _mntent.mnt_fsname = mntbuf->f_mntfromname;
-        _mntent.mnt_dir = mntbuf->f_mntonname;
-        _mntent.mnt_type = mntbuf->f_fstypename;
+        mntent->mnt_fsname = mntbuf->f_mntfromname;
+        mntent->mnt_dir = mntbuf->f_mntonname;
+        mntent->mnt_type = mntbuf->f_fstypename;
 
 #ifdef __NetBSD__
         f_flags = mntbuf->f_flag;
 #else
         f_flags = mntbuf->f_flags;
 #endif
-        tmp = flags2opts (f_flags);
-        if (tmp) {
-                opts_buf[sizeof(opts_buf)-1] = '\0';
-                strncpy (opts_buf, tmp, sizeof(opts_buf)-1);
-                free (tmp);
-        } else {
-                *opts_buf = '\0';
-        }
-        _mntent.mnt_opts = opts_buf;
-        _mntent.mnt_freq = _mntent.mnt_passno = 0;
-        return (&_mntent);
+        mntent->mnt_opts = flags2opts (f_flags, buf, buflen);
+
+        mntent->mnt_freq = mntent->mnt_passno = 0;
 }
 
 struct mntent *
-getmntent (FILE *fp)
+getmntent_r (FILE *fp, struct mntent *mntent, char *buf, int buflen)
 {
-        gf_statfs_t *mntbuf;
+        mntent_state_t *state = (mntent_state_t *)fp;
 
-        if (!fp)
+        if (state->pos >= state->count) {
                 return NULL;
-
-        if (pos == -1 || mntsize == -1)
-                mntsize = getmntinfo (&mntbuf, MNT_NOWAIT);
-
-        ++pos;
-        if (pos == mntsize) {
-                pos = mntsize = -1;
-                return (NULL);
         }
 
-        return (statfs_to_mntent (&mntbuf[pos]));
+        statfs_to_mntent(mntent, &state->statfs[state->pos++], buf, buflen);
+
+        return mntent;
 }
 
-/*
-  Careful using this function ``buffer`` and ``bufsize`` are
-  ignored since there is no stream with strings to populate
-  them on OSX or NetBSD, if one wishes to populate them then
-  perhaps a new function should be written in this source file
-  which uses 'getmntinfo()' to stringify the mntent's
-*/
-
-struct mntent *getmntent_r (FILE *fp, struct mntent *result,
-                            char *buffer, int bufsize)
+struct mntent *
+getmntent (FILE *fp)
 {
-        struct mntent *ment = NULL;
-
-        if (!fp)
-                return NULL;
+        mntent_state_t *state = (mntent_state_t *)fp;
 
-        flockfile (fp);
-        ment = getmntent (fp);
-        memcpy (result, ment, sizeof(*ment));
-        funlockfile (fp);
-
-        return result;
+        return getmntent_r(fp, &state->mntent, state->buf,
+                           sizeof(state->buf));
 }
 
 FILE *
 setmntent (const char *filename, const char *type)
 {
-        FILE *fp = NULL;
-#ifdef GF_DARWIN_HOST_OS
-        fp = fopen (filename, "w");
-#else
-        fp = fopen (filename, type);
-#endif
-        return fp;
+        mntent_state_t *state;
+
+        /* We don't really need to access any file so we'll use the FILE* as
+         * a fake file to store state information.
+         */
+
+        state = malloc(sizeof(mntent_state_t));
+        if (state != NULL) {
+                state->pos = 0;
+                state->count = getmntinfo(&state->statfs, MNT_NOWAIT);
+        }
+
+        return (FILE *)state;
 }
 
 int
 endmntent (FILE *fp)
 {
-        if (fp)
-                fclose (fp);
+        free(fp);
 
         return 1; /* endmntent() always returns 1 */
 }
diff --git a/contrib/qemu/block.c b/contrib/qemu/block.c
deleted file mode 100644
index b56024113b8..00000000000
--- a/contrib/qemu/block.c
+++ /dev/null
@@ -1,4604 +0,0 @@
-/*
- * QEMU System Emulator block driver
- *
- * Copyright (c) 2003 Fabrice Bellard
- *
- * Permission is hereby granted, free of charge, to any person obtaining a copy
- * of this software and associated documentation files (the "Software"), to deal
- * in the Software without restriction, including without limitation the rights
- * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
- * copies of the Software, and to permit persons to whom the Software is
- * furnished to do so, subject to the following conditions:
- *
- * The above copyright notice and this permission notice shall be included in
- * all copies or substantial portions of the Software.
- *
- * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
- * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
- * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
- * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
- * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
- * THE SOFTWARE.
- */
-#include "config-host.h"
-#include "qemu-common.h"
-#include "trace.h"
-#include "monitor/monitor.h"
-#include "block/block_int.h"
-#include "block/blockjob.h"
-#include "qemu/module.h"
-#include "qapi/qmp/qjson.h"
-#include "sysemu/sysemu.h"
-#include "qemu/notify.h"
-#include "block/coroutine.h"
-#include "qmp-commands.h"
-#include "qemu/timer.h"
-
-#ifdef CONFIG_BSD
-#include <sys/types.h>
-#include <sys/stat.h>
-#include <sys/ioctl.h>
-#include <sys/queue.h>
-#ifndef __DragonFly__
-#include <sys/disk.h>
-#endif
-#endif
-
-#ifdef _WIN32
-#include <windows.h>
-#endif
-
-#define NOT_DONE 0x7fffffff /* used while emulated sync operation in progress */
-
-typedef enum {
-    BDRV_REQ_COPY_ON_READ = 0x1,
-    BDRV_REQ_ZERO_WRITE   = 0x2,
-} BdrvRequestFlags;
-
-static void bdrv_dev_change_media_cb(BlockDriverState *bs, bool load);
-static BlockDriverAIOCB *bdrv_aio_readv_em(BlockDriverState *bs,
-        int64_t sector_num, QEMUIOVector *qiov, int nb_sectors,
-        BlockDriverCompletionFunc *cb, void *opaque);
-static BlockDriverAIOCB *bdrv_aio_writev_em(BlockDriverState *bs,
-        int64_t sector_num, QEMUIOVector *qiov, int nb_sectors,
-        BlockDriverCompletionFunc *cb, void *opaque);
-static int coroutine_fn bdrv_co_readv_em(BlockDriverState *bs,
-                                         int64_t sector_num, int nb_sectors,
-                                         QEMUIOVector *iov);
-static int coroutine_fn bdrv_co_writev_em(BlockDriverState *bs,
-                                         int64_t sector_num, int nb_sectors,
-                                         QEMUIOVector *iov);
-static int coroutine_fn bdrv_co_do_readv(BlockDriverState *bs,
-    int64_t sector_num, int nb_sectors, QEMUIOVector *qiov,
-    BdrvRequestFlags flags);
-static int coroutine_fn bdrv_co_do_writev(BlockDriverState *bs,
-    int64_t sector_num, int nb_sectors, QEMUIOVector *qiov,
-    BdrvRequestFlags flags);
-static BlockDriverAIOCB *bdrv_co_aio_rw_vector(BlockDriverState *bs,
-                                               int64_t sector_num,
-                                               QEMUIOVector *qiov,
-                                               int nb_sectors,
-                                               BlockDriverCompletionFunc *cb,
-                                               void *opaque,
-                                               bool is_write);
-static void coroutine_fn bdrv_co_do_rw(void *opaque);
-static int coroutine_fn bdrv_co_do_write_zeroes(BlockDriverState *bs,
-    int64_t sector_num, int nb_sectors);
-
-static bool bdrv_exceed_bps_limits(BlockDriverState *bs, int nb_sectors,
-        bool is_write, double elapsed_time, uint64_t *wait);
-static bool bdrv_exceed_iops_limits(BlockDriverState *bs, bool is_write,
-        double elapsed_time, uint64_t *wait);
-static bool bdrv_exceed_io_limits(BlockDriverState *bs, int nb_sectors,
-        bool is_write, int64_t *wait);
-
-static QTAILQ_HEAD(, BlockDriverState) bdrv_states =
-    QTAILQ_HEAD_INITIALIZER(bdrv_states);
-
-static QLIST_HEAD(, BlockDriver) bdrv_drivers =
-    QLIST_HEAD_INITIALIZER(bdrv_drivers);
-
-/* If non-zero, use only whitelisted block drivers */
-static int use_bdrv_whitelist;
-
-#ifdef _WIN32
-static int is_windows_drive_prefix(const char *filename)
-{
-    return (((filename[0] >= 'a' && filename[0] <= 'z') ||
-             (filename[0] >= 'A' && filename[0] <= 'Z')) &&
-            filename[1] == ':');
-}
-
-int is_windows_drive(const char *filename)
-{
-    if (is_windows_drive_prefix(filename) &&
-        filename[2] == '\0')
-        return 1;
-    if (strstart(filename, "\\\\.\\", NULL) ||
-        strstart(filename, "//./", NULL))
-        return 1;
-    return 0;
-}
-#endif
-
-/* throttling disk I/O limits */
-void bdrv_io_limits_disable(BlockDriverState *bs)
-{
-    bs->io_limits_enabled = false;
-
-    while (qemu_co_queue_next(&bs->throttled_reqs));
-
-    if (bs->block_timer) {
-        qemu_del_timer(bs->block_timer);
-        qemu_free_timer(bs->block_timer);
-        bs->block_timer = NULL;
-    }
-
-    bs->slice_start = 0;
-    bs->slice_end   = 0;
-}
-
-static void bdrv_block_timer(void *opaque)
-{
-    BlockDriverState *bs = opaque;
-
-    qemu_co_queue_next(&bs->throttled_reqs);
-}
-
-void bdrv_io_limits_enable(BlockDriverState *bs)
-{
-    qemu_co_queue_init(&bs->throttled_reqs);
-    bs->block_timer = qemu_new_timer_ns(vm_clock, bdrv_block_timer, bs);
-    bs->io_limits_enabled = true;
-}
-
-bool bdrv_io_limits_enabled(BlockDriverState *bs)
-{
-    BlockIOLimit *io_limits = &bs->io_limits;
-    return io_limits->bps[BLOCK_IO_LIMIT_READ]
-         || io_limits->bps[BLOCK_IO_LIMIT_WRITE]
-         || io_limits->bps[BLOCK_IO_LIMIT_TOTAL]
-         || io_limits->iops[BLOCK_IO_LIMIT_READ]
-         || io_limits->iops[BLOCK_IO_LIMIT_WRITE]
-         || io_limits->iops[BLOCK_IO_LIMIT_TOTAL];
-}
-
-static void bdrv_io_limits_intercept(BlockDriverState *bs,
-                                     bool is_write, int nb_sectors)
-{
-    int64_t wait_time = -1;
-
-    if (!qemu_co_queue_empty(&bs->throttled_reqs)) {
-        qemu_co_queue_wait(&bs->throttled_reqs);
-    }
-
-    /* In fact, we hope to keep each request's timing, in FIFO mode. The next
-     * throttled requests will not be dequeued until the current request is
-     * allowed to be serviced. So if the current request still exceeds the
-     * limits, it will be inserted to the head. All requests followed it will
-     * be still in throttled_reqs queue.
-     */
-
-    while (bdrv_exceed_io_limits(bs, nb_sectors, is_write, &wait_time)) {
-        qemu_mod_timer(bs->block_timer,
-                       wait_time + qemu_get_clock_ns(vm_clock));
-        qemu_co_queue_wait_insert_head(&bs->throttled_reqs);
-    }
-
-    qemu_co_queue_next(&bs->throttled_reqs);
-}
-
-/* check if the path starts with "<protocol>:" */
-static int path_has_protocol(const char *path)
-{
-    const char *p;
-
-#ifdef _WIN32
-    if (is_windows_drive(path) ||
-        is_windows_drive_prefix(path)) {
-        return 0;
-    }
-    p = path + strcspn(path, ":/\\");
-#else
-    p = path + strcspn(path, ":/");
-#endif
-
-    return *p == ':';
-}
-
-int path_is_absolute(const char *path)
-{
-#ifdef _WIN32
-    /* specific case for names like: "\\.\d:" */
-    if (is_windows_drive(path) || is_windows_drive_prefix(path)) {
-        return 1;
-    }
-    return (*path == '/' || *path == '\\');
-#else
-    return (*path == '/');
-#endif
-}
-
-/* if filename is absolute, just copy it to dest. Otherwise, build a
-   path to it by considering it is relative to base_path. URL are
-   supported. */
-void path_combine(char *dest, int dest_size,
-                  const char *base_path,
-                  const char *filename)
-{
-    const char *p, *p1;
-    int len;
-
-    if (dest_size <= 0)
-        return;
-    if (path_is_absolute(filename)) {
-        pstrcpy(dest, dest_size, filename);
-    } else {
-        p = strchr(base_path, ':');
-        if (p)
-            p++;
-        else
-            p = base_path;
-        p1 = strrchr(base_path, '/');
-#ifdef _WIN32
-        {
-            const char *p2;
-            p2 = strrchr(base_path, '\\');
-            if (!p1 || p2 > p1)
-                p1 = p2;
-        }
-#endif
-        if (p1)
-            p1++;
-        else
-            p1 = base_path;
-        if (p1 > p)
-            p = p1;
-        len = p - base_path;
-        if (len > dest_size - 1)
-            len = dest_size - 1;
-        memcpy(dest, base_path, len);
-        dest[len] = '\0';
-        pstrcat(dest, dest_size, filename);
-    }
-}
-
-void bdrv_get_full_backing_filename(BlockDriverState *bs, char *dest, size_t sz)
-{
-    if (bs->backing_file[0] == '\0' || path_has_protocol(bs->backing_file)) {
-        pstrcpy(dest, sz, bs->backing_file);
-    } else {
-        path_combine(dest, sz, bs->filename, bs->backing_file);
-    }
-}
-
-void bdrv_register(BlockDriver *bdrv)
-{
-    /* Block drivers without coroutine functions need emulation */
-    if (!bdrv->bdrv_co_readv) {
-        bdrv->bdrv_co_readv = bdrv_co_readv_em;
-        bdrv->bdrv_co_writev = bdrv_co_writev_em;
-
-        /* bdrv_co_readv_em()/brdv_co_writev_em() work in terms of aio, so if
-         * the block driver lacks aio we need to emulate that too.
-         */
-        if (!bdrv->bdrv_aio_readv) {
-            /* add AIO emulation layer */
-            bdrv->bdrv_aio_readv = bdrv_aio_readv_em;
-            bdrv->bdrv_aio_writev = bdrv_aio_writev_em;
-        }
-    }
-
-    QLIST_INSERT_HEAD(&bdrv_drivers, bdrv, list);
-}
-
-/* create a new block device (by default it is empty) */
-BlockDriverState *bdrv_new(const char *device_name)
-{
-    BlockDriverState *bs;
-
-    bs = g_malloc0(sizeof(BlockDriverState));
-    pstrcpy(bs->device_name, sizeof(bs->device_name), device_name);
-    if (device_name[0] != '\0') {
-        QTAILQ_INSERT_TAIL(&bdrv_states, bs, list);
-    }
-    bdrv_iostatus_disable(bs);
-    notifier_list_init(&bs->close_notifiers);
-    notifier_with_return_list_init(&bs->before_write_notifiers);
-
-    return bs;
-}
-
-void bdrv_add_close_notifier(BlockDriverState *bs, Notifier *notify)
-{
-    notifier_list_add(&bs->close_notifiers, notify);
-}
-
-BlockDriver *bdrv_find_format(const char *format_name)
-{
-    BlockDriver *drv1;
-    QLIST_FOREACH(drv1, &bdrv_drivers, list) {
-        if (!strcmp(drv1->format_name, format_name)) {
-            return drv1;
-        }
-    }
-    return NULL;
-}
-
-static int bdrv_is_whitelisted(BlockDriver *drv, bool read_only)
-{
-    static const char *whitelist_rw[] = {
-        CONFIG_BDRV_RW_WHITELIST
-    };
-    static const char *whitelist_ro[] = {
-        CONFIG_BDRV_RO_WHITELIST
-    };
-    const char **p;
-
-    if (!whitelist_rw[0] && !whitelist_ro[0]) {
-        return 1;               /* no whitelist, anything goes */
-    }
-
-    for (p = whitelist_rw; *p; p++) {
-        if (!strcmp(drv->format_name, *p)) {
-            return 1;
-        }
-    }
-    if (read_only) {
-        for (p = whitelist_ro; *p; p++) {
-            if (!strcmp(drv->format_name, *p)) {
-                return 1;
-            }
-        }
-    }
-    return 0;
-}
-
-BlockDriver *bdrv_find_whitelisted_format(const char *format_name,
-                                          bool read_only)
-{
-    BlockDriver *drv = bdrv_find_format(format_name);
-    return drv && bdrv_is_whitelisted(drv, read_only) ? drv : NULL;
-}
-
-typedef struct CreateCo {
-    BlockDriver *drv;
-    char *filename;
-    QEMUOptionParameter *options;
-    int ret;
-} CreateCo;
-
-static void coroutine_fn bdrv_create_co_entry(void *opaque)
-{
-    CreateCo *cco = opaque;
-    assert(cco->drv);
-
-    cco->ret = cco->drv->bdrv_create(cco->filename, cco->options);
-}
-
-int bdrv_create(BlockDriver *drv, const char* filename,
-    QEMUOptionParameter *options)
-{
-    int ret;
-
-    Coroutine *co;
-    CreateCo cco = {
-        .drv = drv,
-        .filename = g_strdup(filename),
-        .options = options,
-        .ret = NOT_DONE,
-    };
-
-    if (!drv->bdrv_create) {
-        ret = -ENOTSUP;
-        goto out;
-    }
-
-    if (qemu_in_coroutine()) {
-        /* Fast-path if already in coroutine context */
-        bdrv_create_co_entry(&cco);
-    } else {
-        co = qemu_coroutine_create(bdrv_create_co_entry);
-        qemu_coroutine_enter(co, &cco);
-        while (cco.ret == NOT_DONE) {
-            qemu_aio_wait();
-        }
-    }
-
-    ret = cco.ret;
-
-out:
-    g_free(cco.filename);
-    return ret;
-}
-
-int bdrv_create_file(const char* filename, QEMUOptionParameter *options)
-{
-    BlockDriver *drv;
-
-    drv = bdrv_find_protocol(filename, true);
-    if (drv == NULL) {
-        return -ENOENT;
-    }
-
-    return bdrv_create(drv, filename, options);
-}
-
-/*
- * Create a uniquely-named empty temporary file.
- * Return 0 upon success, otherwise a negative errno value.
- */
-int get_tmp_filename(char *filename, int size)
-{
-#ifdef _WIN32
-    char temp_dir[MAX_PATH];
-    /* GetTempFileName requires that its output buffer (4th param)
-       have length MAX_PATH or greater.  */
-    assert(size >= MAX_PATH);
-    return (GetTempPath(MAX_PATH, temp_dir)
-            && GetTempFileName(temp_dir, "qem", 0, filename)
-            ? 0 : -GetLastError());
-#else
-    int fd;
-    const char *tmpdir;
-    tmpdir = getenv("TMPDIR");
-    if (!tmpdir)
-        tmpdir = "/tmp";
-    if (snprintf(filename, size, "%s/vl.XXXXXX", tmpdir) >= size) {
-        return -EOVERFLOW;
-    }
-    fd = mkstemp(filename);
-    if (fd < 0) {
-        return -errno;
-    }
-    if (close(fd) != 0) {
-        unlink(filename);
-        return -errno;
-    }
-    return 0;
-#endif
-}
-
-/*
- * Detect host devices. By convention, /dev/cdrom[N] is always
- * recognized as a host CDROM.
- */
-static BlockDriver *find_hdev_driver(const char *filename)
-{
-    int score_max = 0, score;
-    BlockDriver *drv = NULL, *d;
-
-    QLIST_FOREACH(d, &bdrv_drivers, list) {
-        if (d->bdrv_probe_device) {
-            score = d->bdrv_probe_device(filename);
-            if (score > score_max) {
-                score_max = score;
-                drv = d;
-            }
-        }
-    }
-
-    return drv;
-}
-
-BlockDriver *bdrv_find_protocol(const char *filename,
-                                bool allow_protocol_prefix)
-{
-    BlockDriver *drv1;
-    char protocol[128];
-    int len;
-    const char *p;
-
-    /* TODO Drivers without bdrv_file_open must be specified explicitly */
-
-    /*
-     * XXX(hch): we really should not let host device detection
-     * override an explicit protocol specification, but moving this
-     * later breaks access to device names with colons in them.
-     * Thanks to the brain-dead persistent naming schemes on udev-
-     * based Linux systems those actually are quite common.
-     */
-    drv1 = find_hdev_driver(filename);
-    if (drv1) {
-        return drv1;
-    }
-
-    if (!path_has_protocol(filename) || !allow_protocol_prefix) {
-        return bdrv_find_format("file");
-    }
-
-    p = strchr(filename, ':');
-    assert(p != NULL);
-    len = p - filename;
-    if (len > sizeof(protocol) - 1)
-        len = sizeof(protocol) - 1;
-    memcpy(protocol, filename, len);
-    protocol[len] = '\0';
-    QLIST_FOREACH(drv1, &bdrv_drivers, list) {
-        if (drv1->protocol_name &&
-            !strcmp(drv1->protocol_name, protocol)) {
-            return drv1;
-        }
-    }
-    return NULL;
-}
-
-static int find_image_format(BlockDriverState *bs, const char *filename,
-                             BlockDriver **pdrv)
-{
-    int score, score_max;
-    BlockDriver *drv1, *drv;
-    uint8_t buf[2048];
-    int ret = 0;
-
-    /* Return the raw BlockDriver * to scsi-generic devices or empty drives */
-    if (bs->sg || !bdrv_is_inserted(bs) || bdrv_getlength(bs) == 0) {
-        drv = bdrv_find_format("raw");
-        if (!drv) {
-            ret = -ENOENT;
-        }
-        *pdrv = drv;
-        return ret;
-    }
-
-    ret = bdrv_pread(bs, 0, buf, sizeof(buf));
-    if (ret < 0) {
-        *pdrv = NULL;
-        return ret;
-    }
-
-    score_max = 0;
-    drv = NULL;
-    QLIST_FOREACH(drv1, &bdrv_drivers, list) {
-        if (drv1->bdrv_probe) {
-            score = drv1->bdrv_probe(buf, ret, filename);
-            if (score > score_max) {
-                score_max = score;
-                drv = drv1;
-            }
-        }
-    }
-    if (!drv) {
-        ret = -ENOENT;
-    }
-    *pdrv = drv;
-    return ret;
-}
-
-/**
- * Set the current 'total_sectors' value
- */
-static int refresh_total_sectors(BlockDriverState *bs, int64_t hint)
-{
-    BlockDriver *drv = bs->drv;
-
-    /* Do not attempt drv->bdrv_getlength() on scsi-generic devices */
-    if (bs->sg)
-        return 0;
-
-    /* query actual device if possible, otherwise just trust the hint */
-    if (drv->bdrv_getlength) {
-        int64_t length = drv->bdrv_getlength(bs);
-        if (length < 0) {
-            return length;
-        }
-        hint = length >> BDRV_SECTOR_BITS;
-    }
-
-    bs->total_sectors = hint;
-    return 0;
-}
-
-/**
- * Set open flags for a given discard mode
- *
- * Return 0 on success, -1 if the discard mode was invalid.
- */
-int bdrv_parse_discard_flags(const char *mode, int *flags)
-{
-    *flags &= ~BDRV_O_UNMAP;
-
-    if (!strcmp(mode, "off") || !strcmp(mode, "ignore")) {
-        /* do nothing */
-    } else if (!strcmp(mode, "on") || !strcmp(mode, "unmap")) {
-        *flags |= BDRV_O_UNMAP;
-    } else {
-        return -1;
-    }
-
-    return 0;
-}
-
-/**
- * Set open flags for a given cache mode
- *
- * Return 0 on success, -1 if the cache mode was invalid.
- */
-int bdrv_parse_cache_flags(const char *mode, int *flags)
-{
-    *flags &= ~BDRV_O_CACHE_MASK;
-
-    if (!strcmp(mode, "off") || !strcmp(mode, "none")) {
-        *flags |= BDRV_O_NOCACHE | BDRV_O_CACHE_WB;
-    } else if (!strcmp(mode, "directsync")) {
-        *flags |= BDRV_O_NOCACHE;
-    } else if (!strcmp(mode, "writeback")) {
-        *flags |= BDRV_O_CACHE_WB;
-    } else if (!strcmp(mode, "unsafe")) {
-        *flags |= BDRV_O_CACHE_WB;
-        *flags |= BDRV_O_NO_FLUSH;
-    } else if (!strcmp(mode, "writethrough")) {
-        /* this is the default */
-    } else {
-        return -1;
-    }
-
-    return 0;
-}
-
-/**
- * The copy-on-read flag is actually a reference count so multiple users may
- * use the feature without worrying about clobbering its previous state.
- * Copy-on-read stays enabled until all users have called to disable it.
- */
-void bdrv_enable_copy_on_read(BlockDriverState *bs)
-{
-    bs->copy_on_read++;
-}
-
-void bdrv_disable_copy_on_read(BlockDriverState *bs)
-{
-    assert(bs->copy_on_read > 0);
-    bs->copy_on_read--;
-}
-
-static int bdrv_open_flags(BlockDriverState *bs, int flags)
-{
-    int open_flags = flags | BDRV_O_CACHE_WB;
-
-    /*
-     * Clear flags that are internal to the block layer before opening the
-     * image.
-     */
-    open_flags &= ~(BDRV_O_SNAPSHOT | BDRV_O_NO_BACKING);
-
-    /*
-     * Snapshots should be writable.
-     */
-    if (bs->is_temporary) {
-        open_flags |= BDRV_O_RDWR;
-    }
-
-    return open_flags;
-}
-
-/*
- * Common part for opening disk images and files
- *
- * Removes all processed options from *options.
- */
-static int bdrv_open_common(BlockDriverState *bs, BlockDriverState *file,
-    QDict *options, int flags, BlockDriver *drv)
-{
-    int ret, open_flags;
-    const char *filename;
-
-    assert(drv != NULL);
-    assert(bs->file == NULL);
-    assert(options != NULL && bs->options != options);
-
-    if (file != NULL) {
-        filename = file->filename;
-    } else {
-        filename = qdict_get_try_str(options, "filename");
-    }
-
-    trace_bdrv_open_common(bs, filename ?: "", flags, drv->format_name);
-
-    /* bdrv_open() with directly using a protocol as drv. This layer is already
-     * opened, so assign it to bs (while file becomes a closed BlockDriverState)
-     * and return immediately. */
-    if (file != NULL && drv->bdrv_file_open) {
-        bdrv_swap(file, bs);
-        return 0;
-    }
-
-    bs->open_flags = flags;
-    bs->buffer_alignment = 512;
-    open_flags = bdrv_open_flags(bs, flags);
-    bs->read_only = !(open_flags & BDRV_O_RDWR);
-
-    if (use_bdrv_whitelist && !bdrv_is_whitelisted(drv, bs->read_only)) {
-        return -ENOTSUP;
-    }
-
-    assert(bs->copy_on_read == 0); /* bdrv_new() and bdrv_close() make it so */
-    if (!bs->read_only && (flags & BDRV_O_COPY_ON_READ)) {
-        bdrv_enable_copy_on_read(bs);
-    }
-
-    if (filename != NULL) {
-        pstrcpy(bs->filename, sizeof(bs->filename), filename);
-    } else {
-        bs->filename[0] = '\0';
-    }
-
-    bs->drv = drv;
-    bs->opaque = g_malloc0(drv->instance_size);
-
-    bs->enable_write_cache = !!(flags & BDRV_O_CACHE_WB);
-
-    /* Open the image, either directly or using a protocol */
-    if (drv->bdrv_file_open) {
-        assert(file == NULL);
-        assert(drv->bdrv_parse_filename || filename != NULL);
-        ret = drv->bdrv_file_open(bs, options, open_flags);
-    } else {
-        if (file == NULL) {
-            qerror_report(ERROR_CLASS_GENERIC_ERROR, "Can't use '%s' as a "
-                          "block driver for the protocol level",
-                          drv->format_name);
-            ret = -EINVAL;
-            goto free_and_fail;
-        }
-        assert(file != NULL);
-        bs->file = file;
-        ret = drv->bdrv_open(bs, options, open_flags);
-    }
-
-    if (ret < 0) {
-        goto free_and_fail;
-    }
-
-    ret = refresh_total_sectors(bs, bs->total_sectors);
-    if (ret < 0) {
-        goto free_and_fail;
-    }
-
-#ifndef _WIN32
-    if (bs->is_temporary) {
-        assert(filename != NULL);
-        unlink(filename);
-    }
-#endif
-    return 0;
-
-free_and_fail:
-    bs->file = NULL;
-    g_free(bs->opaque);
-    bs->opaque = NULL;
-    bs->drv = NULL;
-    return ret;
-}
-
-/*
- * Opens a file using a protocol (file, host_device, nbd, ...)
- *
- * options is a QDict of options to pass to the block drivers, or NULL for an
- * empty set of options. The reference to the QDict belongs to the block layer
- * after the call (even on failure), so if the caller intends to reuse the
- * dictionary, it needs to use QINCREF() before calling bdrv_file_open.
- */
-int bdrv_file_open(BlockDriverState **pbs, const char *filename,
-                   QDict *options, int flags)
-{
-    BlockDriverState *bs;
-    BlockDriver *drv;
-    const char *drvname;
-    bool allow_protocol_prefix = false;
-    int ret;
-
-    /* NULL means an empty set of options */
-    if (options == NULL) {
-        options = qdict_new();
-    }
-
-    bs = bdrv_new("");
-    bs->options = options;
-    options = qdict_clone_shallow(options);
-
-    /* Fetch the file name from the options QDict if necessary */
-    if (!filename) {
-        filename = qdict_get_try_str(options, "filename");
-    } else if (filename && !qdict_haskey(options, "filename")) {
-        qdict_put(options, "filename", qstring_from_str(filename));
-        allow_protocol_prefix = true;
-    } else {
-        qerror_report(ERROR_CLASS_GENERIC_ERROR, "Can't specify 'file' and "
-                      "'filename' options at the same time");
-        ret = -EINVAL;
-        goto fail;
-    }
-
-    /* Find the right block driver */
-    drvname = qdict_get_try_str(options, "driver");
-    if (drvname) {
-        drv = bdrv_find_whitelisted_format(drvname, !(flags & BDRV_O_RDWR));
-        qdict_del(options, "driver");
-    } else if (filename) {
-        drv = bdrv_find_protocol(filename, allow_protocol_prefix);
-        if (!drv) {
-            qerror_report(ERROR_CLASS_GENERIC_ERROR, "Unknown protocol");
-        }
-    } else {
-        qerror_report(ERROR_CLASS_GENERIC_ERROR,
-                      "Must specify either driver or file");
-        drv = NULL;
-    }
-
-    if (!drv) {
-        ret = -ENOENT;
-        goto fail;
-    }
-
-    /* Parse the filename and open it */
-    if (drv->bdrv_parse_filename && filename) {
-        Error *local_err = NULL;
-        drv->bdrv_parse_filename(filename, options, &local_err);
-        if (error_is_set(&local_err)) {
-            qerror_report_err(local_err);
-            error_free(local_err);
-            ret = -EINVAL;
-            goto fail;
-        }
-        qdict_del(options, "filename");
-    } else if (!drv->bdrv_parse_filename && !filename) {
-        qerror_report(ERROR_CLASS_GENERIC_ERROR,
-                      "The '%s' block driver requires a file name",
-                      drv->format_name);
-        ret = -EINVAL;
-        goto fail;
-    }
-
-    ret = bdrv_open_common(bs, NULL, options, flags, drv);
-    if (ret < 0) {
-        goto fail;
-    }
-
-    /* Check if any unknown options were used */
-    if (qdict_size(options) != 0) {
-        const QDictEntry *entry = qdict_first(options);
-        qerror_report(ERROR_CLASS_GENERIC_ERROR, "Block protocol '%s' doesn't "
-                      "support the option '%s'",
-                      drv->format_name, entry->key);
-        ret = -EINVAL;
-        goto fail;
-    }
-    QDECREF(options);
-
-    bs->growable = 1;
-    *pbs = bs;
-    return 0;
-
-fail:
-    QDECREF(options);
-    if (!bs->drv) {
-        QDECREF(bs->options);
-    }
-    bdrv_delete(bs);
-    return ret;
-}
-
-/*
- * Opens the backing file for a BlockDriverState if not yet open
- *
- * options is a QDict of options to pass to the block drivers, or NULL for an
- * empty set of options. The reference to the QDict is transferred to this
- * function (even on failure), so if the caller intends to reuse the dictionary,
- * it needs to use QINCREF() before calling bdrv_file_open.
- */
-int bdrv_open_backing_file(BlockDriverState *bs, QDict *options)
-{
-    char backing_filename[PATH_MAX];
-    int back_flags, ret;
-    BlockDriver *back_drv = NULL;
-
-    if (bs->backing_hd != NULL) {
-        QDECREF(options);
-        return 0;
-    }
-
-    /* NULL means an empty set of options */
-    if (options == NULL) {
-        options = qdict_new();
-    }
-
-    bs->open_flags &= ~BDRV_O_NO_BACKING;
-    if (qdict_haskey(options, "file.filename")) {
-        backing_filename[0] = '\0';
-    } else if (bs->backing_file[0] == '\0' && qdict_size(options) == 0) {
-        QDECREF(options);
-        return 0;
-    }
-
-    bs->backing_hd = bdrv_new("");
-    bdrv_get_full_backing_filename(bs, backing_filename,
-                                   sizeof(backing_filename));
-
-    if (bs->backing_format[0] != '\0') {
-        back_drv = bdrv_find_format(bs->backing_format);
-    }
-
-    /* backing files always opened read-only */
-    back_flags = bs->open_flags & ~(BDRV_O_RDWR | BDRV_O_SNAPSHOT);
-
-    ret = bdrv_open(bs->backing_hd,
-                    *backing_filename ? backing_filename : NULL, options,
-                    back_flags, back_drv);
-    if (ret < 0) {
-        bdrv_delete(bs->backing_hd);
-        bs->backing_hd = NULL;
-        bs->open_flags |= BDRV_O_NO_BACKING;
-        return ret;
-    }
-    return 0;
-}
-
-static void extract_subqdict(QDict *src, QDict **dst, const char *start)
-{
-    const QDictEntry *entry, *next;
-    const char *p;
-
-    *dst = qdict_new();
-    entry = qdict_first(src);
-
-    while (entry != NULL) {
-        next = qdict_next(src, entry);
-        if (strstart(entry->key, start, &p)) {
-            qobject_incref(entry->value);
-            qdict_put_obj(*dst, p, entry->value);
-            qdict_del(src, entry->key);
-        }
-        entry = next;
-    }
-}
-
-/*
- * Opens a disk image (raw, qcow2, vmdk, ...)
- *
- * options is a QDict of options to pass to the block drivers, or NULL for an
- * empty set of options. The reference to the QDict belongs to the block layer
- * after the call (even on failure), so if the caller intends to reuse the
- * dictionary, it needs to use QINCREF() before calling bdrv_open.
- */
-int bdrv_open(BlockDriverState *bs, const char *filename, QDict *options,
-              int flags, BlockDriver *drv)
-{
-    int ret;
-    /* TODO: extra byte is a hack to ensure MAX_PATH space on Windows. */
-    char tmp_filename[PATH_MAX + 1];
-    BlockDriverState *file = NULL;
-    QDict *file_options = NULL;
-
-    /* NULL means an empty set of options */
-    if (options == NULL) {
-        options = qdict_new();
-    }
-
-    bs->options = options;
-    options = qdict_clone_shallow(options);
-
-    /* For snapshot=on, create a temporary qcow2 overlay */
-    if (flags & BDRV_O_SNAPSHOT) {
-        BlockDriverState *bs1;
-        int64_t total_size;
-        BlockDriver *bdrv_qcow2;
-        QEMUOptionParameter *create_options;
-        char backing_filename[PATH_MAX];
-
-        if (qdict_size(options) != 0) {
-            error_report("Can't use snapshot=on with driver-specific options");
-            ret = -EINVAL;
-            goto fail;
-        }
-        assert(filename != NULL);
-
-        /* if snapshot, we create a temporary backing file and open it
-           instead of opening 'filename' directly */
-
-        /* if there is a backing file, use it */
-        bs1 = bdrv_new("");
-        ret = bdrv_open(bs1, filename, NULL, 0, drv);
-        if (ret < 0) {
-            bdrv_delete(bs1);
-            goto fail;
-        }
-        total_size = bdrv_getlength(bs1) & BDRV_SECTOR_MASK;
-
-        bdrv_delete(bs1);
-
-        ret = get_tmp_filename(tmp_filename, sizeof(tmp_filename));
-        if (ret < 0) {
-            goto fail;
-        }
-
-        /* Real path is meaningless for protocols */
-        if (path_has_protocol(filename)) {
-            snprintf(backing_filename, sizeof(backing_filename),
-                     "%s", filename);
-        } else if (!realpath(filename, backing_filename)) {
-            ret = -errno;
-            goto fail;
-        }
-
-        bdrv_qcow2 = bdrv_find_format("qcow2");
-        create_options = parse_option_parameters("", bdrv_qcow2->create_options,
-                                                 NULL);
-
-        set_option_parameter_int(create_options, BLOCK_OPT_SIZE, total_size);
-        set_option_parameter(create_options, BLOCK_OPT_BACKING_FILE,
-                             backing_filename);
-        if (drv) {
-            set_option_parameter(create_options, BLOCK_OPT_BACKING_FMT,
-                drv->format_name);
-        }
-
-        ret = bdrv_create(bdrv_qcow2, tmp_filename, create_options);
-        free_option_parameters(create_options);
-        if (ret < 0) {
-            goto fail;
-        }
-
-        filename = tmp_filename;
-        drv = bdrv_qcow2;
-        bs->is_temporary = 1;
-    }
-
-    /* Open image file without format layer */
-    if (flags & BDRV_O_RDWR) {
-        flags |= BDRV_O_ALLOW_RDWR;
-    }
-
-    extract_subqdict(options, &file_options, "file.");
-
-    ret = bdrv_file_open(&file, filename, file_options,
-                         bdrv_open_flags(bs, flags | BDRV_O_UNMAP));
-    if (ret < 0) {
-        goto fail;
-    }
-
-    /* Find the right image format driver */
-    if (!drv) {
-        ret = find_image_format(file, filename, &drv);
-    }
-
-    if (!drv) {
-        goto unlink_and_fail;
-    }
-
-    /* Open the image */
-    ret = bdrv_open_common(bs, file, options, flags, drv);
-    if (ret < 0) {
-        goto unlink_and_fail;
-    }
-
-    if (bs->file != file) {
-        bdrv_delete(file);
-        file = NULL;
-    }
-
-    /* If there is a backing file, use it */
-    if ((flags & BDRV_O_NO_BACKING) == 0) {
-        QDict *backing_options;
-
-        extract_subqdict(options, &backing_options, "backing.");
-        ret = bdrv_open_backing_file(bs, backing_options);
-        if (ret < 0) {
-            goto close_and_fail;
-        }
-    }
-
-    /* Check if any unknown options were used */
-    if (qdict_size(options) != 0) {
-        const QDictEntry *entry = qdict_first(options);
-        qerror_report(ERROR_CLASS_GENERIC_ERROR, "Block format '%s' used by "
-            "device '%s' doesn't support the option '%s'",
-            drv->format_name, bs->device_name, entry->key);
-
-        ret = -EINVAL;
-        goto close_and_fail;
-    }
-    QDECREF(options);
-
-    if (!bdrv_key_required(bs)) {
-        bdrv_dev_change_media_cb(bs, true);
-    }
-
-    /* throttling disk I/O limits */
-    if (bs->io_limits_enabled) {
-        bdrv_io_limits_enable(bs);
-    }
-
-    return 0;
-
-unlink_and_fail:
-    if (file != NULL) {
-        bdrv_delete(file);
-    }
-    if (bs->is_temporary) {
-        unlink(filename);
-    }
-fail:
-    QDECREF(bs->options);
-    QDECREF(options);
-    bs->options = NULL;
-    return ret;
-
-close_and_fail:
-    bdrv_close(bs);
-    QDECREF(options);
-    return ret;
-}
-
-typedef struct BlockReopenQueueEntry {
-     bool prepared;
-     BDRVReopenState state;
-     QSIMPLEQ_ENTRY(BlockReopenQueueEntry) entry;
-} BlockReopenQueueEntry;
-
-/*
- * Adds a BlockDriverState to a simple queue for an atomic, transactional
- * reopen of multiple devices.
- *
- * bs_queue can either be an existing BlockReopenQueue that has had QSIMPLE_INIT
- * already performed, or alternatively may be NULL a new BlockReopenQueue will
- * be created and initialized. This newly created BlockReopenQueue should be
- * passed back in for subsequent calls that are intended to be of the same
- * atomic 'set'.
- *
- * bs is the BlockDriverState to add to the reopen queue.
- *
- * flags contains the open flags for the associated bs
- *
- * returns a pointer to bs_queue, which is either the newly allocated
- * bs_queue, or the existing bs_queue being used.
- *
- */
-BlockReopenQueue *bdrv_reopen_queue(BlockReopenQueue *bs_queue,
-                                    BlockDriverState *bs, int flags)
-{
-    assert(bs != NULL);
-
-    BlockReopenQueueEntry *bs_entry;
-    if (bs_queue == NULL) {
-        bs_queue = g_new0(BlockReopenQueue, 1);
-        QSIMPLEQ_INIT(bs_queue);
-    }
-
-    if (bs->file) {
-        bdrv_reopen_queue(bs_queue, bs->file, flags);
-    }
-
-    bs_entry = g_new0(BlockReopenQueueEntry, 1);
-    QSIMPLEQ_INSERT_TAIL(bs_queue, bs_entry, entry);
-
-    bs_entry->state.bs = bs;
-    bs_entry->state.flags = flags;
-
-    return bs_queue;
-}
-
-/*
- * Reopen multiple BlockDriverStates atomically & transactionally.
- *
- * The queue passed in (bs_queue) must have been built up previous
- * via bdrv_reopen_queue().
- *
- * Reopens all BDS specified in the queue, with the appropriate
- * flags.  All devices are prepared for reopen, and failure of any
- * device will cause all device changes to be abandonded, and intermediate
- * data cleaned up.
- *
- * If all devices prepare successfully, then the changes are committed
- * to all devices.
- *
- */
-int bdrv_reopen_multiple(BlockReopenQueue *bs_queue, Error **errp)
-{
-    int ret = -1;
-    BlockReopenQueueEntry *bs_entry, *next;
-    Error *local_err = NULL;
-
-    assert(bs_queue != NULL);
-
-    bdrv_drain_all();
-
-    QSIMPLEQ_FOREACH(bs_entry, bs_queue, entry) {
-        if (bdrv_reopen_prepare(&bs_entry->state, bs_queue, &local_err)) {
-            error_propagate(errp, local_err);
-            goto cleanup;
-        }
-        bs_entry->prepared = true;
-    }
-
-    /* If we reach this point, we have success and just need to apply the
-     * changes
-     */
-    QSIMPLEQ_FOREACH(bs_entry, bs_queue, entry) {
-        bdrv_reopen_commit(&bs_entry->state);
-    }
-
-    ret = 0;
-
-cleanup:
-    QSIMPLEQ_FOREACH_SAFE(bs_entry, bs_queue, entry, next) {
-        if (ret && bs_entry->prepared) {
-            bdrv_reopen_abort(&bs_entry->state);
-        }
-        g_free(bs_entry);
-    }
-    g_free(bs_queue);
-    return ret;
-}
-
-
-/* Reopen a single BlockDriverState with the specified flags. */
-int bdrv_reopen(BlockDriverState *bs, int bdrv_flags, Error **errp)
-{
-    int ret = -1;
-    Error *local_err = NULL;
-    BlockReopenQueue *queue = bdrv_reopen_queue(NULL, bs, bdrv_flags);
-
-    ret = bdrv_reopen_multiple(queue, &local_err);
-    if (local_err != NULL) {
-        error_propagate(errp, local_err);
-    }
-    return ret;
-}
-
-
-/*
- * Prepares a BlockDriverState for reopen. All changes are staged in the
- * 'opaque' field of the BDRVReopenState, which is used and allocated by
- * the block driver layer .bdrv_reopen_prepare()
- *
- * bs is the BlockDriverState to reopen
- * flags are the new open flags
- * queue is the reopen queue
- *
- * Returns 0 on success, non-zero on error.  On error errp will be set
- * as well.
- *
- * On failure, bdrv_reopen_abort() will be called to clean up any data.
- * It is the responsibility of the caller to then call the abort() or
- * commit() for any other BDS that have been left in a prepare() state
- *
- */
-int bdrv_reopen_prepare(BDRVReopenState *reopen_state, BlockReopenQueue *queue,
-                        Error **errp)
-{
-    int ret = -1;
-    Error *local_err = NULL;
-    BlockDriver *drv;
-
-    assert(reopen_state != NULL);
-    assert(reopen_state->bs->drv != NULL);
-    drv = reopen_state->bs->drv;
-
-    /* if we are to stay read-only, do not allow permission change
-     * to r/w */
-    if (!(reopen_state->bs->open_flags & BDRV_O_ALLOW_RDWR) &&
-        reopen_state->flags & BDRV_O_RDWR) {
-        error_set(errp, QERR_DEVICE_IS_READ_ONLY,
-                  reopen_state->bs->device_name);
-        goto error;
-    }
-
-
-    ret = bdrv_flush(reopen_state->bs);
-    if (ret) {
-        error_set(errp, ERROR_CLASS_GENERIC_ERROR, "Error (%s) flushing drive",
-                  strerror(-ret));
-        goto error;
-    }
-
-    if (drv->bdrv_reopen_prepare) {
-        ret = drv->bdrv_reopen_prepare(reopen_state, queue, &local_err);
-        if (ret) {
-            if (local_err != NULL) {
-                error_propagate(errp, local_err);
-            } else {
-                error_setg(errp, "failed while preparing to reopen image '%s'",
-                           reopen_state->bs->filename);
-            }
-            goto error;
-        }
-    } else {
-        /* It is currently mandatory to have a bdrv_reopen_prepare()
-         * handler for each supported drv. */
-        error_set(errp, QERR_BLOCK_FORMAT_FEATURE_NOT_SUPPORTED,
-                  drv->format_name, reopen_state->bs->device_name,
-                 "reopening of file");
-        ret = -1;
-        goto error;
-    }
-
-    ret = 0;
-
-error:
-    return ret;
-}
-
-/*
- * Takes the staged changes for the reopen from bdrv_reopen_prepare(), and
- * makes them final by swapping the staging BlockDriverState contents into
- * the active BlockDriverState contents.
- */
-void bdrv_reopen_commit(BDRVReopenState *reopen_state)
-{
-    BlockDriver *drv;
-
-    assert(reopen_state != NULL);
-    drv = reopen_state->bs->drv;
-    assert(drv != NULL);
-
-    /* If there are any driver level actions to take */
-    if (drv->bdrv_reopen_commit) {
-        drv->bdrv_reopen_commit(reopen_state);
-    }
-
-    /* set BDS specific flags now */
-    reopen_state->bs->open_flags         = reopen_state->flags;
-    reopen_state->bs->enable_write_cache = !!(reopen_state->flags &
-                                              BDRV_O_CACHE_WB);
-    reopen_state->bs->read_only = !(reopen_state->flags & BDRV_O_RDWR);
-}
-
-/*
- * Abort the reopen, and delete and free the staged changes in
- * reopen_state
- */
-void bdrv_reopen_abort(BDRVReopenState *reopen_state)
-{
-    BlockDriver *drv;
-
-    assert(reopen_state != NULL);
-    drv = reopen_state->bs->drv;
-    assert(drv != NULL);
-
-    if (drv->bdrv_reopen_abort) {
-        drv->bdrv_reopen_abort(reopen_state);
-    }
-}
-
-
-void bdrv_close(BlockDriverState *bs)
-{
-    if (bs->job) {
-        block_job_cancel_sync(bs->job);
-    }
-    bdrv_drain_all(); /* complete I/O */
-    bdrv_flush(bs);
-    bdrv_drain_all(); /* in case flush left pending I/O */
-    notifier_list_notify(&bs->close_notifiers, bs);
-
-    if (bs->drv) {
-        if (bs->backing_hd) {
-            bdrv_delete(bs->backing_hd);
-            bs->backing_hd = NULL;
-        }
-        bs->drv->bdrv_close(bs);
-        g_free(bs->opaque);
-#ifdef _WIN32
-        if (bs->is_temporary) {
-            unlink(bs->filename);
-        }
-#endif
-        bs->opaque = NULL;
-        bs->drv = NULL;
-        bs->copy_on_read = 0;
-        bs->backing_file[0] = '\0';
-        bs->backing_format[0] = '\0';
-        bs->total_sectors = 0;
-        bs->encrypted = 0;
-        bs->valid_key = 0;
-        bs->sg = 0;
-        bs->growable = 0;
-        QDECREF(bs->options);
-        bs->options = NULL;
-
-        if (bs->file != NULL) {
-            bdrv_delete(bs->file);
-            bs->file = NULL;
-        }
-    }
-
-    bdrv_dev_change_media_cb(bs, false);
-
-    /*throttling disk I/O limits*/
-    if (bs->io_limits_enabled) {
-        bdrv_io_limits_disable(bs);
-    }
-}
-
-void bdrv_close_all(void)
-{
-    BlockDriverState *bs;
-
-    QTAILQ_FOREACH(bs, &bdrv_states, list) {
-        bdrv_close(bs);
-    }
-}
-
-/*
- * Wait for pending requests to complete across all BlockDriverStates
- *
- * This function does not flush data to disk, use bdrv_flush_all() for that
- * after calling this function.
- *
- * Note that completion of an asynchronous I/O operation can trigger any
- * number of other I/O operations on other devices---for example a coroutine
- * can be arbitrarily complex and a constant flow of I/O can come until the
- * coroutine is complete.  Because of this, it is not possible to have a
- * function to drain a single device's I/O queue.
- */
-void bdrv_drain_all(void)
-{
-    BlockDriverState *bs;
-    bool busy;
-
-    do {
-        busy = qemu_aio_wait();
-
-        /* FIXME: We do not have timer support here, so this is effectively
-         * a busy wait.
-         */
-        QTAILQ_FOREACH(bs, &bdrv_states, list) {
-            if (!qemu_co_queue_empty(&bs->throttled_reqs)) {
-                qemu_co_queue_restart_all(&bs->throttled_reqs);
-                busy = true;
-            }
-        }
-    } while (busy);
-
-    /* If requests are still pending there is a bug somewhere */
-    QTAILQ_FOREACH(bs, &bdrv_states, list) {
-        assert(QLIST_EMPTY(&bs->tracked_requests));
-        assert(qemu_co_queue_empty(&bs->throttled_reqs));
-    }
-}
-
-/* make a BlockDriverState anonymous by removing from bdrv_state list.
-   Also, NULL terminate the device_name to prevent double remove */
-void bdrv_make_anon(BlockDriverState *bs)
-{
-    if (bs->device_name[0] != '\0') {
-        QTAILQ_REMOVE(&bdrv_states, bs, list);
-    }
-    bs->device_name[0] = '\0';
-}
-
-static void bdrv_rebind(BlockDriverState *bs)
-{
-    if (bs->drv && bs->drv->bdrv_rebind) {
-        bs->drv->bdrv_rebind(bs);
-    }
-}
-
-static void bdrv_move_feature_fields(BlockDriverState *bs_dest,
-                                     BlockDriverState *bs_src)
-{
-    /* move some fields that need to stay attached to the device */
-    bs_dest->open_flags         = bs_src->open_flags;
-
-    /* dev info */
-    bs_dest->dev_ops            = bs_src->dev_ops;
-    bs_dest->dev_opaque         = bs_src->dev_opaque;
-    bs_dest->dev                = bs_src->dev;
-    bs_dest->buffer_alignment   = bs_src->buffer_alignment;
-    bs_dest->copy_on_read       = bs_src->copy_on_read;
-
-    bs_dest->enable_write_cache = bs_src->enable_write_cache;
-
-    /* i/o timing parameters */
-    bs_dest->slice_start        = bs_src->slice_start;
-    bs_dest->slice_end          = bs_src->slice_end;
-    bs_dest->slice_submitted    = bs_src->slice_submitted;
-    bs_dest->io_limits          = bs_src->io_limits;
-    bs_dest->throttled_reqs     = bs_src->throttled_reqs;
-    bs_dest->block_timer        = bs_src->block_timer;
-    bs_dest->io_limits_enabled  = bs_src->io_limits_enabled;
-
-    /* r/w error */
-    bs_dest->on_read_error      = bs_src->on_read_error;
-    bs_dest->on_write_error     = bs_src->on_write_error;
-
-    /* i/o status */
-    bs_dest->iostatus_enabled   = bs_src->iostatus_enabled;
-    bs_dest->iostatus           = bs_src->iostatus;
-
-    /* dirty bitmap */
-    bs_dest->dirty_bitmap       = bs_src->dirty_bitmap;
-
-    /* job */
-    bs_dest->in_use             = bs_src->in_use;
-    bs_dest->job                = bs_src->job;
-
-    /* keep the same entry in bdrv_states */
-    pstrcpy(bs_dest->device_name, sizeof(bs_dest->device_name),
-            bs_src->device_name);
-    bs_dest->list = bs_src->list;
-}
-
-/*
- * Swap bs contents for two image chains while they are live,
- * while keeping required fields on the BlockDriverState that is
- * actually attached to a device.
- *
- * This will modify the BlockDriverState fields, and swap contents
- * between bs_new and bs_old. Both bs_new and bs_old are modified.
- *
- * bs_new is required to be anonymous.
- *
- * This function does not create any image files.
- */
-void bdrv_swap(BlockDriverState *bs_new, BlockDriverState *bs_old)
-{
-    BlockDriverState tmp;
-
-    /* bs_new must be anonymous and shouldn't have anything fancy enabled */
-    assert(bs_new->device_name[0] == '\0');
-    assert(bs_new->dirty_bitmap == NULL);
-    assert(bs_new->job == NULL);
-    assert(bs_new->dev == NULL);
-    assert(bs_new->in_use == 0);
-    assert(bs_new->io_limits_enabled == false);
-    assert(bs_new->block_timer == NULL);
-
-    tmp = *bs_new;
-    *bs_new = *bs_old;
-    *bs_old = tmp;
-
-    /* there are some fields that should not be swapped, move them back */
-    bdrv_move_feature_fields(&tmp, bs_old);
-    bdrv_move_feature_fields(bs_old, bs_new);
-    bdrv_move_feature_fields(bs_new, &tmp);
-
-    /* bs_new shouldn't be in bdrv_states even after the swap!  */
-    assert(bs_new->device_name[0] == '\0');
-
-    /* Check a few fields that should remain attached to the device */
-    assert(bs_new->dev == NULL);
-    assert(bs_new->job == NULL);
-    assert(bs_new->in_use == 0);
-    assert(bs_new->io_limits_enabled == false);
-    assert(bs_new->block_timer == NULL);
-
-    bdrv_rebind(bs_new);
-    bdrv_rebind(bs_old);
-}
-
-/*
- * Add new bs contents at the top of an image chain while the chain is
- * live, while keeping required fields on the top layer.
- *
- * This will modify the BlockDriverState fields, and swap contents
- * between bs_new and bs_top. Both bs_new and bs_top are modified.
- *
- * bs_new is required to be anonymous.
- *
- * This function does not create any image files.
- */
-void bdrv_append(BlockDriverState *bs_new, BlockDriverState *bs_top)
-{
-    bdrv_swap(bs_new, bs_top);
-
-    /* The contents of 'tmp' will become bs_top, as we are
-     * swapping bs_new and bs_top contents. */
-    bs_top->backing_hd = bs_new;
-    bs_top->open_flags &= ~BDRV_O_NO_BACKING;
-    pstrcpy(bs_top->backing_file, sizeof(bs_top->backing_file),
-            bs_new->filename);
-    pstrcpy(bs_top->backing_format, sizeof(bs_top->backing_format),
-            bs_new->drv ? bs_new->drv->format_name : "");
-}
-
-void bdrv_delete(BlockDriverState *bs)
-{
-    assert(!bs->dev);
-    assert(!bs->job);
-    assert(!bs->in_use);
-
-    /* remove from list, if necessary */
-    bdrv_make_anon(bs);
-
-    bdrv_close(bs);
-
-    g_free(bs);
-}
-
-int bdrv_attach_dev(BlockDriverState *bs, void *dev)
-/* TODO change to DeviceState *dev when all users are qdevified */
-{
-    if (bs->dev) {
-        return -EBUSY;
-    }
-    bs->dev = dev;
-    bdrv_iostatus_reset(bs);
-    return 0;
-}
-
-/* TODO qdevified devices don't use this, remove when devices are qdevified */
-void bdrv_attach_dev_nofail(BlockDriverState *bs, void *dev)
-{
-    if (bdrv_attach_dev(bs, dev) < 0) {
-        abort();
-    }
-}
-
-void bdrv_detach_dev(BlockDriverState *bs, void *dev)
-/* TODO change to DeviceState *dev when all users are qdevified */
-{
-    assert(bs->dev == dev);
-    bs->dev = NULL;
-    bs->dev_ops = NULL;
-    bs->dev_opaque = NULL;
-    bs->buffer_alignment = 512;
-}
-
-/* TODO change to return DeviceState * when all users are qdevified */
-void *bdrv_get_attached_dev(BlockDriverState *bs)
-{
-    return bs->dev;
-}
-
-void bdrv_set_dev_ops(BlockDriverState *bs, const BlockDevOps *ops,
-                      void *opaque)
-{
-    bs->dev_ops = ops;
-    bs->dev_opaque = opaque;
-}
-
-void bdrv_emit_qmp_error_event(const BlockDriverState *bdrv,
-                               enum MonitorEvent ev,
-                               BlockErrorAction action, bool is_read)
-{
-    QObject *data;
-    const char *action_str;
-
-    switch (action) {
-    case BDRV_ACTION_REPORT:
-        action_str = "report";
-        break;
-    case BDRV_ACTION_IGNORE:
-        action_str = "ignore";
-        break;
-    case BDRV_ACTION_STOP:
-        action_str = "stop";
-        break;
-    default:
-        abort();
-    }
-
-    data = qobject_from_jsonf("{ 'device': %s, 'action': %s, 'operation': %s }",
-                              bdrv->device_name,
-                              action_str,
-                              is_read ? "read" : "write");
-    monitor_protocol_event(ev, data);
-
-    qobject_decref(data);
-}
-
-static void bdrv_emit_qmp_eject_event(BlockDriverState *bs, bool ejected)
-{
-    QObject *data;
-
-    data = qobject_from_jsonf("{ 'device': %s, 'tray-open': %i }",
-                              bdrv_get_device_name(bs), ejected);
-    monitor_protocol_event(QEVENT_DEVICE_TRAY_MOVED, data);
-
-    qobject_decref(data);
-}
-
-static void bdrv_dev_change_media_cb(BlockDriverState *bs, bool load)
-{
-    if (bs->dev_ops && bs->dev_ops->change_media_cb) {
-        bool tray_was_closed = !bdrv_dev_is_tray_open(bs);
-        bs->dev_ops->change_media_cb(bs->dev_opaque, load);
-        if (tray_was_closed) {
-            /* tray open */
-            bdrv_emit_qmp_eject_event(bs, true);
-        }
-        if (load) {
-            /* tray close */
-            bdrv_emit_qmp_eject_event(bs, false);
-        }
-    }
-}
-
-bool bdrv_dev_has_removable_media(BlockDriverState *bs)
-{
-    return !bs->dev || (bs->dev_ops && bs->dev_ops->change_media_cb);
-}
-
-void bdrv_dev_eject_request(BlockDriverState *bs, bool force)
-{
-    if (bs->dev_ops && bs->dev_ops->eject_request_cb) {
-        bs->dev_ops->eject_request_cb(bs->dev_opaque, force);
-    }
-}
-
-bool bdrv_dev_is_tray_open(BlockDriverState *bs)
-{
-    if (bs->dev_ops && bs->dev_ops->is_tray_open) {
-        return bs->dev_ops->is_tray_open(bs->dev_opaque);
-    }
-    return false;
-}
-
-static void bdrv_dev_resize_cb(BlockDriverState *bs)
-{
-    if (bs->dev_ops && bs->dev_ops->resize_cb) {
-        bs->dev_ops->resize_cb(bs->dev_opaque);
-    }
-}
-
-bool bdrv_dev_is_medium_locked(BlockDriverState *bs)
-{
-    if (bs->dev_ops && bs->dev_ops->is_medium_locked) {
-        return bs->dev_ops->is_medium_locked(bs->dev_opaque);
-    }
-    return false;
-}
-
-/*
- * Run consistency checks on an image
- *
- * Returns 0 if the check could be completed (it doesn't mean that the image is
- * free of errors) or -errno when an internal error occurred. The results of the
- * check are stored in res.
- */
-int bdrv_check(BlockDriverState *bs, BdrvCheckResult *res, BdrvCheckMode fix)
-{
-    if (bs->drv->bdrv_check == NULL) {
-        return -ENOTSUP;
-    }
-
-    memset(res, 0, sizeof(*res));
-    return bs->drv->bdrv_check(bs, res, fix);
-}
-
-#define COMMIT_BUF_SECTORS 2048
-
-/* commit COW file into the raw image */
-int bdrv_commit(BlockDriverState *bs)
-{
-    BlockDriver *drv = bs->drv;
-    int64_t sector, total_sectors;
-    int n, ro, open_flags;
-    int ret = 0;
-    uint8_t *buf;
-    char filename[PATH_MAX];
-
-    if (!drv)
-        return -ENOMEDIUM;
-    
-    if (!bs->backing_hd) {
-        return -ENOTSUP;
-    }
-
-    if (bdrv_in_use(bs) || bdrv_in_use(bs->backing_hd)) {
-        return -EBUSY;
-    }
-
-    ro = bs->backing_hd->read_only;
-    /* Use pstrcpy (not strncpy): filename must be NUL-terminated. */
-    pstrcpy(filename, sizeof(filename), bs->backing_hd->filename);
-    open_flags =  bs->backing_hd->open_flags;
-
-    if (ro) {
-        if (bdrv_reopen(bs->backing_hd, open_flags | BDRV_O_RDWR, NULL)) {
-            return -EACCES;
-        }
-    }
-
-    total_sectors = bdrv_getlength(bs) >> BDRV_SECTOR_BITS;
-    buf = g_malloc(COMMIT_BUF_SECTORS * BDRV_SECTOR_SIZE);
-
-    for (sector = 0; sector < total_sectors; sector += n) {
-        if (bdrv_is_allocated(bs, sector, COMMIT_BUF_SECTORS, &n)) {
-
-            if (bdrv_read(bs, sector, buf, n) != 0) {
-                ret = -EIO;
-                goto ro_cleanup;
-            }
-
-            if (bdrv_write(bs->backing_hd, sector, buf, n) != 0) {
-                ret = -EIO;
-                goto ro_cleanup;
-            }
-        }
-    }
-
-    if (drv->bdrv_make_empty) {
-        ret = drv->bdrv_make_empty(bs);
-        bdrv_flush(bs);
-    }
-
-    /*
-     * Make sure all data we wrote to the backing device is actually
-     * stable on disk.
-     */
-    if (bs->backing_hd)
-        bdrv_flush(bs->backing_hd);
-
-ro_cleanup:
-    g_free(buf);
-
-    if (ro) {
-        /* ignoring error return here */
-        bdrv_reopen(bs->backing_hd, open_flags & ~BDRV_O_RDWR, NULL);
-    }
-
-    return ret;
-}
-
-int bdrv_commit_all(void)
-{
-    BlockDriverState *bs;
-
-    QTAILQ_FOREACH(bs, &bdrv_states, list) {
-        if (bs->drv && bs->backing_hd) {
-            int ret = bdrv_commit(bs);
-            if (ret < 0) {
-                return ret;
-            }
-        }
-    }
-    return 0;
-}
-
-/**
- * Remove an active request from the tracked requests list
- *
- * This function should be called when a tracked request is completing.
- */
-static void tracked_request_end(BdrvTrackedRequest *req)
-{
-    QLIST_REMOVE(req, list);
-    qemu_co_queue_restart_all(&req->wait_queue);
-}
-
-/**
- * Add an active request to the tracked requests list
- */
-static void tracked_request_begin(BdrvTrackedRequest *req,
-                                  BlockDriverState *bs,
-                                  int64_t sector_num,
-                                  int nb_sectors, bool is_write)
-{
-    *req = (BdrvTrackedRequest){
-        .bs = bs,
-        .sector_num = sector_num,
-        .nb_sectors = nb_sectors,
-        .is_write = is_write,
-        .co = qemu_coroutine_self(),
-    };
-
-    qemu_co_queue_init(&req->wait_queue);
-
-    QLIST_INSERT_HEAD(&bs->tracked_requests, req, list);
-}
-
-/**
- * Round a region to cluster boundaries
- */
-void bdrv_round_to_clusters(BlockDriverState *bs,
-                            int64_t sector_num, int nb_sectors,
-                            int64_t *cluster_sector_num,
-                            int *cluster_nb_sectors)
-{
-    BlockDriverInfo bdi;
-
-    if (bdrv_get_info(bs, &bdi) < 0 || bdi.cluster_size == 0) {
-        *cluster_sector_num = sector_num;
-        *cluster_nb_sectors = nb_sectors;
-    } else {
-        int64_t c = bdi.cluster_size / BDRV_SECTOR_SIZE;
-        *cluster_sector_num = QEMU_ALIGN_DOWN(sector_num, c);
-        *cluster_nb_sectors = QEMU_ALIGN_UP(sector_num - *cluster_sector_num +
-                                            nb_sectors, c);
-    }
-}
-
-static bool tracked_request_overlaps(BdrvTrackedRequest *req,
-                                     int64_t sector_num, int nb_sectors) {
-    /*        aaaa   bbbb */
-    if (sector_num >= req->sector_num + req->nb_sectors) {
-        return false;
-    }
-    /* bbbb   aaaa        */
-    if (req->sector_num >= sector_num + nb_sectors) {
-        return false;
-    }
-    return true;
-}
-
-static void coroutine_fn wait_for_overlapping_requests(BlockDriverState *bs,
-        int64_t sector_num, int nb_sectors)
-{
-    BdrvTrackedRequest *req;
-    int64_t cluster_sector_num;
-    int cluster_nb_sectors;
-    bool retry;
-
-    /* If we touch the same cluster it counts as an overlap.  This guarantees
-     * that allocating writes will be serialized and not race with each other
-     * for the same cluster.  For example, in copy-on-read it ensures that the
-     * CoR read and write operations are atomic and guest writes cannot
-     * interleave between them.
-     */
-    bdrv_round_to_clusters(bs, sector_num, nb_sectors,
-                           &cluster_sector_num, &cluster_nb_sectors);
-
-    do {
-        retry = false;
-        QLIST_FOREACH(req, &bs->tracked_requests, list) {
-            if (tracked_request_overlaps(req, cluster_sector_num,
-                                         cluster_nb_sectors)) {
-                /* Hitting this means there was a reentrant request, for
-                 * example, a block driver issuing nested requests.  This must
-                 * never happen since it means deadlock.
-                 */
-                assert(qemu_coroutine_self() != req->co);
-
-                qemu_co_queue_wait(&req->wait_queue);
-                retry = true;
-                break;
-            }
-        }
-    } while (retry);
-}
-
-/*
- * Return values:
- * 0        - success
- * -EINVAL  - backing format specified, but no file
- * -ENOSPC  - can't update the backing file because no space is left in the
- *            image file header
- * -ENOTSUP - format driver doesn't support changing the backing file
- */
-int bdrv_change_backing_file(BlockDriverState *bs,
-    const char *backing_file, const char *backing_fmt)
-{
-    BlockDriver *drv = bs->drv;
-    int ret;
-
-    /* Backing file format doesn't make sense without a backing file */
-    if (backing_fmt && !backing_file) {
-        return -EINVAL;
-    }
-
-    if (drv->bdrv_change_backing_file != NULL) {
-        ret = drv->bdrv_change_backing_file(bs, backing_file, backing_fmt);
-    } else {
-        ret = -ENOTSUP;
-    }
-
-    if (ret == 0) {
-        pstrcpy(bs->backing_file, sizeof(bs->backing_file), backing_file ?: "");
-        pstrcpy(bs->backing_format, sizeof(bs->backing_format), backing_fmt ?: "");
-    }
-    return ret;
-}
-
-/*
- * Finds the image layer in the chain that has 'bs' as its backing file.
- *
- * active is the current topmost image.
- *
- * Returns NULL if bs is not found in active's image chain,
- * or if active == bs.
- */
-BlockDriverState *bdrv_find_overlay(BlockDriverState *active,
-                                    BlockDriverState *bs)
-{
-    BlockDriverState *overlay = NULL;
-    BlockDriverState *intermediate;
-
-    assert(active != NULL);
-    assert(bs != NULL);
-
-    /* if bs is the same as active, then by definition it has no overlay
-     */
-    if (active == bs) {
-        return NULL;
-    }
-
-    intermediate = active;
-    while (intermediate->backing_hd) {
-        if (intermediate->backing_hd == bs) {
-            overlay = intermediate;
-            break;
-        }
-        intermediate = intermediate->backing_hd;
-    }
-
-    return overlay;
-}
-
-typedef struct BlkIntermediateStates {
-    BlockDriverState *bs;
-    QSIMPLEQ_ENTRY(BlkIntermediateStates) entry;
-} BlkIntermediateStates;
-
-
-/*
- * Drops images above 'base' up to and including 'top', and sets the image
- * above 'top' to have base as its backing file.
- *
- * Requires that the overlay to 'top' is opened r/w, so that the backing file
- * information in 'bs' can be properly updated.
- *
- * E.g., this will convert the following chain:
- * bottom <- base <- intermediate <- top <- active
- *
- * to
- *
- * bottom <- base <- active
- *
- * It is allowed for bottom==base, in which case it converts:
- *
- * base <- intermediate <- top <- active
- *
- * to
- *
- * base <- active
- *
- * Error conditions:
- *  if active == top, that is considered an error
- *
- */
-int bdrv_drop_intermediate(BlockDriverState *active, BlockDriverState *top,
-                           BlockDriverState *base)
-{
-    BlockDriverState *intermediate;
-    BlockDriverState *base_bs = NULL;
-    BlockDriverState *new_top_bs = NULL;
-    BlkIntermediateStates *intermediate_state, *next;
-    int ret = -EIO;
-
-    QSIMPLEQ_HEAD(states_to_delete, BlkIntermediateStates) states_to_delete;
-    QSIMPLEQ_INIT(&states_to_delete);
-
-    if (!top->drv || !base->drv) {
-        goto exit;
-    }
-
-    new_top_bs = bdrv_find_overlay(active, top);
-
-    if (new_top_bs == NULL) {
-        /* we could not find the image above 'top', this is an error */
-        goto exit;
-    }
-
-    /* special case of new_top_bs->backing_hd already pointing to base - nothing
-     * to do, no intermediate images */
-    if (new_top_bs->backing_hd == base) {
-        ret = 0;
-        goto exit;
-    }
-
-    intermediate = top;
-
-    /* now we will go down through the list, and add each BDS we find
-     * into our deletion queue, until we hit the 'base'
-     */
-    while (intermediate) {
-        intermediate_state = g_malloc0(sizeof(BlkIntermediateStates));
-        intermediate_state->bs = intermediate;
-        QSIMPLEQ_INSERT_TAIL(&states_to_delete, intermediate_state, entry);
-
-        if (intermediate->backing_hd == base) {
-            base_bs = intermediate->backing_hd;
-            break;
-        }
-        intermediate = intermediate->backing_hd;
-    }
-    if (base_bs == NULL) {
-        /* something went wrong, we did not end at the base. safely
-         * unravel everything, and exit with error */
-        goto exit;
-    }
-
-    /* success - we can delete the intermediate states, and link top->base */
-    ret = bdrv_change_backing_file(new_top_bs, base_bs->filename,
-                                   base_bs->drv ? base_bs->drv->format_name : "");
-    if (ret) {
-        goto exit;
-    }
-    new_top_bs->backing_hd = base_bs;
-
-
-    QSIMPLEQ_FOREACH_SAFE(intermediate_state, &states_to_delete, entry, next) {
-        /* so that bdrv_close() does not recursively close the chain */
-        intermediate_state->bs->backing_hd = NULL;
-        bdrv_delete(intermediate_state->bs);
-    }
-    ret = 0;
-
-exit:
-    QSIMPLEQ_FOREACH_SAFE(intermediate_state, &states_to_delete, entry, next) {
-        g_free(intermediate_state);
-    }
-    return ret;
-}
-
-
-static int bdrv_check_byte_request(BlockDriverState *bs, int64_t offset,
-                                   size_t size)
-{
-    int64_t len;
-
-    if (!bdrv_is_inserted(bs))
-        return -ENOMEDIUM;
-
-    if (bs->growable)
-        return 0;
-
-    len = bdrv_getlength(bs);
-
-    if (offset < 0)
-        return -EIO;
-
-    if ((offset > len) || (len - offset < size))
-        return -EIO;
-
-    return 0;
-}
-
-static int bdrv_check_request(BlockDriverState *bs, int64_t sector_num,
-                              int nb_sectors)
-{
-    return bdrv_check_byte_request(bs, sector_num * BDRV_SECTOR_SIZE,
-                                   nb_sectors * BDRV_SECTOR_SIZE);
-}
-
-typedef struct RwCo {
-    BlockDriverState *bs;
-    int64_t sector_num;
-    int nb_sectors;
-    QEMUIOVector *qiov;
-    bool is_write;
-    int ret;
-} RwCo;
-
-static void coroutine_fn bdrv_rw_co_entry(void *opaque)
-{
-    RwCo *rwco = opaque;
-
-    if (!rwco->is_write) {
-        rwco->ret = bdrv_co_do_readv(rwco->bs, rwco->sector_num,
-                                     rwco->nb_sectors, rwco->qiov, 0);
-    } else {
-        rwco->ret = bdrv_co_do_writev(rwco->bs, rwco->sector_num,
-                                      rwco->nb_sectors, rwco->qiov, 0);
-    }
-}
-
-/*
- * Process a vectored synchronous request using coroutines
- */
-static int bdrv_rwv_co(BlockDriverState *bs, int64_t sector_num,
-                       QEMUIOVector *qiov, bool is_write)
-{
-    Coroutine *co;
-    RwCo rwco = {
-        .bs = bs,
-        .sector_num = sector_num,
-        .nb_sectors = qiov->size >> BDRV_SECTOR_BITS,
-        .qiov = qiov,
-        .is_write = is_write,
-        .ret = NOT_DONE,
-    };
-    assert((qiov->size & (BDRV_SECTOR_SIZE - 1)) == 0);
-
-    /**
-     * In sync call context, when the vcpu is blocked, this throttling timer
-     * will not fire; so the I/O throttling function has to be disabled here
-     * if it has been enabled.
-     */
-    if (bs->io_limits_enabled) {
-        fprintf(stderr, "Disabling I/O throttling on '%s' due "
-                        "to synchronous I/O.\n", bdrv_get_device_name(bs));
-        bdrv_io_limits_disable(bs);
-    }
-
-    if (qemu_in_coroutine()) {
-        /* Fast-path if already in coroutine context */
-        bdrv_rw_co_entry(&rwco);
-    } else {
-        co = qemu_coroutine_create(bdrv_rw_co_entry);
-        qemu_coroutine_enter(co, &rwco);
-        while (rwco.ret == NOT_DONE) {
-            qemu_aio_wait();
-        }
-    }
-    return rwco.ret;
-}
-
-/*
- * Process a synchronous request using coroutines
- */
-static int bdrv_rw_co(BlockDriverState *bs, int64_t sector_num, uint8_t *buf,
-                      int nb_sectors, bool is_write)
-{
-    QEMUIOVector qiov;
-    struct iovec iov = {
-        .iov_base = (void *)buf,
-        .iov_len = nb_sectors * BDRV_SECTOR_SIZE,
-    };
-
-    qemu_iovec_init_external(&qiov, &iov, 1);
-    return bdrv_rwv_co(bs, sector_num, &qiov, is_write);
-}
-
-/* return < 0 if error. See bdrv_write() for the return codes */
-int bdrv_read(BlockDriverState *bs, int64_t sector_num,
-              uint8_t *buf, int nb_sectors)
-{
-    return bdrv_rw_co(bs, sector_num, buf, nb_sectors, false);
-}
-
-/* Just like bdrv_read(), but with I/O throttling temporarily disabled */
-int bdrv_read_unthrottled(BlockDriverState *bs, int64_t sector_num,
-                          uint8_t *buf, int nb_sectors)
-{
-    bool enabled;
-    int ret;
-
-    enabled = bs->io_limits_enabled;
-    bs->io_limits_enabled = false;
-    ret = bdrv_read(bs, 0, buf, 1);
-    bs->io_limits_enabled = enabled;
-    return ret;
-}
-
-/* Return < 0 if error. Important errors are:
-  -EIO         generic I/O error (may happen for all errors)
-  -ENOMEDIUM   No media inserted.
-  -EINVAL      Invalid sector number or nb_sectors
-  -EACCES      Trying to write a read-only device
-*/
-int bdrv_write(BlockDriverState *bs, int64_t sector_num,
-               const uint8_t *buf, int nb_sectors)
-{
-    return bdrv_rw_co(bs, sector_num, (uint8_t *)buf, nb_sectors, true);
-}
-
-int bdrv_writev(BlockDriverState *bs, int64_t sector_num, QEMUIOVector *qiov)
-{
-    return bdrv_rwv_co(bs, sector_num, qiov, true);
-}
-
-int bdrv_pread(BlockDriverState *bs, int64_t offset,
-               void *buf, int count1)
-{
-    uint8_t tmp_buf[BDRV_SECTOR_SIZE];
-    int len, nb_sectors, count;
-    int64_t sector_num;
-    int ret;
-
-    count = count1;
-    /* first read to align to sector start */
-    len = (BDRV_SECTOR_SIZE - offset) & (BDRV_SECTOR_SIZE - 1);
-    if (len > count)
-        len = count;
-    sector_num = offset >> BDRV_SECTOR_BITS;
-    if (len > 0) {
-        if ((ret = bdrv_read(bs, sector_num, tmp_buf, 1)) < 0)
-            return ret;
-        memcpy(buf, tmp_buf + (offset & (BDRV_SECTOR_SIZE - 1)), len);
-        count -= len;
-        if (count == 0)
-            return count1;
-        sector_num++;
-        buf += len;
-    }
-
-    /* read the sectors "in place" */
-    nb_sectors = count >> BDRV_SECTOR_BITS;
-    if (nb_sectors > 0) {
-        if ((ret = bdrv_read(bs, sector_num, buf, nb_sectors)) < 0)
-            return ret;
-        sector_num += nb_sectors;
-        len = nb_sectors << BDRV_SECTOR_BITS;
-        buf += len;
-        count -= len;
-    }
-
-    /* add data from the last sector */
-    if (count > 0) {
-        if ((ret = bdrv_read(bs, sector_num, tmp_buf, 1)) < 0)
-            return ret;
-        memcpy(buf, tmp_buf, count);
-    }
-    return count1;
-}
-
-int bdrv_pwritev(BlockDriverState *bs, int64_t offset, QEMUIOVector *qiov)
-{
-    uint8_t tmp_buf[BDRV_SECTOR_SIZE];
-    int len, nb_sectors, count;
-    int64_t sector_num;
-    int ret;
-
-    count = qiov->size;
-
-    /* first write to align to sector start */
-    len = (BDRV_SECTOR_SIZE - offset) & (BDRV_SECTOR_SIZE - 1);
-    if (len > count)
-        len = count;
-    sector_num = offset >> BDRV_SECTOR_BITS;
-    if (len > 0) {
-        if ((ret = bdrv_read(bs, sector_num, tmp_buf, 1)) < 0)
-            return ret;
-        qemu_iovec_to_buf(qiov, 0, tmp_buf + (offset & (BDRV_SECTOR_SIZE - 1)),
-                          len);
-        if ((ret = bdrv_write(bs, sector_num, tmp_buf, 1)) < 0)
-            return ret;
-        count -= len;
-        if (count == 0)
-            return qiov->size;
-        sector_num++;
-    }
-
-    /* write the sectors "in place" */
-    nb_sectors = count >> BDRV_SECTOR_BITS;
-    if (nb_sectors > 0) {
-        QEMUIOVector qiov_inplace;
-
-        qemu_iovec_init(&qiov_inplace, qiov->niov);
-        qemu_iovec_concat(&qiov_inplace, qiov, len,
-                          nb_sectors << BDRV_SECTOR_BITS);
-        ret = bdrv_writev(bs, sector_num, &qiov_inplace);
-        qemu_iovec_destroy(&qiov_inplace);
-        if (ret < 0) {
-            return ret;
-        }
-
-        sector_num += nb_sectors;
-        len = nb_sectors << BDRV_SECTOR_BITS;
-        count -= len;
-    }
-
-    /* add data from the last sector */
-    if (count > 0) {
-        if ((ret = bdrv_read(bs, sector_num, tmp_buf, 1)) < 0)
-            return ret;
-        qemu_iovec_to_buf(qiov, qiov->size - count, tmp_buf, count);
-        if ((ret = bdrv_write(bs, sector_num, tmp_buf, 1)) < 0)
-            return ret;
-    }
-    return qiov->size;
-}
-
-int bdrv_pwrite(BlockDriverState *bs, int64_t offset,
-                const void *buf, int count1)
-{
-    QEMUIOVector qiov;
-    struct iovec iov = {
-        .iov_base   = (void *) buf,
-        .iov_len    = count1,
-    };
-
-    qemu_iovec_init_external(&qiov, &iov, 1);
-    return bdrv_pwritev(bs, offset, &qiov);
-}
-
-/*
- * Writes to the file and ensures that no writes are reordered across this
- * request (acts as a barrier)
- *
- * Returns 0 on success, -errno in error cases.
- */
-int bdrv_pwrite_sync(BlockDriverState *bs, int64_t offset,
-    const void *buf, int count)
-{
-    int ret;
-
-    ret = bdrv_pwrite(bs, offset, buf, count);
-    if (ret < 0) {
-        return ret;
-    }
-
-    /* No flush needed for cache modes that already do it */
-    if (bs->enable_write_cache) {
-        bdrv_flush(bs);
-    }
-
-    return 0;
-}
-
-static int coroutine_fn bdrv_co_do_copy_on_readv(BlockDriverState *bs,
-        int64_t sector_num, int nb_sectors, QEMUIOVector *qiov)
-{
-    /* Perform I/O through a temporary buffer so that users who scribble over
-     * their read buffer while the operation is in progress do not end up
-     * modifying the image file.  This is critical for zero-copy guest I/O
-     * where anything might happen inside guest memory.
-     */
-    void *bounce_buffer;
-
-    BlockDriver *drv = bs->drv;
-    struct iovec iov;
-    QEMUIOVector bounce_qiov;
-    int64_t cluster_sector_num;
-    int cluster_nb_sectors;
-    size_t skip_bytes;
-    int ret;
-
-    /* Cover entire cluster so no additional backing file I/O is required when
-     * allocating cluster in the image file.
-     */
-    bdrv_round_to_clusters(bs, sector_num, nb_sectors,
-                           &cluster_sector_num, &cluster_nb_sectors);
-
-    trace_bdrv_co_do_copy_on_readv(bs, sector_num, nb_sectors,
-                                   cluster_sector_num, cluster_nb_sectors);
-
-    iov.iov_len = cluster_nb_sectors * BDRV_SECTOR_SIZE;
-    iov.iov_base = bounce_buffer = qemu_blockalign(bs, iov.iov_len);
-    qemu_iovec_init_external(&bounce_qiov, &iov, 1);
-
-    ret = drv->bdrv_co_readv(bs, cluster_sector_num, cluster_nb_sectors,
-                             &bounce_qiov);
-    if (ret < 0) {
-        goto err;
-    }
-
-    if (drv->bdrv_co_write_zeroes &&
-        buffer_is_zero(bounce_buffer, iov.iov_len)) {
-        ret = bdrv_co_do_write_zeroes(bs, cluster_sector_num,
-                                      cluster_nb_sectors);
-    } else {
-        /* This does not change the data on the disk, it is not necessary
-         * to flush even in cache=writethrough mode.
-         */
-        ret = drv->bdrv_co_writev(bs, cluster_sector_num, cluster_nb_sectors,
-                                  &bounce_qiov);
-    }
-
-    if (ret < 0) {
-        /* It might be okay to ignore write errors for guest requests.  If this
-         * is a deliberate copy-on-read then we don't want to ignore the error.
-         * Simply report it in all cases.
-         */
-        goto err;
-    }
-
-    skip_bytes = (sector_num - cluster_sector_num) * BDRV_SECTOR_SIZE;
-    qemu_iovec_from_buf(qiov, 0, bounce_buffer + skip_bytes,
-                        nb_sectors * BDRV_SECTOR_SIZE);
-
-err:
-    qemu_vfree(bounce_buffer);
-    return ret;
-}
-
-/*
- * Handle a read request in coroutine context
- */
-static int coroutine_fn bdrv_co_do_readv(BlockDriverState *bs,
-    int64_t sector_num, int nb_sectors, QEMUIOVector *qiov,
-    BdrvRequestFlags flags)
-{
-    BlockDriver *drv = bs->drv;
-    BdrvTrackedRequest req;
-    int ret;
-
-    if (!drv) {
-        return -ENOMEDIUM;
-    }
-    if (bdrv_check_request(bs, sector_num, nb_sectors)) {
-        return -EIO;
-    }
-
-    /* throttling disk read I/O */
-    if (bs->io_limits_enabled) {
-        bdrv_io_limits_intercept(bs, false, nb_sectors);
-    }
-
-    if (bs->copy_on_read) {
-        flags |= BDRV_REQ_COPY_ON_READ;
-    }
-    if (flags & BDRV_REQ_COPY_ON_READ) {
-        bs->copy_on_read_in_flight++;
-    }
-
-    if (bs->copy_on_read_in_flight) {
-        wait_for_overlapping_requests(bs, sector_num, nb_sectors);
-    }
-
-    tracked_request_begin(&req, bs, sector_num, nb_sectors, false);
-
-    if (flags & BDRV_REQ_COPY_ON_READ) {
-        int pnum;
-
-        ret = bdrv_co_is_allocated(bs, sector_num, nb_sectors, &pnum);
-        if (ret < 0) {
-            goto out;
-        }
-
-        if (!ret || pnum != nb_sectors) {
-            ret = bdrv_co_do_copy_on_readv(bs, sector_num, nb_sectors, qiov);
-            goto out;
-        }
-    }
-
-    ret = drv->bdrv_co_readv(bs, sector_num, nb_sectors, qiov);
-
-out:
-    tracked_request_end(&req);
-
-    if (flags & BDRV_REQ_COPY_ON_READ) {
-        bs->copy_on_read_in_flight--;
-    }
-
-    return ret;
-}
-
-int coroutine_fn bdrv_co_readv(BlockDriverState *bs, int64_t sector_num,
-    int nb_sectors, QEMUIOVector *qiov)
-{
-    trace_bdrv_co_readv(bs, sector_num, nb_sectors);
-
-    return bdrv_co_do_readv(bs, sector_num, nb_sectors, qiov, 0);
-}
-
-int coroutine_fn bdrv_co_copy_on_readv(BlockDriverState *bs,
-    int64_t sector_num, int nb_sectors, QEMUIOVector *qiov)
-{
-    trace_bdrv_co_copy_on_readv(bs, sector_num, nb_sectors);
-
-    return bdrv_co_do_readv(bs, sector_num, nb_sectors, qiov,
-                            BDRV_REQ_COPY_ON_READ);
-}
-
-static int coroutine_fn bdrv_co_do_write_zeroes(BlockDriverState *bs,
-    int64_t sector_num, int nb_sectors)
-{
-    BlockDriver *drv = bs->drv;
-    QEMUIOVector qiov;
-    struct iovec iov;
-    int ret;
-
-    /* TODO Emulate only part of misaligned requests instead of letting block
-     * drivers return -ENOTSUP and emulate everything */
-
-    /* First try the efficient write zeroes operation */
-    if (drv->bdrv_co_write_zeroes) {
-        ret = drv->bdrv_co_write_zeroes(bs, sector_num, nb_sectors);
-        if (ret != -ENOTSUP) {
-            return ret;
-        }
-    }
-
-    /* Fall back to bounce buffer if write zeroes is unsupported */
-    iov.iov_len  = nb_sectors * BDRV_SECTOR_SIZE;
-    iov.iov_base = qemu_blockalign(bs, iov.iov_len);
-    memset(iov.iov_base, 0, iov.iov_len);
-    qemu_iovec_init_external(&qiov, &iov, 1);
-
-    ret = drv->bdrv_co_writev(bs, sector_num, nb_sectors, &qiov);
-
-    qemu_vfree(iov.iov_base);
-    return ret;
-}
-
-/*
- * Handle a write request in coroutine context
- */
-static int coroutine_fn bdrv_co_do_writev(BlockDriverState *bs,
-    int64_t sector_num, int nb_sectors, QEMUIOVector *qiov,
-    BdrvRequestFlags flags)
-{
-    BlockDriver *drv = bs->drv;
-    BdrvTrackedRequest req;
-    int ret;
-
-    if (!bs->drv) {
-        return -ENOMEDIUM;
-    }
-    if (bs->read_only) {
-        return -EACCES;
-    }
-    if (bdrv_check_request(bs, sector_num, nb_sectors)) {
-        return -EIO;
-    }
-
-    /* throttling disk write I/O */
-    if (bs->io_limits_enabled) {
-        bdrv_io_limits_intercept(bs, true, nb_sectors);
-    }
-
-    if (bs->copy_on_read_in_flight) {
-        wait_for_overlapping_requests(bs, sector_num, nb_sectors);
-    }
-
-    tracked_request_begin(&req, bs, sector_num, nb_sectors, true);
-
-    ret = notifier_with_return_list_notify(&bs->before_write_notifiers, &req);
-
-    if (ret < 0) {
-        /* Do nothing, write notifier decided to fail this request */
-    } else if (flags & BDRV_REQ_ZERO_WRITE) {
-        ret = bdrv_co_do_write_zeroes(bs, sector_num, nb_sectors);
-    } else {
-        ret = drv->bdrv_co_writev(bs, sector_num, nb_sectors, qiov);
-    }
-
-    if (ret == 0 && !bs->enable_write_cache) {
-        ret = bdrv_co_flush(bs);
-    }
-
-    if (bs->dirty_bitmap) {
-        bdrv_set_dirty(bs, sector_num, nb_sectors);
-    }
-
-    if (bs->wr_highest_sector < sector_num + nb_sectors - 1) {
-        bs->wr_highest_sector = sector_num + nb_sectors - 1;
-    }
-
-    tracked_request_end(&req);
-
-    return ret;
-}
-
-int coroutine_fn bdrv_co_writev(BlockDriverState *bs, int64_t sector_num,
-    int nb_sectors, QEMUIOVector *qiov)
-{
-    trace_bdrv_co_writev(bs, sector_num, nb_sectors);
-
-    return bdrv_co_do_writev(bs, sector_num, nb_sectors, qiov, 0);
-}
-
-int coroutine_fn bdrv_co_write_zeroes(BlockDriverState *bs,
-                                      int64_t sector_num, int nb_sectors)
-{
-    trace_bdrv_co_write_zeroes(bs, sector_num, nb_sectors);
-
-    return bdrv_co_do_writev(bs, sector_num, nb_sectors, NULL,
-                             BDRV_REQ_ZERO_WRITE);
-}
-
-/**
- * Truncate file to 'offset' bytes (needed only for file protocols)
- */
-int bdrv_truncate(BlockDriverState *bs, int64_t offset)
-{
-    BlockDriver *drv = bs->drv;
-    int ret;
-    if (!drv)
-        return -ENOMEDIUM;
-    if (!drv->bdrv_truncate)
-        return -ENOTSUP;
-    if (bs->read_only)
-        return -EACCES;
-    if (bdrv_in_use(bs))
-        return -EBUSY;
-    ret = drv->bdrv_truncate(bs, offset);
-    if (ret == 0) {
-        ret = refresh_total_sectors(bs, offset >> BDRV_SECTOR_BITS);
-        bdrv_dev_resize_cb(bs);
-    }
-    return ret;
-}
-
-/**
- * Length of a allocated file in bytes. Sparse files are counted by actual
- * allocated space. Return < 0 if error or unknown.
- */
-int64_t bdrv_get_allocated_file_size(BlockDriverState *bs)
-{
-    BlockDriver *drv = bs->drv;
-    if (!drv) {
-        return -ENOMEDIUM;
-    }
-    if (drv->bdrv_get_allocated_file_size) {
-        return drv->bdrv_get_allocated_file_size(bs);
-    }
-    if (bs->file) {
-        return bdrv_get_allocated_file_size(bs->file);
-    }
-    return -ENOTSUP;
-}
-
-/**
- * Length of a file in bytes. Return < 0 if error or unknown.
- */
-int64_t bdrv_getlength(BlockDriverState *bs)
-{
-    BlockDriver *drv = bs->drv;
-    if (!drv)
-        return -ENOMEDIUM;
-
-    if (bs->growable || bdrv_dev_has_removable_media(bs)) {
-        if (drv->bdrv_getlength) {
-            return drv->bdrv_getlength(bs);
-        }
-    }
-    return bs->total_sectors * BDRV_SECTOR_SIZE;
-}
-
-/* return 0 as number of sectors if no device present or error */
-void bdrv_get_geometry(BlockDriverState *bs, uint64_t *nb_sectors_ptr)
-{
-    int64_t length;
-    length = bdrv_getlength(bs);
-    if (length < 0)
-        length = 0;
-    else
-        length = length >> BDRV_SECTOR_BITS;
-    *nb_sectors_ptr = length;
-}
-
-/* throttling disk io limits */
-void bdrv_set_io_limits(BlockDriverState *bs,
-                        BlockIOLimit *io_limits)
-{
-    bs->io_limits = *io_limits;
-    bs->io_limits_enabled = bdrv_io_limits_enabled(bs);
-}
-
-void bdrv_set_on_error(BlockDriverState *bs, BlockdevOnError on_read_error,
-                       BlockdevOnError on_write_error)
-{
-    bs->on_read_error = on_read_error;
-    bs->on_write_error = on_write_error;
-}
-
-BlockdevOnError bdrv_get_on_error(BlockDriverState *bs, bool is_read)
-{
-    return is_read ? bs->on_read_error : bs->on_write_error;
-}
-
-BlockErrorAction bdrv_get_error_action(BlockDriverState *bs, bool is_read, int error)
-{
-    BlockdevOnError on_err = is_read ? bs->on_read_error : bs->on_write_error;
-
-    switch (on_err) {
-    case BLOCKDEV_ON_ERROR_ENOSPC:
-        return (error == ENOSPC) ? BDRV_ACTION_STOP : BDRV_ACTION_REPORT;
-    case BLOCKDEV_ON_ERROR_STOP:
-        return BDRV_ACTION_STOP;
-    case BLOCKDEV_ON_ERROR_REPORT:
-        return BDRV_ACTION_REPORT;
-    case BLOCKDEV_ON_ERROR_IGNORE:
-        return BDRV_ACTION_IGNORE;
-    default:
-        abort();
-    }
-}
-
-/* This is done by device models because, while the block layer knows
- * about the error, it does not know whether an operation comes from
- * the device or the block layer (from a job, for example).
- */
-void bdrv_error_action(BlockDriverState *bs, BlockErrorAction action,
-                       bool is_read, int error)
-{
-    assert(error >= 0);
-    bdrv_emit_qmp_error_event(bs, QEVENT_BLOCK_IO_ERROR, action, is_read);
-    if (action == BDRV_ACTION_STOP) {
-        vm_stop(RUN_STATE_IO_ERROR);
-        bdrv_iostatus_set_err(bs, error);
-    }
-}
-
-int bdrv_is_read_only(BlockDriverState *bs)
-{
-    return bs->read_only;
-}
-
-int bdrv_is_sg(BlockDriverState *bs)
-{
-    return bs->sg;
-}
-
-int bdrv_enable_write_cache(BlockDriverState *bs)
-{
-    return bs->enable_write_cache;
-}
-
-void bdrv_set_enable_write_cache(BlockDriverState *bs, bool wce)
-{
-    bs->enable_write_cache = wce;
-
-    /* so a reopen() will preserve wce */
-    if (wce) {
-        bs->open_flags |= BDRV_O_CACHE_WB;
-    } else {
-        bs->open_flags &= ~BDRV_O_CACHE_WB;
-    }
-}
-
-int bdrv_is_encrypted(BlockDriverState *bs)
-{
-    if (bs->backing_hd && bs->backing_hd->encrypted)
-        return 1;
-    return bs->encrypted;
-}
-
-int bdrv_key_required(BlockDriverState *bs)
-{
-    BlockDriverState *backing_hd = bs->backing_hd;
-
-    if (backing_hd && backing_hd->encrypted && !backing_hd->valid_key)
-        return 1;
-    return (bs->encrypted && !bs->valid_key);
-}
-
-int bdrv_set_key(BlockDriverState *bs, const char *key)
-{
-    int ret;
-    if (bs->backing_hd && bs->backing_hd->encrypted) {
-        ret = bdrv_set_key(bs->backing_hd, key);
-        if (ret < 0)
-            return ret;
-        if (!bs->encrypted)
-            return 0;
-    }
-    if (!bs->encrypted) {
-        return -EINVAL;
-    } else if (!bs->drv || !bs->drv->bdrv_set_key) {
-        return -ENOMEDIUM;
-    }
-    ret = bs->drv->bdrv_set_key(bs, key);
-    if (ret < 0) {
-        bs->valid_key = 0;
-    } else if (!bs->valid_key) {
-        bs->valid_key = 1;
-        /* call the change callback now, we skipped it on open */
-        bdrv_dev_change_media_cb(bs, true);
-    }
-    return ret;
-}
-
-const char *bdrv_get_format_name(BlockDriverState *bs)
-{
-    return bs->drv ? bs->drv->format_name : NULL;
-}
-
-void bdrv_iterate_format(void (*it)(void *opaque, const char *name),
-                         void *opaque)
-{
-    BlockDriver *drv;
-
-    QLIST_FOREACH(drv, &bdrv_drivers, list) {
-        it(opaque, drv->format_name);
-    }
-}
-
-BlockDriverState *bdrv_find(const char *name)
-{
-    BlockDriverState *bs;
-
-    QTAILQ_FOREACH(bs, &bdrv_states, list) {
-        if (!strcmp(name, bs->device_name)) {
-            return bs;
-        }
-    }
-    return NULL;
-}
-
-BlockDriverState *bdrv_next(BlockDriverState *bs)
-{
-    if (!bs) {
-        return QTAILQ_FIRST(&bdrv_states);
-    }
-    return QTAILQ_NEXT(bs, list);
-}
-
-void bdrv_iterate(void (*it)(void *opaque, BlockDriverState *bs), void *opaque)
-{
-    BlockDriverState *bs;
-
-    QTAILQ_FOREACH(bs, &bdrv_states, list) {
-        it(opaque, bs);
-    }
-}
-
-const char *bdrv_get_device_name(BlockDriverState *bs)
-{
-    return bs->device_name;
-}
-
-int bdrv_get_flags(BlockDriverState *bs)
-{
-    return bs->open_flags;
-}
-
-int bdrv_flush_all(void)
-{
-    BlockDriverState *bs;
-    int result = 0;
-
-    QTAILQ_FOREACH(bs, &bdrv_states, list) {
-        int ret = bdrv_flush(bs);
-        if (ret < 0 && !result) {
-            result = ret;
-        }
-    }
-
-    return result;
-}
-
-int bdrv_has_zero_init_1(BlockDriverState *bs)
-{
-    return 1;
-}
-
-int bdrv_has_zero_init(BlockDriverState *bs)
-{
-    assert(bs->drv);
-
-    if (bs->drv->bdrv_has_zero_init) {
-        return bs->drv->bdrv_has_zero_init(bs);
-    }
-
-    /* safe default */
-    return 0;
-}
-
-typedef struct BdrvCoIsAllocatedData {
-    BlockDriverState *bs;
-    BlockDriverState *base;
-    int64_t sector_num;
-    int nb_sectors;
-    int *pnum;
-    int ret;
-    bool done;
-} BdrvCoIsAllocatedData;
-
-/*
- * Returns true iff the specified sector is present in the disk image. Drivers
- * not implementing the functionality are assumed to not support backing files,
- * hence all their sectors are reported as allocated.
- *
- * If 'sector_num' is beyond the end of the disk image the return value is 0
- * and 'pnum' is set to 0.
- *
- * 'pnum' is set to the number of sectors (including and immediately following
- * the specified sector) that are known to be in the same
- * allocated/unallocated state.
- *
- * 'nb_sectors' is the max value 'pnum' should be set to.  If nb_sectors goes
- * beyond the end of the disk image it will be clamped.
- */
-int coroutine_fn bdrv_co_is_allocated(BlockDriverState *bs, int64_t sector_num,
-                                      int nb_sectors, int *pnum)
-{
-    int64_t n;
-
-    if (sector_num >= bs->total_sectors) {
-        *pnum = 0;
-        return 0;
-    }
-
-    n = bs->total_sectors - sector_num;
-    if (n < nb_sectors) {
-        nb_sectors = n;
-    }
-
-    if (!bs->drv->bdrv_co_is_allocated) {
-        *pnum = nb_sectors;
-        return 1;
-    }
-
-    return bs->drv->bdrv_co_is_allocated(bs, sector_num, nb_sectors, pnum);
-}
-
-/* Coroutine wrapper for bdrv_is_allocated() */
-static void coroutine_fn bdrv_is_allocated_co_entry(void *opaque)
-{
-    BdrvCoIsAllocatedData *data = opaque;
-    BlockDriverState *bs = data->bs;
-
-    data->ret = bdrv_co_is_allocated(bs, data->sector_num, data->nb_sectors,
-                                     data->pnum);
-    data->done = true;
-}
-
-/*
- * Synchronous wrapper around bdrv_co_is_allocated().
- *
- * See bdrv_co_is_allocated() for details.
- */
-int bdrv_is_allocated(BlockDriverState *bs, int64_t sector_num, int nb_sectors,
-                      int *pnum)
-{
-    Coroutine *co;
-    BdrvCoIsAllocatedData data = {
-        .bs = bs,
-        .sector_num = sector_num,
-        .nb_sectors = nb_sectors,
-        .pnum = pnum,
-        .done = false,
-    };
-
-    co = qemu_coroutine_create(bdrv_is_allocated_co_entry);
-    qemu_coroutine_enter(co, &data);
-    while (!data.done) {
-        qemu_aio_wait();
-    }
-    return data.ret;
-}
-
-/*
- * Given an image chain: ... -> [BASE] -> [INTER1] -> [INTER2] -> [TOP]
- *
- * Return true if the given sector is allocated in any image between
- * BASE and TOP (inclusive).  BASE can be NULL to check if the given
- * sector is allocated in any image of the chain.  Return false otherwise.
- *
- * 'pnum' is set to the number of sectors (including and immediately following
- *  the specified sector) that are known to be in the same
- *  allocated/unallocated state.
- *
- */
-int coroutine_fn bdrv_co_is_allocated_above(BlockDriverState *top,
-                                            BlockDriverState *base,
-                                            int64_t sector_num,
-                                            int nb_sectors, int *pnum)
-{
-    BlockDriverState *intermediate;
-    int ret, n = nb_sectors;
-
-    intermediate = top;
-    while (intermediate && intermediate != base) {
-        int pnum_inter;
-        ret = bdrv_co_is_allocated(intermediate, sector_num, nb_sectors,
-                                   &pnum_inter);
-        if (ret < 0) {
-            return ret;
-        } else if (ret) {
-            *pnum = pnum_inter;
-            return 1;
-        }
-
-        /*
-         * [sector_num, nb_sectors] is unallocated on top but intermediate
-         * might have
-         *
-         * [sector_num+x, nr_sectors] allocated.
-         */
-        if (n > pnum_inter &&
-            (intermediate == top ||
-             sector_num + pnum_inter < intermediate->total_sectors)) {
-            n = pnum_inter;
-        }
-
-        intermediate = intermediate->backing_hd;
-    }
-
-    *pnum = n;
-    return 0;
-}
-
-/* Coroutine wrapper for bdrv_is_allocated_above() */
-static void coroutine_fn bdrv_is_allocated_above_co_entry(void *opaque)
-{
-    BdrvCoIsAllocatedData *data = opaque;
-    BlockDriverState *top = data->bs;
-    BlockDriverState *base = data->base;
-
-    data->ret = bdrv_co_is_allocated_above(top, base, data->sector_num,
-                                           data->nb_sectors, data->pnum);
-    data->done = true;
-}
-
-/*
- * Synchronous wrapper around bdrv_co_is_allocated_above().
- *
- * See bdrv_co_is_allocated_above() for details.
- */
-int bdrv_is_allocated_above(BlockDriverState *top, BlockDriverState *base,
-                            int64_t sector_num, int nb_sectors, int *pnum)
-{
-    Coroutine *co;
-    BdrvCoIsAllocatedData data = {
-        .bs = top,
-        .base = base,
-        .sector_num = sector_num,
-        .nb_sectors = nb_sectors,
-        .pnum = pnum,
-        .done = false,
-    };
-
-    co = qemu_coroutine_create(bdrv_is_allocated_above_co_entry);
-    qemu_coroutine_enter(co, &data);
-    while (!data.done) {
-        qemu_aio_wait();
-    }
-    return data.ret;
-}
-
-const char *bdrv_get_encrypted_filename(BlockDriverState *bs)
-{
-    if (bs->backing_hd && bs->backing_hd->encrypted)
-        return bs->backing_file;
-    else if (bs->encrypted)
-        return bs->filename;
-    else
-        return NULL;
-}
-
-void bdrv_get_backing_filename(BlockDriverState *bs,
-                               char *filename, int filename_size)
-{
-    pstrcpy(filename, filename_size, bs->backing_file);
-}
-
-int bdrv_write_compressed(BlockDriverState *bs, int64_t sector_num,
-                          const uint8_t *buf, int nb_sectors)
-{
-    BlockDriver *drv = bs->drv;
-    if (!drv)
-        return -ENOMEDIUM;
-    if (!drv->bdrv_write_compressed)
-        return -ENOTSUP;
-    if (bdrv_check_request(bs, sector_num, nb_sectors))
-        return -EIO;
-
-    assert(!bs->dirty_bitmap);
-
-    return drv->bdrv_write_compressed(bs, sector_num, buf, nb_sectors);
-}
-
-int bdrv_get_info(BlockDriverState *bs, BlockDriverInfo *bdi)
-{
-    BlockDriver *drv = bs->drv;
-    if (!drv)
-        return -ENOMEDIUM;
-    if (!drv->bdrv_get_info)
-        return -ENOTSUP;
-    memset(bdi, 0, sizeof(*bdi));
-    return drv->bdrv_get_info(bs, bdi);
-}
-
-int bdrv_save_vmstate(BlockDriverState *bs, const uint8_t *buf,
-                      int64_t pos, int size)
-{
-    QEMUIOVector qiov;
-    struct iovec iov = {
-        .iov_base   = (void *) buf,
-        .iov_len    = size,
-    };
-
-    qemu_iovec_init_external(&qiov, &iov, 1);
-    return bdrv_writev_vmstate(bs, &qiov, pos);
-}
-
-int bdrv_writev_vmstate(BlockDriverState *bs, QEMUIOVector *qiov, int64_t pos)
-{
-    BlockDriver *drv = bs->drv;
-
-    if (!drv) {
-        return -ENOMEDIUM;
-    } else if (drv->bdrv_save_vmstate) {
-        return drv->bdrv_save_vmstate(bs, qiov, pos);
-    } else if (bs->file) {
-        return bdrv_writev_vmstate(bs->file, qiov, pos);
-    }
-
-    return -ENOTSUP;
-}
-
-int bdrv_load_vmstate(BlockDriverState *bs, uint8_t *buf,
-                      int64_t pos, int size)
-{
-    BlockDriver *drv = bs->drv;
-    if (!drv)
-        return -ENOMEDIUM;
-    if (drv->bdrv_load_vmstate)
-        return drv->bdrv_load_vmstate(bs, buf, pos, size);
-    if (bs->file)
-        return bdrv_load_vmstate(bs->file, buf, pos, size);
-    return -ENOTSUP;
-}
-
-void bdrv_debug_event(BlockDriverState *bs, BlkDebugEvent event)
-{
-    if (!bs || !bs->drv || !bs->drv->bdrv_debug_event) {
-        return;
-    }
-
-    bs->drv->bdrv_debug_event(bs, event);
-}
-
-int bdrv_debug_breakpoint(BlockDriverState *bs, const char *event,
-                          const char *tag)
-{
-    while (bs && bs->drv && !bs->drv->bdrv_debug_breakpoint) {
-        bs = bs->file;
-    }
-
-    if (bs && bs->drv && bs->drv->bdrv_debug_breakpoint) {
-        return bs->drv->bdrv_debug_breakpoint(bs, event, tag);
-    }
-
-    return -ENOTSUP;
-}
-
-int bdrv_debug_resume(BlockDriverState *bs, const char *tag)
-{
-    while (bs && bs->drv && !bs->drv->bdrv_debug_resume) {
-        bs = bs->file;
-    }
-
-    if (bs && bs->drv && bs->drv->bdrv_debug_resume) {
-        return bs->drv->bdrv_debug_resume(bs, tag);
-    }
-
-    return -ENOTSUP;
-}
-
-bool bdrv_debug_is_suspended(BlockDriverState *bs, const char *tag)
-{
-    while (bs && bs->drv && !bs->drv->bdrv_debug_is_suspended) {
-        bs = bs->file;
-    }
-
-    if (bs && bs->drv && bs->drv->bdrv_debug_is_suspended) {
-        return bs->drv->bdrv_debug_is_suspended(bs, tag);
-    }
-
-    return false;
-}
-
-int bdrv_is_snapshot(BlockDriverState *bs)
-{
-    return !!(bs->open_flags & BDRV_O_SNAPSHOT);
-}
-
-/* backing_file can either be relative, or absolute, or a protocol.  If it is
- * relative, it must be relative to the chain.  So, passing in bs->filename
- * from a BDS as backing_file should not be done, as that may be relative to
- * the CWD rather than the chain. */
-BlockDriverState *bdrv_find_backing_image(BlockDriverState *bs,
-        const char *backing_file)
-{
-    char *filename_full = NULL;
-    char *backing_file_full = NULL;
-    char *filename_tmp = NULL;
-    int is_protocol = 0;
-    BlockDriverState *curr_bs = NULL;
-    BlockDriverState *retval = NULL;
-
-    if (!bs || !bs->drv || !backing_file) {
-        return NULL;
-    }
-
-    filename_full     = g_malloc(PATH_MAX);
-    backing_file_full = g_malloc(PATH_MAX);
-    filename_tmp      = g_malloc(PATH_MAX);
-
-    is_protocol = path_has_protocol(backing_file);
-
-    for (curr_bs = bs; curr_bs->backing_hd; curr_bs = curr_bs->backing_hd) {
-
-        /* If either of the filename paths is actually a protocol, then
-         * compare unmodified paths; otherwise make paths relative */
-        if (is_protocol || path_has_protocol(curr_bs->backing_file)) {
-            if (strcmp(backing_file, curr_bs->backing_file) == 0) {
-                retval = curr_bs->backing_hd;
-                break;
-            }
-        } else {
-            /* If not an absolute filename path, make it relative to the current
-             * image's filename path */
-            path_combine(filename_tmp, PATH_MAX, curr_bs->filename,
-                         backing_file);
-
-            /* We are going to compare absolute pathnames */
-            if (!realpath(filename_tmp, filename_full)) {
-                continue;
-            }
-
-            /* We need to make sure the backing filename we are comparing against
-             * is relative to the current image filename (or absolute) */
-            path_combine(filename_tmp, PATH_MAX, curr_bs->filename,
-                         curr_bs->backing_file);
-
-            if (!realpath(filename_tmp, backing_file_full)) {
-                continue;
-            }
-
-            if (strcmp(backing_file_full, filename_full) == 0) {
-                retval = curr_bs->backing_hd;
-                break;
-            }
-        }
-    }
-
-    g_free(filename_full);
-    g_free(backing_file_full);
-    g_free(filename_tmp);
-    return retval;
-}
-
-int bdrv_get_backing_file_depth(BlockDriverState *bs)
-{
-    if (!bs->drv) {
-        return 0;
-    }
-
-    if (!bs->backing_hd) {
-        return 0;
-    }
-
-    return 1 + bdrv_get_backing_file_depth(bs->backing_hd);
-}
-
-BlockDriverState *bdrv_find_base(BlockDriverState *bs)
-{
-    BlockDriverState *curr_bs = NULL;
-
-    if (!bs) {
-        return NULL;
-    }
-
-    curr_bs = bs;
-
-    while (curr_bs->backing_hd) {
-        curr_bs = curr_bs->backing_hd;
-    }
-    return curr_bs;
-}
-
-/**************************************************************/
-/* async I/Os */
-
-BlockDriverAIOCB *bdrv_aio_readv(BlockDriverState *bs, int64_t sector_num,
-                                 QEMUIOVector *qiov, int nb_sectors,
-                                 BlockDriverCompletionFunc *cb, void *opaque)
-{
-    trace_bdrv_aio_readv(bs, sector_num, nb_sectors, opaque);
-
-    return bdrv_co_aio_rw_vector(bs, sector_num, qiov, nb_sectors,
-                                 cb, opaque, false);
-}
-
-BlockDriverAIOCB *bdrv_aio_writev(BlockDriverState *bs, int64_t sector_num,
-                                  QEMUIOVector *qiov, int nb_sectors,
-                                  BlockDriverCompletionFunc *cb, void *opaque)
-{
-    trace_bdrv_aio_writev(bs, sector_num, nb_sectors, opaque);
-
-    return bdrv_co_aio_rw_vector(bs, sector_num, qiov, nb_sectors,
-                                 cb, opaque, true);
-}
-
-
-typedef struct MultiwriteCB {
-    int error;
-    int num_requests;
-    int num_callbacks;
-    struct {
-        BlockDriverCompletionFunc *cb;
-        void *opaque;
-        QEMUIOVector *free_qiov;
-    } callbacks[];
-} MultiwriteCB;
-
-static void multiwrite_user_cb(MultiwriteCB *mcb)
-{
-    int i;
-
-    for (i = 0; i < mcb->num_callbacks; i++) {
-        mcb->callbacks[i].cb(mcb->callbacks[i].opaque, mcb->error);
-        if (mcb->callbacks[i].free_qiov) {
-            qemu_iovec_destroy(mcb->callbacks[i].free_qiov);
-        }
-        g_free(mcb->callbacks[i].free_qiov);
-    }
-}
-
-static void multiwrite_cb(void *opaque, int ret)
-{
-    MultiwriteCB *mcb = opaque;
-
-    trace_multiwrite_cb(mcb, ret);
-
-    if (ret < 0 && !mcb->error) {
-        mcb->error = ret;
-    }
-
-    mcb->num_requests--;
-    if (mcb->num_requests == 0) {
-        multiwrite_user_cb(mcb);
-        g_free(mcb);
-    }
-}
-
-static int multiwrite_req_compare(const void *a, const void *b)
-{
-    const BlockRequest *req1 = a, *req2 = b;
-
-    /*
-     * Note that we can't simply subtract req2->sector from req1->sector
-     * here as that could overflow the return value.
-     */
-    if (req1->sector > req2->sector) {
-        return 1;
-    } else if (req1->sector < req2->sector) {
-        return -1;
-    } else {
-        return 0;
-    }
-}
-
-/*
- * Takes a bunch of requests and tries to merge them. Returns the number of
- * requests that remain after merging.
- */
-static int multiwrite_merge(BlockDriverState *bs, BlockRequest *reqs,
-    int num_reqs, MultiwriteCB *mcb)
-{
-    int i, outidx;
-
-    // Sort requests by start sector
-    qsort(reqs, num_reqs, sizeof(*reqs), &multiwrite_req_compare);
-
-    // Check if adjacent requests touch the same clusters. If so, combine them,
-    // filling up gaps with zero sectors.
-    outidx = 0;
-    for (i = 1; i < num_reqs; i++) {
-        int merge = 0;
-        int64_t oldreq_last = reqs[outidx].sector + reqs[outidx].nb_sectors;
-
-        // Handle exactly sequential writes and overlapping writes.
-        if (reqs[i].sector <= oldreq_last) {
-            merge = 1;
-        }
-
-        if (reqs[outidx].qiov->niov + reqs[i].qiov->niov + 1 > IOV_MAX) {
-            merge = 0;
-        }
-
-        if (merge) {
-            size_t size;
-            QEMUIOVector *qiov = g_malloc0(sizeof(*qiov));
-            qemu_iovec_init(qiov,
-                reqs[outidx].qiov->niov + reqs[i].qiov->niov + 1);
-
-            // Add the first request to the merged one. If the requests are
-            // overlapping, drop the last sectors of the first request.
-            size = (reqs[i].sector - reqs[outidx].sector) << 9;
-            qemu_iovec_concat(qiov, reqs[outidx].qiov, 0, size);
-
-            // We should need to add any zeros between the two requests
-            assert (reqs[i].sector <= oldreq_last);
-
-            // Add the second request
-            qemu_iovec_concat(qiov, reqs[i].qiov, 0, reqs[i].qiov->size);
-
-            reqs[outidx].nb_sectors = qiov->size >> 9;
-            reqs[outidx].qiov = qiov;
-
-            mcb->callbacks[i].free_qiov = reqs[outidx].qiov;
-        } else {
-            outidx++;
-            reqs[outidx].sector     = reqs[i].sector;
-            reqs[outidx].nb_sectors = reqs[i].nb_sectors;
-            reqs[outidx].qiov       = reqs[i].qiov;
-        }
-    }
-
-    return outidx + 1;
-}
-
-/*
- * Submit multiple AIO write requests at once.
- *
- * On success, the function returns 0 and all requests in the reqs array have
- * been submitted. In error case this function returns -1, and any of the
- * requests may or may not be submitted yet. In particular, this means that the
- * callback will be called for some of the requests, for others it won't. The
- * caller must check the error field of the BlockRequest to wait for the right
- * callbacks (if error != 0, no callback will be called).
- *
- * The implementation may modify the contents of the reqs array, e.g. to merge
- * requests. However, the fields opaque and error are left unmodified as they
- * are used to signal failure for a single request to the caller.
- */
-int bdrv_aio_multiwrite(BlockDriverState *bs, BlockRequest *reqs, int num_reqs)
-{
-    MultiwriteCB *mcb;
-    int i;
-
-    /* don't submit writes if we don't have a medium */
-    if (bs->drv == NULL) {
-        for (i = 0; i < num_reqs; i++) {
-            reqs[i].error = -ENOMEDIUM;
-        }
-        return -1;
-    }
-
-    if (num_reqs == 0) {
-        return 0;
-    }
-
-    // Create MultiwriteCB structure
-    mcb = g_malloc0(sizeof(*mcb) + num_reqs * sizeof(*mcb->callbacks));
-    mcb->num_requests = 0;
-    mcb->num_callbacks = num_reqs;
-
-    for (i = 0; i < num_reqs; i++) {
-        mcb->callbacks[i].cb = reqs[i].cb;
-        mcb->callbacks[i].opaque = reqs[i].opaque;
-    }
-
-    // Check for mergable requests
-    num_reqs = multiwrite_merge(bs, reqs, num_reqs, mcb);
-
-    trace_bdrv_aio_multiwrite(mcb, mcb->num_callbacks, num_reqs);
-
-    /* Run the aio requests. */
-    mcb->num_requests = num_reqs;
-    for (i = 0; i < num_reqs; i++) {
-        bdrv_aio_writev(bs, reqs[i].sector, reqs[i].qiov,
-            reqs[i].nb_sectors, multiwrite_cb, mcb);
-    }
-
-    return 0;
-}
-
-void bdrv_aio_cancel(BlockDriverAIOCB *acb)
-{
-    acb->aiocb_info->cancel(acb);
-}
-
-/* block I/O throttling */
-static bool bdrv_exceed_bps_limits(BlockDriverState *bs, int nb_sectors,
-                 bool is_write, double elapsed_time, uint64_t *wait)
-{
-    uint64_t bps_limit = 0;
-    uint64_t extension;
-    double   bytes_limit, bytes_base, bytes_res;
-    double   slice_time, wait_time;
-
-    if (bs->io_limits.bps[BLOCK_IO_LIMIT_TOTAL]) {
-        bps_limit = bs->io_limits.bps[BLOCK_IO_LIMIT_TOTAL];
-    } else if (bs->io_limits.bps[is_write]) {
-        bps_limit = bs->io_limits.bps[is_write];
-    } else {
-        if (wait) {
-            *wait = 0;
-        }
-
-        return false;
-    }
-
-    slice_time = bs->slice_end - bs->slice_start;
-    slice_time /= (NANOSECONDS_PER_SECOND);
-    bytes_limit = bps_limit * slice_time;
-    bytes_base  = bs->slice_submitted.bytes[is_write];
-    if (bs->io_limits.bps[BLOCK_IO_LIMIT_TOTAL]) {
-        bytes_base += bs->slice_submitted.bytes[!is_write];
-    }
-
-    /* bytes_base: the bytes of data which have been read/written; and
-     *             it is obtained from the history statistic info.
-     * bytes_res: the remaining bytes of data which need to be read/written.
-     * (bytes_base + bytes_res) / bps_limit: used to calcuate
-     *             the total time for completing reading/writting all data.
-     */
-    bytes_res   = (unsigned) nb_sectors * BDRV_SECTOR_SIZE;
-
-    if (bytes_base + bytes_res <= bytes_limit) {
-        if (wait) {
-            *wait = 0;
-        }
-
-        return false;
-    }
-
-    /* Calc approx time to dispatch */
-    wait_time = (bytes_base + bytes_res) / bps_limit - elapsed_time;
-
-    /* When the I/O rate at runtime exceeds the limits,
-     * bs->slice_end need to be extended in order that the current statistic
-     * info can be kept until the timer fire, so it is increased and tuned
-     * based on the result of experiment.
-     */
-    extension = wait_time * NANOSECONDS_PER_SECOND;
-    extension = DIV_ROUND_UP(extension, BLOCK_IO_SLICE_TIME) *
-                BLOCK_IO_SLICE_TIME;
-    bs->slice_end += extension;
-    if (wait) {
-        *wait = wait_time * NANOSECONDS_PER_SECOND;
-    }
-
-    return true;
-}
-
-static bool bdrv_exceed_iops_limits(BlockDriverState *bs, bool is_write,
-                             double elapsed_time, uint64_t *wait)
-{
-    uint64_t iops_limit = 0;
-    double   ios_limit, ios_base;
-    double   slice_time, wait_time;
-
-    if (bs->io_limits.iops[BLOCK_IO_LIMIT_TOTAL]) {
-        iops_limit = bs->io_limits.iops[BLOCK_IO_LIMIT_TOTAL];
-    } else if (bs->io_limits.iops[is_write]) {
-        iops_limit = bs->io_limits.iops[is_write];
-    } else {
-        if (wait) {
-            *wait = 0;
-        }
-
-        return false;
-    }
-
-    slice_time = bs->slice_end - bs->slice_start;
-    slice_time /= (NANOSECONDS_PER_SECOND);
-    ios_limit  = iops_limit * slice_time;
-    ios_base   = bs->slice_submitted.ios[is_write];
-    if (bs->io_limits.iops[BLOCK_IO_LIMIT_TOTAL]) {
-        ios_base += bs->slice_submitted.ios[!is_write];
-    }
-
-    if (ios_base + 1 <= ios_limit) {
-        if (wait) {
-            *wait = 0;
-        }
-
-        return false;
-    }
-
-    /* Calc approx time to dispatch, in seconds */
-    wait_time = (ios_base + 1) / iops_limit;
-    if (wait_time > elapsed_time) {
-        wait_time = wait_time - elapsed_time;
-    } else {
-        wait_time = 0;
-    }
-
-    /* Exceeded current slice, extend it by another slice time */
-    bs->slice_end += BLOCK_IO_SLICE_TIME;
-    if (wait) {
-        *wait = wait_time * NANOSECONDS_PER_SECOND;
-    }
-
-    return true;
-}
-
-static bool bdrv_exceed_io_limits(BlockDriverState *bs, int nb_sectors,
-                           bool is_write, int64_t *wait)
-{
-    int64_t  now, max_wait;
-    uint64_t bps_wait = 0, iops_wait = 0;
-    double   elapsed_time;
-    int      bps_ret, iops_ret;
-
-    now = qemu_get_clock_ns(vm_clock);
-    if (now > bs->slice_end) {
-        bs->slice_start = now;
-        bs->slice_end   = now + BLOCK_IO_SLICE_TIME;
-        memset(&bs->slice_submitted, 0, sizeof(bs->slice_submitted));
-    }
-
-    elapsed_time  = now - bs->slice_start;
-    elapsed_time  /= (NANOSECONDS_PER_SECOND);
-
-    bps_ret  = bdrv_exceed_bps_limits(bs, nb_sectors,
-                                      is_write, elapsed_time, &bps_wait);
-    iops_ret = bdrv_exceed_iops_limits(bs, is_write,
-                                      elapsed_time, &iops_wait);
-    if (bps_ret || iops_ret) {
-        max_wait = bps_wait > iops_wait ? bps_wait : iops_wait;
-        if (wait) {
-            *wait = max_wait;
-        }
-
-        now = qemu_get_clock_ns(vm_clock);
-        if (bs->slice_end < now + max_wait) {
-            bs->slice_end = now + max_wait;
-        }
-
-        return true;
-    }
-
-    if (wait) {
-        *wait = 0;
-    }
-
-    bs->slice_submitted.bytes[is_write] += (int64_t)nb_sectors *
-                                           BDRV_SECTOR_SIZE;
-    bs->slice_submitted.ios[is_write]++;
-
-    return false;
-}
-
-/**************************************************************/
-/* async block device emulation */
-
-typedef struct BlockDriverAIOCBSync {
-    BlockDriverAIOCB common;
-    QEMUBH *bh;
-    int ret;
-    /* vector translation state */
-    QEMUIOVector *qiov;
-    uint8_t *bounce;
-    int is_write;
-} BlockDriverAIOCBSync;
-
-static void bdrv_aio_cancel_em(BlockDriverAIOCB *blockacb)
-{
-    BlockDriverAIOCBSync *acb =
-        container_of(blockacb, BlockDriverAIOCBSync, common);
-    qemu_bh_delete(acb->bh);
-    acb->bh = NULL;
-    qemu_aio_release(acb);
-}
-
-static const AIOCBInfo bdrv_em_aiocb_info = {
-    .aiocb_size         = sizeof(BlockDriverAIOCBSync),
-    .cancel             = bdrv_aio_cancel_em,
-};
-
-static void bdrv_aio_bh_cb(void *opaque)
-{
-    BlockDriverAIOCBSync *acb = opaque;
-
-    if (!acb->is_write)
-        qemu_iovec_from_buf(acb->qiov, 0, acb->bounce, acb->qiov->size);
-    qemu_vfree(acb->bounce);
-    acb->common.cb(acb->common.opaque, acb->ret);
-    qemu_bh_delete(acb->bh);
-    acb->bh = NULL;
-    qemu_aio_release(acb);
-}
-
-static BlockDriverAIOCB *bdrv_aio_rw_vector(BlockDriverState *bs,
-                                            int64_t sector_num,
-                                            QEMUIOVector *qiov,
-                                            int nb_sectors,
-                                            BlockDriverCompletionFunc *cb,
-                                            void *opaque,
-                                            int is_write)
-
-{
-    BlockDriverAIOCBSync *acb;
-
-    acb = qemu_aio_get(&bdrv_em_aiocb_info, bs, cb, opaque);
-    acb->is_write = is_write;
-    acb->qiov = qiov;
-    acb->bounce = qemu_blockalign(bs, qiov->size);
-    acb->bh = qemu_bh_new(bdrv_aio_bh_cb, acb);
-
-    if (is_write) {
-        qemu_iovec_to_buf(acb->qiov, 0, acb->bounce, qiov->size);
-        acb->ret = bs->drv->bdrv_write(bs, sector_num, acb->bounce, nb_sectors);
-    } else {
-        acb->ret = bs->drv->bdrv_read(bs, sector_num, acb->bounce, nb_sectors);
-    }
-
-    qemu_bh_schedule(acb->bh);
-
-    return &acb->common;
-}
-
-static BlockDriverAIOCB *bdrv_aio_readv_em(BlockDriverState *bs,
-        int64_t sector_num, QEMUIOVector *qiov, int nb_sectors,
-        BlockDriverCompletionFunc *cb, void *opaque)
-{
-    return bdrv_aio_rw_vector(bs, sector_num, qiov, nb_sectors, cb, opaque, 0);
-}
-
-static BlockDriverAIOCB *bdrv_aio_writev_em(BlockDriverState *bs,
-        int64_t sector_num, QEMUIOVector *qiov, int nb_sectors,
-        BlockDriverCompletionFunc *cb, void *opaque)
-{
-    return bdrv_aio_rw_vector(bs, sector_num, qiov, nb_sectors, cb, opaque, 1);
-}
-
-
-typedef struct BlockDriverAIOCBCoroutine {
-    BlockDriverAIOCB common;
-    BlockRequest req;
-    bool is_write;
-    bool *done;
-    QEMUBH* bh;
-} BlockDriverAIOCBCoroutine;
-
-static void bdrv_aio_co_cancel_em(BlockDriverAIOCB *blockacb)
-{
-    BlockDriverAIOCBCoroutine *acb =
-        container_of(blockacb, BlockDriverAIOCBCoroutine, common);
-    bool done = false;
-
-    acb->done = &done;
-    while (!done) {
-        qemu_aio_wait();
-    }
-}
-
-static const AIOCBInfo bdrv_em_co_aiocb_info = {
-    .aiocb_size         = sizeof(BlockDriverAIOCBCoroutine),
-    .cancel             = bdrv_aio_co_cancel_em,
-};
-
-static void bdrv_co_em_bh(void *opaque)
-{
-    BlockDriverAIOCBCoroutine *acb = opaque;
-
-    acb->common.cb(acb->common.opaque, acb->req.error);
-
-    if (acb->done) {
-        *acb->done = true;
-    }
-
-    qemu_bh_delete(acb->bh);
-    qemu_aio_release(acb);
-}
-
-/* Invoke bdrv_co_do_readv/bdrv_co_do_writev */
-static void coroutine_fn bdrv_co_do_rw(void *opaque)
-{
-    BlockDriverAIOCBCoroutine *acb = opaque;
-    BlockDriverState *bs = acb->common.bs;
-
-    if (!acb->is_write) {
-        acb->req.error = bdrv_co_do_readv(bs, acb->req.sector,
-            acb->req.nb_sectors, acb->req.qiov, 0);
-    } else {
-        acb->req.error = bdrv_co_do_writev(bs, acb->req.sector,
-            acb->req.nb_sectors, acb->req.qiov, 0);
-    }
-
-    acb->bh = qemu_bh_new(bdrv_co_em_bh, acb);
-    qemu_bh_schedule(acb->bh);
-}
-
-static BlockDriverAIOCB *bdrv_co_aio_rw_vector(BlockDriverState *bs,
-                                               int64_t sector_num,
-                                               QEMUIOVector *qiov,
-                                               int nb_sectors,
-                                               BlockDriverCompletionFunc *cb,
-                                               void *opaque,
-                                               bool is_write)
-{
-    Coroutine *co;
-    BlockDriverAIOCBCoroutine *acb;
-
-    acb = qemu_aio_get(&bdrv_em_co_aiocb_info, bs, cb, opaque);
-    acb->req.sector = sector_num;
-    acb->req.nb_sectors = nb_sectors;
-    acb->req.qiov = qiov;
-    acb->is_write = is_write;
-    acb->done = NULL;
-
-    co = qemu_coroutine_create(bdrv_co_do_rw);
-    qemu_coroutine_enter(co, acb);
-
-    return &acb->common;
-}
-
-static void coroutine_fn bdrv_aio_flush_co_entry(void *opaque)
-{
-    BlockDriverAIOCBCoroutine *acb = opaque;
-    BlockDriverState *bs = acb->common.bs;
-
-    acb->req.error = bdrv_co_flush(bs);
-    acb->bh = qemu_bh_new(bdrv_co_em_bh, acb);
-    qemu_bh_schedule(acb->bh);
-}
-
-BlockDriverAIOCB *bdrv_aio_flush(BlockDriverState *bs,
-        BlockDriverCompletionFunc *cb, void *opaque)
-{
-    trace_bdrv_aio_flush(bs, opaque);
-
-    Coroutine *co;
-    BlockDriverAIOCBCoroutine *acb;
-
-    acb = qemu_aio_get(&bdrv_em_co_aiocb_info, bs, cb, opaque);
-    acb->done = NULL;
-
-    co = qemu_coroutine_create(bdrv_aio_flush_co_entry);
-    qemu_coroutine_enter(co, acb);
-
-    return &acb->common;
-}
-
-static void coroutine_fn bdrv_aio_discard_co_entry(void *opaque)
-{
-    BlockDriverAIOCBCoroutine *acb = opaque;
-    BlockDriverState *bs = acb->common.bs;
-
-    acb->req.error = bdrv_co_discard(bs, acb->req.sector, acb->req.nb_sectors);
-    acb->bh = qemu_bh_new(bdrv_co_em_bh, acb);
-    qemu_bh_schedule(acb->bh);
-}
-
-BlockDriverAIOCB *bdrv_aio_discard(BlockDriverState *bs,
-        int64_t sector_num, int nb_sectors,
-        BlockDriverCompletionFunc *cb, void *opaque)
-{
-    Coroutine *co;
-    BlockDriverAIOCBCoroutine *acb;
-
-    trace_bdrv_aio_discard(bs, sector_num, nb_sectors, opaque);
-
-    acb = qemu_aio_get(&bdrv_em_co_aiocb_info, bs, cb, opaque);
-    acb->req.sector = sector_num;
-    acb->req.nb_sectors = nb_sectors;
-    acb->done = NULL;
-    co = qemu_coroutine_create(bdrv_aio_discard_co_entry);
-    qemu_coroutine_enter(co, acb);
-
-    return &acb->common;
-}
-
-void bdrv_init(void)
-{
-    module_call_init(MODULE_INIT_BLOCK);
-}
-
-void bdrv_init_with_whitelist(void)
-{
-    use_bdrv_whitelist = 1;
-    bdrv_init();
-}
-
-void *qemu_aio_get(const AIOCBInfo *aiocb_info, BlockDriverState *bs,
-                   BlockDriverCompletionFunc *cb, void *opaque)
-{
-    BlockDriverAIOCB *acb;
-
-    acb = g_slice_alloc(aiocb_info->aiocb_size);
-    acb->aiocb_info = aiocb_info;
-    acb->bs = bs;
-    acb->cb = cb;
-    acb->opaque = opaque;
-    return acb;
-}
-
-void qemu_aio_release(void *p)
-{
-    BlockDriverAIOCB *acb = p;
-    g_slice_free1(acb->aiocb_info->aiocb_size, acb);
-}
-
-/**************************************************************/
-/* Coroutine block device emulation */
-
-typedef struct CoroutineIOCompletion {
-    Coroutine *coroutine;
-    int ret;
-} CoroutineIOCompletion;
-
-static void bdrv_co_io_em_complete(void *opaque, int ret)
-{
-    CoroutineIOCompletion *co = opaque;
-
-    co->ret = ret;
-    qemu_coroutine_enter(co->coroutine, NULL);
-}
-
-static int coroutine_fn bdrv_co_io_em(BlockDriverState *bs, int64_t sector_num,
-                                      int nb_sectors, QEMUIOVector *iov,
-                                      bool is_write)
-{
-    CoroutineIOCompletion co = {
-        .coroutine = qemu_coroutine_self(),
-    };
-    BlockDriverAIOCB *acb;
-
-    if (is_write) {
-        acb = bs->drv->bdrv_aio_writev(bs, sector_num, iov, nb_sectors,
-                                       bdrv_co_io_em_complete, &co);
-    } else {
-        acb = bs->drv->bdrv_aio_readv(bs, sector_num, iov, nb_sectors,
-                                      bdrv_co_io_em_complete, &co);
-    }
-
-    trace_bdrv_co_io_em(bs, sector_num, nb_sectors, is_write, acb);
-    if (!acb) {
-        return -EIO;
-    }
-    qemu_coroutine_yield();
-
-    return co.ret;
-}
-
-static int coroutine_fn bdrv_co_readv_em(BlockDriverState *bs,
-                                         int64_t sector_num, int nb_sectors,
-                                         QEMUIOVector *iov)
-{
-    return bdrv_co_io_em(bs, sector_num, nb_sectors, iov, false);
-}
-
-static int coroutine_fn bdrv_co_writev_em(BlockDriverState *bs,
-                                         int64_t sector_num, int nb_sectors,
-                                         QEMUIOVector *iov)
-{
-    return bdrv_co_io_em(bs, sector_num, nb_sectors, iov, true);
-}
-
-static void coroutine_fn bdrv_flush_co_entry(void *opaque)
-{
-    RwCo *rwco = opaque;
-
-    rwco->ret = bdrv_co_flush(rwco->bs);
-}
-
-int coroutine_fn bdrv_co_flush(BlockDriverState *bs)
-{
-    int ret;
-
-    if (!bs || !bdrv_is_inserted(bs) || bdrv_is_read_only(bs)) {
-        return 0;
-    }
-
-    /* Write back cached data to the OS even with cache=unsafe */
-    BLKDBG_EVENT(bs->file, BLKDBG_FLUSH_TO_OS);
-    if (bs->drv->bdrv_co_flush_to_os) {
-        ret = bs->drv->bdrv_co_flush_to_os(bs);
-        if (ret < 0) {
-            return ret;
-        }
-    }
-
-    /* But don't actually force it to the disk with cache=unsafe */
-    if (bs->open_flags & BDRV_O_NO_FLUSH) {
-        goto flush_parent;
-    }
-
-    BLKDBG_EVENT(bs->file, BLKDBG_FLUSH_TO_DISK);
-    if (bs->drv->bdrv_co_flush_to_disk) {
-        ret = bs->drv->bdrv_co_flush_to_disk(bs);
-    } else if (bs->drv->bdrv_aio_flush) {
-        BlockDriverAIOCB *acb;
-        CoroutineIOCompletion co = {
-            .coroutine = qemu_coroutine_self(),
-        };
-
-        acb = bs->drv->bdrv_aio_flush(bs, bdrv_co_io_em_complete, &co);
-        if (acb == NULL) {
-            ret = -EIO;
-        } else {
-            qemu_coroutine_yield();
-            ret = co.ret;
-        }
-    } else {
-        /*
-         * Some block drivers always operate in either writethrough or unsafe
-         * mode and don't support bdrv_flush therefore. Usually qemu doesn't
-         * know how the server works (because the behaviour is hardcoded or
-         * depends on server-side configuration), so we can't ensure that
-         * everything is safe on disk. Returning an error doesn't work because
-         * that would break guests even if the server operates in writethrough
-         * mode.
-         *
-         * Let's hope the user knows what he's doing.
-         */
-        ret = 0;
-    }
-    if (ret < 0) {
-        return ret;
-    }
-
-    /* Now flush the underlying protocol.  It will also have BDRV_O_NO_FLUSH
-     * in the case of cache=unsafe, so there are no useless flushes.
-     */
-flush_parent:
-    return bdrv_co_flush(bs->file);
-}
-
-void bdrv_invalidate_cache(BlockDriverState *bs)
-{
-    if (bs->drv && bs->drv->bdrv_invalidate_cache) {
-        bs->drv->bdrv_invalidate_cache(bs);
-    }
-}
-
-void bdrv_invalidate_cache_all(void)
-{
-    BlockDriverState *bs;
-
-    QTAILQ_FOREACH(bs, &bdrv_states, list) {
-        bdrv_invalidate_cache(bs);
-    }
-}
-
-void bdrv_clear_incoming_migration_all(void)
-{
-    BlockDriverState *bs;
-
-    QTAILQ_FOREACH(bs, &bdrv_states, list) {
-        bs->open_flags = bs->open_flags & ~(BDRV_O_INCOMING);
-    }
-}
-
-int bdrv_flush(BlockDriverState *bs)
-{
-    Coroutine *co;
-    RwCo rwco = {
-        .bs = bs,
-        .ret = NOT_DONE,
-    };
-
-    if (qemu_in_coroutine()) {
-        /* Fast-path if already in coroutine context */
-        bdrv_flush_co_entry(&rwco);
-    } else {
-        co = qemu_coroutine_create(bdrv_flush_co_entry);
-        qemu_coroutine_enter(co, &rwco);
-        while (rwco.ret == NOT_DONE) {
-            qemu_aio_wait();
-        }
-    }
-
-    return rwco.ret;
-}
-
-static void coroutine_fn bdrv_discard_co_entry(void *opaque)
-{
-    RwCo *rwco = opaque;
-
-    rwco->ret = bdrv_co_discard(rwco->bs, rwco->sector_num, rwco->nb_sectors);
-}
-
-int coroutine_fn bdrv_co_discard(BlockDriverState *bs, int64_t sector_num,
-                                 int nb_sectors)
-{
-    if (!bs->drv) {
-        return -ENOMEDIUM;
-    } else if (bdrv_check_request(bs, sector_num, nb_sectors)) {
-        return -EIO;
-    } else if (bs->read_only) {
-        return -EROFS;
-    }
-
-    if (bs->dirty_bitmap) {
-        bdrv_reset_dirty(bs, sector_num, nb_sectors);
-    }
-
-    /* Do nothing if disabled.  */
-    if (!(bs->open_flags & BDRV_O_UNMAP)) {
-        return 0;
-    }
-
-    if (bs->drv->bdrv_co_discard) {
-        return bs->drv->bdrv_co_discard(bs, sector_num, nb_sectors);
-    } else if (bs->drv->bdrv_aio_discard) {
-        BlockDriverAIOCB *acb;
-        CoroutineIOCompletion co = {
-            .coroutine = qemu_coroutine_self(),
-        };
-
-        acb = bs->drv->bdrv_aio_discard(bs, sector_num, nb_sectors,
-                                        bdrv_co_io_em_complete, &co);
-        if (acb == NULL) {
-            return -EIO;
-        } else {
-            qemu_coroutine_yield();
-            return co.ret;
-        }
-    } else {
-        return 0;
-    }
-}
-
-int bdrv_discard(BlockDriverState *bs, int64_t sector_num, int nb_sectors)
-{
-    Coroutine *co;
-    RwCo rwco = {
-        .bs = bs,
-        .sector_num = sector_num,
-        .nb_sectors = nb_sectors,
-        .ret = NOT_DONE,
-    };
-
-    if (qemu_in_coroutine()) {
-        /* Fast-path if already in coroutine context */
-        bdrv_discard_co_entry(&rwco);
-    } else {
-        co = qemu_coroutine_create(bdrv_discard_co_entry);
-        qemu_coroutine_enter(co, &rwco);
-        while (rwco.ret == NOT_DONE) {
-            qemu_aio_wait();
-        }
-    }
-
-    return rwco.ret;
-}
-
-/**************************************************************/
-/* removable device support */
-
-/**
- * Return TRUE if the media is present
- */
-int bdrv_is_inserted(BlockDriverState *bs)
-{
-    BlockDriver *drv = bs->drv;
-
-    if (!drv)
-        return 0;
-    if (!drv->bdrv_is_inserted)
-        return 1;
-    return drv->bdrv_is_inserted(bs);
-}
-
-/**
- * Return whether the media changed since the last call to this
- * function, or -ENOTSUP if we don't know.  Most drivers don't know.
- */
-int bdrv_media_changed(BlockDriverState *bs)
-{
-    BlockDriver *drv = bs->drv;
-
-    if (drv && drv->bdrv_media_changed) {
-        return drv->bdrv_media_changed(bs);
-    }
-    return -ENOTSUP;
-}
-
-/**
- * If eject_flag is TRUE, eject the media. Otherwise, close the tray
- */
-void bdrv_eject(BlockDriverState *bs, bool eject_flag)
-{
-    BlockDriver *drv = bs->drv;
-
-    if (drv && drv->bdrv_eject) {
-        drv->bdrv_eject(bs, eject_flag);
-    }
-
-    if (bs->device_name[0] != '\0') {
-        bdrv_emit_qmp_eject_event(bs, eject_flag);
-    }
-}
-
-/**
- * Lock or unlock the media (if it is locked, the user won't be able
- * to eject it manually).
- */
-void bdrv_lock_medium(BlockDriverState *bs, bool locked)
-{
-    BlockDriver *drv = bs->drv;
-
-    trace_bdrv_lock_medium(bs, locked);
-
-    if (drv && drv->bdrv_lock_medium) {
-        drv->bdrv_lock_medium(bs, locked);
-    }
-}
-
-/* needed for generic scsi interface */
-
-int bdrv_ioctl(BlockDriverState *bs, unsigned long int req, void *buf)
-{
-    BlockDriver *drv = bs->drv;
-
-    if (drv && drv->bdrv_ioctl)
-        return drv->bdrv_ioctl(bs, req, buf);
-    return -ENOTSUP;
-}
-
-BlockDriverAIOCB *bdrv_aio_ioctl(BlockDriverState *bs,
-        unsigned long int req, void *buf,
-        BlockDriverCompletionFunc *cb, void *opaque)
-{
-    BlockDriver *drv = bs->drv;
-
-    if (drv && drv->bdrv_aio_ioctl)
-        return drv->bdrv_aio_ioctl(bs, req, buf, cb, opaque);
-    return NULL;
-}
-
-void bdrv_set_buffer_alignment(BlockDriverState *bs, int align)
-{
-    bs->buffer_alignment = align;
-}
-
-void *qemu_blockalign(BlockDriverState *bs, size_t size)
-{
-    return qemu_memalign((bs && bs->buffer_alignment) ? bs->buffer_alignment : 512, size);
-}
-
-/*
- * Check if all memory in this vector is sector aligned.
- */
-bool bdrv_qiov_is_aligned(BlockDriverState *bs, QEMUIOVector *qiov)
-{
-    int i;
-
-    for (i = 0; i < qiov->niov; i++) {
-        if ((uintptr_t) qiov->iov[i].iov_base % bs->buffer_alignment) {
-            return false;
-        }
-    }
-
-    return true;
-}
-
-void bdrv_set_dirty_tracking(BlockDriverState *bs, int granularity)
-{
-    int64_t bitmap_size;
-
-    assert((granularity & (granularity - 1)) == 0);
-
-    if (granularity) {
-        granularity >>= BDRV_SECTOR_BITS;
-        assert(!bs->dirty_bitmap);
-        bitmap_size = (bdrv_getlength(bs) >> BDRV_SECTOR_BITS);
-        bs->dirty_bitmap = hbitmap_alloc(bitmap_size, ffs(granularity) - 1);
-    } else {
-        if (bs->dirty_bitmap) {
-            hbitmap_free(bs->dirty_bitmap);
-            bs->dirty_bitmap = NULL;
-        }
-    }
-}
-
-int bdrv_get_dirty(BlockDriverState *bs, int64_t sector)
-{
-    if (bs->dirty_bitmap) {
-        return hbitmap_get(bs->dirty_bitmap, sector);
-    } else {
-        return 0;
-    }
-}
-
-void bdrv_dirty_iter_init(BlockDriverState *bs, HBitmapIter *hbi)
-{
-    hbitmap_iter_init(hbi, bs->dirty_bitmap, 0);
-}
-
-void bdrv_set_dirty(BlockDriverState *bs, int64_t cur_sector,
-                    int nr_sectors)
-{
-    hbitmap_set(bs->dirty_bitmap, cur_sector, nr_sectors);
-}
-
-void bdrv_reset_dirty(BlockDriverState *bs, int64_t cur_sector,
-                      int nr_sectors)
-{
-    hbitmap_reset(bs->dirty_bitmap, cur_sector, nr_sectors);
-}
-
-int64_t bdrv_get_dirty_count(BlockDriverState *bs)
-{
-    if (bs->dirty_bitmap) {
-        return hbitmap_count(bs->dirty_bitmap);
-    } else {
-        return 0;
-    }
-}
-
-void bdrv_set_in_use(BlockDriverState *bs, int in_use)
-{
-    assert(bs->in_use != in_use);
-    bs->in_use = in_use;
-}
-
-int bdrv_in_use(BlockDriverState *bs)
-{
-    return bs->in_use;
-}
-
-void bdrv_iostatus_enable(BlockDriverState *bs)
-{
-    bs->iostatus_enabled = true;
-    bs->iostatus = BLOCK_DEVICE_IO_STATUS_OK;
-}
-
-/* The I/O status is only enabled if the drive explicitly
- * enables it _and_ the VM is configured to stop on errors */
-bool bdrv_iostatus_is_enabled(const BlockDriverState *bs)
-{
-    return (bs->iostatus_enabled &&
-           (bs->on_write_error == BLOCKDEV_ON_ERROR_ENOSPC ||
-            bs->on_write_error == BLOCKDEV_ON_ERROR_STOP   ||
-            bs->on_read_error == BLOCKDEV_ON_ERROR_STOP));
-}
-
-void bdrv_iostatus_disable(BlockDriverState *bs)
-{
-    bs->iostatus_enabled = false;
-}
-
-void bdrv_iostatus_reset(BlockDriverState *bs)
-{
-    if (bdrv_iostatus_is_enabled(bs)) {
-        bs->iostatus = BLOCK_DEVICE_IO_STATUS_OK;
-        if (bs->job) {
-            block_job_iostatus_reset(bs->job);
-        }
-    }
-}
-
-void bdrv_iostatus_set_err(BlockDriverState *bs, int error)
-{
-    assert(bdrv_iostatus_is_enabled(bs));
-    if (bs->iostatus == BLOCK_DEVICE_IO_STATUS_OK) {
-        bs->iostatus = error == ENOSPC ? BLOCK_DEVICE_IO_STATUS_NOSPACE :
-                                         BLOCK_DEVICE_IO_STATUS_FAILED;
-    }
-}
-
-void
-bdrv_acct_start(BlockDriverState *bs, BlockAcctCookie *cookie, int64_t bytes,
-        enum BlockAcctType type)
-{
-    assert(type < BDRV_MAX_IOTYPE);
-
-    cookie->bytes = bytes;
-    cookie->start_time_ns = get_clock();
-    cookie->type = type;
-}
-
-void
-bdrv_acct_done(BlockDriverState *bs, BlockAcctCookie *cookie)
-{
-    assert(cookie->type < BDRV_MAX_IOTYPE);
-
-    bs->nr_bytes[cookie->type] += cookie->bytes;
-    bs->nr_ops[cookie->type]++;
-    bs->total_time_ns[cookie->type] += get_clock() - cookie->start_time_ns;
-}
-
-void bdrv_img_create(const char *filename, const char *fmt,
-                     const char *base_filename, const char *base_fmt,
-                     char *options, uint64_t img_size, int flags,
-                     Error **errp, bool quiet)
-{
-    QEMUOptionParameter *param = NULL, *create_options = NULL;
-    QEMUOptionParameter *backing_fmt, *backing_file, *size;
-    BlockDriverState *bs = NULL;
-    BlockDriver *drv, *proto_drv;
-    BlockDriver *backing_drv = NULL;
-    int ret = 0;
-
-    /* Find driver and parse its options */
-    drv = bdrv_find_format(fmt);
-    if (!drv) {
-        error_setg(errp, "Unknown file format '%s'", fmt);
-        return;
-    }
-
-    proto_drv = bdrv_find_protocol(filename, true);
-    if (!proto_drv) {
-        error_setg(errp, "Unknown protocol '%s'", filename);
-        return;
-    }
-
-    create_options = append_option_parameters(create_options,
-                                              drv->create_options);
-    create_options = append_option_parameters(create_options,
-                                              proto_drv->create_options);
-
-    /* Create parameter list with default values */
-    param = parse_option_parameters("", create_options, param);
-
-    set_option_parameter_int(param, BLOCK_OPT_SIZE, img_size);
-
-    /* Parse -o options */
-    if (options) {
-        param = parse_option_parameters(options, create_options, param);
-        if (param == NULL) {
-            error_setg(errp, "Invalid options for file format '%s'.", fmt);
-            goto out;
-        }
-    }
-
-    if (base_filename) {
-        if (set_option_parameter(param, BLOCK_OPT_BACKING_FILE,
-                                 base_filename)) {
-            error_setg(errp, "Backing file not supported for file format '%s'",
-                       fmt);
-            goto out;
-        }
-    }
-
-    if (base_fmt) {
-        if (set_option_parameter(param, BLOCK_OPT_BACKING_FMT, base_fmt)) {
-            error_setg(errp, "Backing file format not supported for file "
-                             "format '%s'", fmt);
-            goto out;
-        }
-    }
-
-    backing_file = get_option_parameter(param, BLOCK_OPT_BACKING_FILE);
-    if (backing_file && backing_file->value.s) {
-        if (!strcmp(filename, backing_file->value.s)) {
-            error_setg(errp, "Error: Trying to create an image with the "
-                             "same filename as the backing file");
-            goto out;
-        }
-    }
-
-    backing_fmt = get_option_parameter(param, BLOCK_OPT_BACKING_FMT);
-    if (backing_fmt && backing_fmt->value.s) {
-        backing_drv = bdrv_find_format(backing_fmt->value.s);
-        if (!backing_drv) {
-            error_setg(errp, "Unknown backing file format '%s'",
-                       backing_fmt->value.s);
-            goto out;
-        }
-    }
-
-    // The size for the image must always be specified, with one exception:
-    // If we are using a backing file, we can obtain the size from there
-    size = get_option_parameter(param, BLOCK_OPT_SIZE);
-    if (size && size->value.n == -1) {
-        if (backing_file && backing_file->value.s) {
-            uint64_t size;
-            char buf[32];
-            int back_flags;
-
-            /* backing files always opened read-only */
-            back_flags =
-                flags & ~(BDRV_O_RDWR | BDRV_O_SNAPSHOT | BDRV_O_NO_BACKING);
-
-            bs = bdrv_new("");
-
-            ret = bdrv_open(bs, backing_file->value.s, NULL, back_flags,
-                            backing_drv);
-            if (ret < 0) {
-                error_setg_errno(errp, -ret, "Could not open '%s'",
-                                 backing_file->value.s);
-                goto out;
-            }
-            bdrv_get_geometry(bs, &size);
-            size *= 512;
-
-            snprintf(buf, sizeof(buf), "%" PRId64, size);
-            set_option_parameter(param, BLOCK_OPT_SIZE, buf);
-        } else {
-            error_setg(errp, "Image creation needs a size parameter");
-            goto out;
-        }
-    }
-
-    if (!quiet) {
-        printf("Formatting '%s', fmt=%s ", filename, fmt);
-        print_option_parameters(param);
-        puts("");
-    }
-    ret = bdrv_create(drv, filename, param);
-    if (ret < 0) {
-        if (ret == -ENOTSUP) {
-            error_setg(errp,"Formatting or formatting option not supported for "
-                            "file format '%s'", fmt);
-        } else if (ret == -EFBIG) {
-            const char *cluster_size_hint = "";
-            if (get_option_parameter(create_options, BLOCK_OPT_CLUSTER_SIZE)) {
-                cluster_size_hint = " (try using a larger cluster size)";
-            }
-            error_setg(errp, "The image size is too large for file format '%s'%s",
-                       fmt, cluster_size_hint);
-        } else {
-            error_setg(errp, "%s: error while creating %s: %s", filename, fmt,
-                       strerror(-ret));
-        }
-    }
-
-out:
-    free_option_parameters(create_options);
-    free_option_parameters(param);
-
-    if (bs) {
-        bdrv_delete(bs);
-    }
-}
-
-AioContext *bdrv_get_aio_context(BlockDriverState *bs)
-{
-    /* Currently BlockDriverState always uses the main loop AioContext */
-    return qemu_get_aio_context();
-}
-
-void bdrv_add_before_write_notifier(BlockDriverState *bs,
-                                    NotifierWithReturn *notifier)
-{
-    notifier_with_return_list_add(&bs->before_write_notifiers, notifier);
-}
diff --git a/contrib/qemu/block/qcow.c b/contrib/qemu/block/qcow.c
deleted file mode 100644
index 5239bd68f1c..00000000000
--- a/contrib/qemu/block/qcow.c
+++ /dev/null
@@ -1,914 +0,0 @@
-/*
- * Block driver for the QCOW format
- *
- * Copyright (c) 2004-2006 Fabrice Bellard
- *
- * Permission is hereby granted, free of charge, to any person obtaining a copy
- * of this software and associated documentation files (the "Software"), to deal
- * in the Software without restriction, including without limitation the rights
- * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
- * copies of the Software, and to permit persons to whom the Software is
- * furnished to do so, subject to the following conditions:
- *
- * The above copyright notice and this permission notice shall be included in
- * all copies or substantial portions of the Software.
- *
- * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
- * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
- * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
- * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
- * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
- * THE SOFTWARE.
- */
-#include "qemu-common.h"
-#include "block/block_int.h"
-#include "qemu/module.h"
-#include <zlib.h>
-#include "qemu/aes.h"
-#include "migration/migration.h"
-
-/**************************************************************/
-/* QEMU COW block driver with compression and encryption support */
-
-#define QCOW_MAGIC (('Q' << 24) | ('F' << 16) | ('I' << 8) | 0xfb)
-#define QCOW_VERSION 1
-
-#define QCOW_CRYPT_NONE 0
-#define QCOW_CRYPT_AES  1
-
-#define QCOW_OFLAG_COMPRESSED (1LL << 63)
-
-typedef struct QCowHeader {
-    uint32_t magic;
-    uint32_t version;
-    uint64_t backing_file_offset;
-    uint32_t backing_file_size;
-    uint32_t mtime;
-    uint64_t size; /* in bytes */
-    uint8_t cluster_bits;
-    uint8_t l2_bits;
-    uint32_t crypt_method;
-    uint64_t l1_table_offset;
-} QCowHeader;
-
-#define L2_CACHE_SIZE 16
-
-typedef struct BDRVQcowState {
-    int cluster_bits;
-    int cluster_size;
-    int cluster_sectors;
-    int l2_bits;
-    int l2_size;
-    int l1_size;
-    uint64_t cluster_offset_mask;
-    uint64_t l1_table_offset;
-    uint64_t *l1_table;
-    uint64_t *l2_cache;
-    uint64_t l2_cache_offsets[L2_CACHE_SIZE];
-    uint32_t l2_cache_counts[L2_CACHE_SIZE];
-    uint8_t *cluster_cache;
-    uint8_t *cluster_data;
-    uint64_t cluster_cache_offset;
-    uint32_t crypt_method; /* current crypt method, 0 if no key yet */
-    uint32_t crypt_method_header;
-    AES_KEY aes_encrypt_key;
-    AES_KEY aes_decrypt_key;
-    CoMutex lock;
-    Error *migration_blocker;
-} BDRVQcowState;
-
-static int decompress_cluster(BlockDriverState *bs, uint64_t cluster_offset);
-
-static int qcow_probe(const uint8_t *buf, int buf_size, const char *filename)
-{
-    const QCowHeader *cow_header = (const void *)buf;
-
-    if (buf_size >= sizeof(QCowHeader) &&
-        be32_to_cpu(cow_header->magic) == QCOW_MAGIC &&
-        be32_to_cpu(cow_header->version) == QCOW_VERSION)
-        return 100;
-    else
-        return 0;
-}
-
-static int qcow_open(BlockDriverState *bs, QDict *options, int flags)
-{
-    BDRVQcowState *s = bs->opaque;
-    int len, i, shift, ret;
-    QCowHeader header;
-
-    ret = bdrv_pread(bs->file, 0, &header, sizeof(header));
-    if (ret < 0) {
-        goto fail;
-    }
-    be32_to_cpus(&header.magic);
-    be32_to_cpus(&header.version);
-    be64_to_cpus(&header.backing_file_offset);
-    be32_to_cpus(&header.backing_file_size);
-    be32_to_cpus(&header.mtime);
-    be64_to_cpus(&header.size);
-    be32_to_cpus(&header.crypt_method);
-    be64_to_cpus(&header.l1_table_offset);
-
-    if (header.magic != QCOW_MAGIC) {
-        ret = -EMEDIUMTYPE;
-        goto fail;
-    }
-    if (header.version != QCOW_VERSION) {
-        char version[64];
-        snprintf(version, sizeof(version), "QCOW version %d", header.version);
-        qerror_report(QERR_UNKNOWN_BLOCK_FORMAT_FEATURE,
-            bs->device_name, "qcow", version);
-        ret = -ENOTSUP;
-        goto fail;
-    }
-
-    if (header.size <= 1 || header.cluster_bits < 9) {
-        ret = -EINVAL;
-        goto fail;
-    }
-    if (header.crypt_method > QCOW_CRYPT_AES) {
-        ret = -EINVAL;
-        goto fail;
-    }
-    s->crypt_method_header = header.crypt_method;
-    if (s->crypt_method_header) {
-        bs->encrypted = 1;
-    }
-    s->cluster_bits = header.cluster_bits;
-    s->cluster_size = 1 << s->cluster_bits;
-    s->cluster_sectors = 1 << (s->cluster_bits - 9);
-    s->l2_bits = header.l2_bits;
-    s->l2_size = 1 << s->l2_bits;
-    bs->total_sectors = header.size / 512;
-    s->cluster_offset_mask = (1LL << (63 - s->cluster_bits)) - 1;
-
-    /* read the level 1 table */
-    shift = s->cluster_bits + s->l2_bits;
-    s->l1_size = (header.size + (1LL << shift) - 1) >> shift;
-
-    s->l1_table_offset = header.l1_table_offset;
-    s->l1_table = g_malloc(s->l1_size * sizeof(uint64_t));
-
-    ret = bdrv_pread(bs->file, s->l1_table_offset, s->l1_table,
-               s->l1_size * sizeof(uint64_t));
-    if (ret < 0) {
-        goto fail;
-    }
-
-    for(i = 0;i < s->l1_size; i++) {
-        be64_to_cpus(&s->l1_table[i]);
-    }
-    /* alloc L2 cache */
-    s->l2_cache = g_malloc(s->l2_size * L2_CACHE_SIZE * sizeof(uint64_t));
-    s->cluster_cache = g_malloc(s->cluster_size);
-    s->cluster_data = g_malloc(s->cluster_size);
-    s->cluster_cache_offset = -1;
-
-    /* read the backing file name */
-    if (header.backing_file_offset != 0) {
-        len = header.backing_file_size;
-        if (len > 1023) {
-            len = 1023;
-        }
-        ret = bdrv_pread(bs->file, header.backing_file_offset,
-                   bs->backing_file, len);
-        if (ret < 0) {
-            goto fail;
-        }
-        bs->backing_file[len] = '\0';
-    }
-
-    /* Disable migration when qcow images are used */
-    error_set(&s->migration_blocker,
-              QERR_BLOCK_FORMAT_FEATURE_NOT_SUPPORTED,
-              "qcow", bs->device_name, "live migration");
-    migrate_add_blocker(s->migration_blocker);
-
-    qemu_co_mutex_init(&s->lock);
-    return 0;
-
- fail:
-    g_free(s->l1_table);
-    g_free(s->l2_cache);
-    g_free(s->cluster_cache);
-    g_free(s->cluster_data);
-    return ret;
-}
-
-
-/* We have nothing to do for QCOW reopen, stubs just return
- * success */
-static int qcow_reopen_prepare(BDRVReopenState *state,
-                               BlockReopenQueue *queue, Error **errp)
-{
-    return 0;
-}
-
-static int qcow_set_key(BlockDriverState *bs, const char *key)
-{
-    BDRVQcowState *s = bs->opaque;
-    uint8_t keybuf[16];
-    int len, i;
-
-    memset(keybuf, 0, 16);
-    len = strlen(key);
-    if (len > 16)
-        len = 16;
-    /* XXX: we could compress the chars to 7 bits to increase
-       entropy */
-    for(i = 0;i < len;i++) {
-        keybuf[i] = key[i];
-    }
-    s->crypt_method = s->crypt_method_header;
-
-    if (AES_set_encrypt_key(keybuf, 128, &s->aes_encrypt_key) != 0)
-        return -1;
-    if (AES_set_decrypt_key(keybuf, 128, &s->aes_decrypt_key) != 0)
-        return -1;
-    return 0;
-}
-
-/* The crypt function is compatible with the linux cryptoloop
-   algorithm for < 4 GB images. NOTE: out_buf == in_buf is
-   supported */
-static void encrypt_sectors(BDRVQcowState *s, int64_t sector_num,
-                            uint8_t *out_buf, const uint8_t *in_buf,
-                            int nb_sectors, int enc,
-                            const AES_KEY *key)
-{
-    union {
-        uint64_t ll[2];
-        uint8_t b[16];
-    } ivec;
-    int i;
-
-    for(i = 0; i < nb_sectors; i++) {
-        ivec.ll[0] = cpu_to_le64(sector_num);
-        ivec.ll[1] = 0;
-        AES_cbc_encrypt(in_buf, out_buf, 512, key,
-                        ivec.b, enc);
-        sector_num++;
-        in_buf += 512;
-        out_buf += 512;
-    }
-}
-
-/* 'allocate' is:
- *
- * 0 to not allocate.
- *
- * 1 to allocate a normal cluster (for sector indexes 'n_start' to
- * 'n_end')
- *
- * 2 to allocate a compressed cluster of size
- * 'compressed_size'. 'compressed_size' must be > 0 and <
- * cluster_size
- *
- * return 0 if not allocated.
- */
-static uint64_t get_cluster_offset(BlockDriverState *bs,
-                                   uint64_t offset, int allocate,
-                                   int compressed_size,
-                                   int n_start, int n_end)
-{
-    BDRVQcowState *s = bs->opaque;
-    int min_index, i, j, l1_index, l2_index;
-    uint64_t l2_offset, *l2_table, cluster_offset, tmp;
-    uint32_t min_count;
-    int new_l2_table;
-
-    l1_index = offset >> (s->l2_bits + s->cluster_bits);
-    l2_offset = s->l1_table[l1_index];
-    new_l2_table = 0;
-    if (!l2_offset) {
-        if (!allocate)
-            return 0;
-        /* allocate a new l2 entry */
-        l2_offset = bdrv_getlength(bs->file);
-        /* round to cluster size */
-        l2_offset = (l2_offset + s->cluster_size - 1) & ~(s->cluster_size - 1);
-        /* update the L1 entry */
-        s->l1_table[l1_index] = l2_offset;
-        tmp = cpu_to_be64(l2_offset);
-        if (bdrv_pwrite_sync(bs->file,
-                s->l1_table_offset + l1_index * sizeof(tmp),
-                &tmp, sizeof(tmp)) < 0)
-            return 0;
-        new_l2_table = 1;
-    }
-    for(i = 0; i < L2_CACHE_SIZE; i++) {
-        if (l2_offset == s->l2_cache_offsets[i]) {
-            /* increment the hit count */
-            if (++s->l2_cache_counts[i] == 0xffffffff) {
-                for(j = 0; j < L2_CACHE_SIZE; j++) {
-                    s->l2_cache_counts[j] >>= 1;
-                }
-            }
-            l2_table = s->l2_cache + (i << s->l2_bits);
-            goto found;
-        }
-    }
-    /* not found: load a new entry in the least used one */
-    min_index = 0;
-    min_count = 0xffffffff;
-    for(i = 0; i < L2_CACHE_SIZE; i++) {
-        if (s->l2_cache_counts[i] < min_count) {
-            min_count = s->l2_cache_counts[i];
-            min_index = i;
-        }
-    }
-    l2_table = s->l2_cache + (min_index << s->l2_bits);
-    if (new_l2_table) {
-        memset(l2_table, 0, s->l2_size * sizeof(uint64_t));
-        if (bdrv_pwrite_sync(bs->file, l2_offset, l2_table,
-                s->l2_size * sizeof(uint64_t)) < 0)
-            return 0;
-    } else {
-        if (bdrv_pread(bs->file, l2_offset, l2_table, s->l2_size * sizeof(uint64_t)) !=
-            s->l2_size * sizeof(uint64_t))
-            return 0;
-    }
-    s->l2_cache_offsets[min_index] = l2_offset;
-    s->l2_cache_counts[min_index] = 1;
- found:
-    l2_index = (offset >> s->cluster_bits) & (s->l2_size - 1);
-    cluster_offset = be64_to_cpu(l2_table[l2_index]);
-    if (!cluster_offset ||
-        ((cluster_offset & QCOW_OFLAG_COMPRESSED) && allocate == 1)) {
-        if (!allocate)
-            return 0;
-        /* allocate a new cluster */
-        if ((cluster_offset & QCOW_OFLAG_COMPRESSED) &&
-            (n_end - n_start) < s->cluster_sectors) {
-            /* if the cluster is already compressed, we must
-               decompress it in the case it is not completely
-               overwritten */
-            if (decompress_cluster(bs, cluster_offset) < 0)
-                return 0;
-            cluster_offset = bdrv_getlength(bs->file);
-            cluster_offset = (cluster_offset + s->cluster_size - 1) &
-                ~(s->cluster_size - 1);
-            /* write the cluster content */
-            if (bdrv_pwrite(bs->file, cluster_offset, s->cluster_cache, s->cluster_size) !=
-                s->cluster_size)
-                return -1;
-        } else {
-            cluster_offset = bdrv_getlength(bs->file);
-            if (allocate == 1) {
-                /* round to cluster size */
-                cluster_offset = (cluster_offset + s->cluster_size - 1) &
-                    ~(s->cluster_size - 1);
-                bdrv_truncate(bs->file, cluster_offset + s->cluster_size);
-                /* if encrypted, we must initialize the cluster
-                   content which won't be written */
-                if (s->crypt_method &&
-                    (n_end - n_start) < s->cluster_sectors) {
-                    uint64_t start_sect;
-                    start_sect = (offset & ~(s->cluster_size - 1)) >> 9;
-                    memset(s->cluster_data + 512, 0x00, 512);
-                    for(i = 0; i < s->cluster_sectors; i++) {
-                        if (i < n_start || i >= n_end) {
-                            encrypt_sectors(s, start_sect + i,
-                                            s->cluster_data,
-                                            s->cluster_data + 512, 1, 1,
-                                            &s->aes_encrypt_key);
-                            if (bdrv_pwrite(bs->file, cluster_offset + i * 512,
-                                            s->cluster_data, 512) != 512)
-                                return -1;
-                        }
-                    }
-                }
-            } else if (allocate == 2) {
-                cluster_offset |= QCOW_OFLAG_COMPRESSED |
-                    (uint64_t)compressed_size << (63 - s->cluster_bits);
-            }
-        }
-        /* update L2 table */
-        tmp = cpu_to_be64(cluster_offset);
-        l2_table[l2_index] = tmp;
-        if (bdrv_pwrite_sync(bs->file, l2_offset + l2_index * sizeof(tmp),
-                &tmp, sizeof(tmp)) < 0)
-            return 0;
-    }
-    return cluster_offset;
-}
-
-static int coroutine_fn qcow_co_is_allocated(BlockDriverState *bs,
-        int64_t sector_num, int nb_sectors, int *pnum)
-{
-    BDRVQcowState *s = bs->opaque;
-    int index_in_cluster, n;
-    uint64_t cluster_offset;
-
-    qemu_co_mutex_lock(&s->lock);
-    cluster_offset = get_cluster_offset(bs, sector_num << 9, 0, 0, 0, 0);
-    qemu_co_mutex_unlock(&s->lock);
-    index_in_cluster = sector_num & (s->cluster_sectors - 1);
-    n = s->cluster_sectors - index_in_cluster;
-    if (n > nb_sectors)
-        n = nb_sectors;
-    *pnum = n;
-    return (cluster_offset != 0);
-}
-
-static int decompress_buffer(uint8_t *out_buf, int out_buf_size,
-                             const uint8_t *buf, int buf_size)
-{
-    z_stream strm1, *strm = &strm1;
-    int ret, out_len;
-
-    memset(strm, 0, sizeof(*strm));
-
-    strm->next_in = (uint8_t *)buf;
-    strm->avail_in = buf_size;
-    strm->next_out = out_buf;
-    strm->avail_out = out_buf_size;
-
-    ret = inflateInit2(strm, -12);
-    if (ret != Z_OK)
-        return -1;
-    ret = inflate(strm, Z_FINISH);
-    out_len = strm->next_out - out_buf;
-    if ((ret != Z_STREAM_END && ret != Z_BUF_ERROR) ||
-        out_len != out_buf_size) {
-        inflateEnd(strm);
-        return -1;
-    }
-    inflateEnd(strm);
-    return 0;
-}
-
-static int decompress_cluster(BlockDriverState *bs, uint64_t cluster_offset)
-{
-    BDRVQcowState *s = bs->opaque;
-    int ret, csize;
-    uint64_t coffset;
-
-    coffset = cluster_offset & s->cluster_offset_mask;
-    if (s->cluster_cache_offset != coffset) {
-        csize = cluster_offset >> (63 - s->cluster_bits);
-        csize &= (s->cluster_size - 1);
-        ret = bdrv_pread(bs->file, coffset, s->cluster_data, csize);
-        if (ret != csize)
-            return -1;
-        if (decompress_buffer(s->cluster_cache, s->cluster_size,
-                              s->cluster_data, csize) < 0) {
-            return -1;
-        }
-        s->cluster_cache_offset = coffset;
-    }
-    return 0;
-}
-
-static coroutine_fn int qcow_co_readv(BlockDriverState *bs, int64_t sector_num,
-                         int nb_sectors, QEMUIOVector *qiov)
-{
-    BDRVQcowState *s = bs->opaque;
-    int index_in_cluster;
-    int ret = 0, n;
-    uint64_t cluster_offset;
-    struct iovec hd_iov;
-    QEMUIOVector hd_qiov;
-    uint8_t *buf;
-    void *orig_buf;
-
-    if (qiov->niov > 1) {
-        buf = orig_buf = qemu_blockalign(bs, qiov->size);
-    } else {
-        orig_buf = NULL;
-        buf = (uint8_t *)qiov->iov->iov_base;
-    }
-
-    qemu_co_mutex_lock(&s->lock);
-
-    while (nb_sectors != 0) {
-        /* prepare next request */
-        cluster_offset = get_cluster_offset(bs, sector_num << 9,
-                                                 0, 0, 0, 0);
-        index_in_cluster = sector_num & (s->cluster_sectors - 1);
-        n = s->cluster_sectors - index_in_cluster;
-        if (n > nb_sectors) {
-            n = nb_sectors;
-        }
-
-        if (!cluster_offset) {
-            if (bs->backing_hd) {
-                /* read from the base image */
-                hd_iov.iov_base = (void *)buf;
-                hd_iov.iov_len = n * 512;
-                qemu_iovec_init_external(&hd_qiov, &hd_iov, 1);
-                qemu_co_mutex_unlock(&s->lock);
-                ret = bdrv_co_readv(bs->backing_hd, sector_num,
-                                    n, &hd_qiov);
-                qemu_co_mutex_lock(&s->lock);
-                if (ret < 0) {
-                    goto fail;
-                }
-            } else {
-                /* Note: in this case, no need to wait */
-                memset(buf, 0, 512 * n);
-            }
-        } else if (cluster_offset & QCOW_OFLAG_COMPRESSED) {
-            /* add AIO support for compressed blocks ? */
-            if (decompress_cluster(bs, cluster_offset) < 0) {
-                goto fail;
-            }
-            memcpy(buf,
-                   s->cluster_cache + index_in_cluster * 512, 512 * n);
-        } else {
-            if ((cluster_offset & 511) != 0) {
-                goto fail;
-            }
-            hd_iov.iov_base = (void *)buf;
-            hd_iov.iov_len = n * 512;
-            qemu_iovec_init_external(&hd_qiov, &hd_iov, 1);
-            qemu_co_mutex_unlock(&s->lock);
-            ret = bdrv_co_readv(bs->file,
-                                (cluster_offset >> 9) + index_in_cluster,
-                                n, &hd_qiov);
-            qemu_co_mutex_lock(&s->lock);
-            if (ret < 0) {
-                break;
-            }
-            if (s->crypt_method) {
-                encrypt_sectors(s, sector_num, buf, buf,
-                                n, 0,
-                                &s->aes_decrypt_key);
-            }
-        }
-        ret = 0;
-
-        nb_sectors -= n;
-        sector_num += n;
-        buf += n * 512;
-    }
-
-done:
-    qemu_co_mutex_unlock(&s->lock);
-
-    if (qiov->niov > 1) {
-        qemu_iovec_from_buf(qiov, 0, orig_buf, qiov->size);
-        qemu_vfree(orig_buf);
-    }
-
-    return ret;
-
-fail:
-    ret = -EIO;
-    goto done;
-}
-
-static coroutine_fn int qcow_co_writev(BlockDriverState *bs, int64_t sector_num,
-                          int nb_sectors, QEMUIOVector *qiov)
-{
-    BDRVQcowState *s = bs->opaque;
-    int index_in_cluster;
-    uint64_t cluster_offset;
-    const uint8_t *src_buf;
-    int ret = 0, n;
-    uint8_t *cluster_data = NULL;
-    struct iovec hd_iov;
-    QEMUIOVector hd_qiov;
-    uint8_t *buf;
-    void *orig_buf;
-
-    s->cluster_cache_offset = -1; /* disable compressed cache */
-
-    if (qiov->niov > 1) {
-        buf = orig_buf = qemu_blockalign(bs, qiov->size);
-        qemu_iovec_to_buf(qiov, 0, buf, qiov->size);
-    } else {
-        orig_buf = NULL;
-        buf = (uint8_t *)qiov->iov->iov_base;
-    }
-
-    qemu_co_mutex_lock(&s->lock);
-
-    while (nb_sectors != 0) {
-
-        index_in_cluster = sector_num & (s->cluster_sectors - 1);
-        n = s->cluster_sectors - index_in_cluster;
-        if (n > nb_sectors) {
-            n = nb_sectors;
-        }
-        cluster_offset = get_cluster_offset(bs, sector_num << 9, 1, 0,
-                                            index_in_cluster,
-                                            index_in_cluster + n);
-        if (!cluster_offset || (cluster_offset & 511) != 0) {
-            ret = -EIO;
-            break;
-        }
-        if (s->crypt_method) {
-            if (!cluster_data) {
-                cluster_data = g_malloc0(s->cluster_size);
-            }
-            encrypt_sectors(s, sector_num, cluster_data, buf,
-                            n, 1, &s->aes_encrypt_key);
-            src_buf = cluster_data;
-        } else {
-            src_buf = buf;
-        }
-
-        hd_iov.iov_base = (void *)src_buf;
-        hd_iov.iov_len = n * 512;
-        qemu_iovec_init_external(&hd_qiov, &hd_iov, 1);
-        qemu_co_mutex_unlock(&s->lock);
-        ret = bdrv_co_writev(bs->file,
-                             (cluster_offset >> 9) + index_in_cluster,
-                             n, &hd_qiov);
-        qemu_co_mutex_lock(&s->lock);
-        if (ret < 0) {
-            break;
-        }
-        ret = 0;
-
-        nb_sectors -= n;
-        sector_num += n;
-        buf += n * 512;
-    }
-    qemu_co_mutex_unlock(&s->lock);
-
-    if (qiov->niov > 1) {
-        qemu_vfree(orig_buf);
-    }
-    g_free(cluster_data);
-
-    return ret;
-}
-
-static void qcow_close(BlockDriverState *bs)
-{
-    BDRVQcowState *s = bs->opaque;
-
-    g_free(s->l1_table);
-    g_free(s->l2_cache);
-    g_free(s->cluster_cache);
-    g_free(s->cluster_data);
-
-    migrate_del_blocker(s->migration_blocker);
-    error_free(s->migration_blocker);
-}
-
-static int qcow_create(const char *filename, QEMUOptionParameter *options)
-{
-    int header_size, backing_filename_len, l1_size, shift, i;
-    QCowHeader header;
-    uint8_t *tmp;
-    int64_t total_size = 0;
-    const char *backing_file = NULL;
-    int flags = 0;
-    int ret;
-    BlockDriverState *qcow_bs;
-
-    /* Read out options */
-    while (options && options->name) {
-        if (!strcmp(options->name, BLOCK_OPT_SIZE)) {
-            total_size = options->value.n / 512;
-        } else if (!strcmp(options->name, BLOCK_OPT_BACKING_FILE)) {
-            backing_file = options->value.s;
-        } else if (!strcmp(options->name, BLOCK_OPT_ENCRYPT)) {
-            flags |= options->value.n ? BLOCK_FLAG_ENCRYPT : 0;
-        }
-        options++;
-    }
-
-    ret = bdrv_create_file(filename, options);
-    if (ret < 0) {
-        return ret;
-    }
-
-    ret = bdrv_file_open(&qcow_bs, filename, NULL, BDRV_O_RDWR);
-    if (ret < 0) {
-        return ret;
-    }
-
-    ret = bdrv_truncate(qcow_bs, 0);
-    if (ret < 0) {
-        goto exit;
-    }
-
-    memset(&header, 0, sizeof(header));
-    header.magic = cpu_to_be32(QCOW_MAGIC);
-    header.version = cpu_to_be32(QCOW_VERSION);
-    header.size = cpu_to_be64(total_size * 512);
-    header_size = sizeof(header);
-    backing_filename_len = 0;
-    if (backing_file) {
-        if (strcmp(backing_file, "fat:")) {
-            header.backing_file_offset = cpu_to_be64(header_size);
-            backing_filename_len = strlen(backing_file);
-            header.backing_file_size = cpu_to_be32(backing_filename_len);
-            header_size += backing_filename_len;
-        } else {
-            /* special backing file for vvfat */
-            backing_file = NULL;
-        }
-        header.cluster_bits = 9; /* 512 byte cluster to avoid copying
-                                    unmodifyed sectors */
-        header.l2_bits = 12; /* 32 KB L2 tables */
-    } else {
-        header.cluster_bits = 12; /* 4 KB clusters */
-        header.l2_bits = 9; /* 4 KB L2 tables */
-    }
-    header_size = (header_size + 7) & ~7;
-    shift = header.cluster_bits + header.l2_bits;
-    l1_size = ((total_size * 512) + (1LL << shift) - 1) >> shift;
-
-    header.l1_table_offset = cpu_to_be64(header_size);
-    if (flags & BLOCK_FLAG_ENCRYPT) {
-        header.crypt_method = cpu_to_be32(QCOW_CRYPT_AES);
-    } else {
-        header.crypt_method = cpu_to_be32(QCOW_CRYPT_NONE);
-    }
-
-    /* write all the data */
-    ret = bdrv_pwrite(qcow_bs, 0, &header, sizeof(header));
-    if (ret != sizeof(header)) {
-        goto exit;
-    }
-
-    if (backing_file) {
-        ret = bdrv_pwrite(qcow_bs, sizeof(header),
-            backing_file, backing_filename_len);
-        if (ret != backing_filename_len) {
-            goto exit;
-        }
-    }
-
-    tmp = g_malloc0(BDRV_SECTOR_SIZE);
-    for (i = 0; i < ((sizeof(uint64_t)*l1_size + BDRV_SECTOR_SIZE - 1)/
-        BDRV_SECTOR_SIZE); i++) {
-        ret = bdrv_pwrite(qcow_bs, header_size +
-            BDRV_SECTOR_SIZE*i, tmp, BDRV_SECTOR_SIZE);
-        if (ret != BDRV_SECTOR_SIZE) {
-            g_free(tmp);
-            goto exit;
-        }
-    }
-
-    g_free(tmp);
-    ret = 0;
-exit:
-    bdrv_delete(qcow_bs);
-    return ret;
-}
-
-static int qcow_make_empty(BlockDriverState *bs)
-{
-    BDRVQcowState *s = bs->opaque;
-    uint32_t l1_length = s->l1_size * sizeof(uint64_t);
-    int ret;
-
-    memset(s->l1_table, 0, l1_length);
-    if (bdrv_pwrite_sync(bs->file, s->l1_table_offset, s->l1_table,
-            l1_length) < 0)
-        return -1;
-    ret = bdrv_truncate(bs->file, s->l1_table_offset + l1_length);
-    if (ret < 0)
-        return ret;
-
-    memset(s->l2_cache, 0, s->l2_size * L2_CACHE_SIZE * sizeof(uint64_t));
-    memset(s->l2_cache_offsets, 0, L2_CACHE_SIZE * sizeof(uint64_t));
-    memset(s->l2_cache_counts, 0, L2_CACHE_SIZE * sizeof(uint32_t));
-
-    return 0;
-}
-
-/* XXX: put compressed sectors first, then all the cluster aligned
-   tables to avoid losing bytes in alignment */
-static int qcow_write_compressed(BlockDriverState *bs, int64_t sector_num,
-                                 const uint8_t *buf, int nb_sectors)
-{
-    BDRVQcowState *s = bs->opaque;
-    z_stream strm;
-    int ret, out_len;
-    uint8_t *out_buf;
-    uint64_t cluster_offset;
-
-    if (nb_sectors != s->cluster_sectors) {
-        ret = -EINVAL;
-
-        /* Zero-pad last write if image size is not cluster aligned */
-        if (sector_num + nb_sectors == bs->total_sectors &&
-            nb_sectors < s->cluster_sectors) {
-            uint8_t *pad_buf = qemu_blockalign(bs, s->cluster_size);
-            memset(pad_buf, 0, s->cluster_size);
-            memcpy(pad_buf, buf, nb_sectors * BDRV_SECTOR_SIZE);
-            ret = qcow_write_compressed(bs, sector_num,
-                                        pad_buf, s->cluster_sectors);
-            qemu_vfree(pad_buf);
-        }
-        return ret;
-    }
-
-    out_buf = g_malloc(s->cluster_size + (s->cluster_size / 1000) + 128);
-
-    /* best compression, small window, no zlib header */
-    memset(&strm, 0, sizeof(strm));
-    ret = deflateInit2(&strm, Z_DEFAULT_COMPRESSION,
-                       Z_DEFLATED, -12,
-                       9, Z_DEFAULT_STRATEGY);
-    if (ret != 0) {
-        ret = -EINVAL;
-        goto fail;
-    }
-
-    strm.avail_in = s->cluster_size;
-    strm.next_in = (uint8_t *)buf;
-    strm.avail_out = s->cluster_size;
-    strm.next_out = out_buf;
-
-    ret = deflate(&strm, Z_FINISH);
-    if (ret != Z_STREAM_END && ret != Z_OK) {
-        deflateEnd(&strm);
-        ret = -EINVAL;
-        goto fail;
-    }
-    out_len = strm.next_out - out_buf;
-
-    deflateEnd(&strm);
-
-    if (ret != Z_STREAM_END || out_len >= s->cluster_size) {
-        /* could not compress: write normal cluster */
-        ret = bdrv_write(bs, sector_num, buf, s->cluster_sectors);
-        if (ret < 0) {
-            goto fail;
-        }
-    } else {
-        cluster_offset = get_cluster_offset(bs, sector_num << 9, 2,
-                                            out_len, 0, 0);
-        if (cluster_offset == 0) {
-            ret = -EIO;
-            goto fail;
-        }
-
-        cluster_offset &= s->cluster_offset_mask;
-        ret = bdrv_pwrite(bs->file, cluster_offset, out_buf, out_len);
-        if (ret < 0) {
-            goto fail;
-        }
-    }
-
-    ret = 0;
-fail:
-    g_free(out_buf);
-    return ret;
-}
-
-static int qcow_get_info(BlockDriverState *bs, BlockDriverInfo *bdi)
-{
-    BDRVQcowState *s = bs->opaque;
-    bdi->cluster_size = s->cluster_size;
-    return 0;
-}
-
-
-static QEMUOptionParameter qcow_create_options[] = {
-    {
-        .name = BLOCK_OPT_SIZE,
-        .type = OPT_SIZE,
-        .help = "Virtual disk size"
-    },
-    {
-        .name = BLOCK_OPT_BACKING_FILE,
-        .type = OPT_STRING,
-        .help = "File name of a base image"
-    },
-    {
-        .name = BLOCK_OPT_ENCRYPT,
-        .type = OPT_FLAG,
-        .help = "Encrypt the image"
-    },
-    { NULL }
-};
-
-static BlockDriver bdrv_qcow = {
-    .format_name	= "qcow",
-    .instance_size	= sizeof(BDRVQcowState),
-    .bdrv_probe		= qcow_probe,
-    .bdrv_open		= qcow_open,
-    .bdrv_close		= qcow_close,
-    .bdrv_reopen_prepare = qcow_reopen_prepare,
-    .bdrv_create	= qcow_create,
-    .bdrv_has_zero_init     = bdrv_has_zero_init_1,
-
-    .bdrv_co_readv          = qcow_co_readv,
-    .bdrv_co_writev         = qcow_co_writev,
-    .bdrv_co_is_allocated   = qcow_co_is_allocated,
-
-    .bdrv_set_key           = qcow_set_key,
-    .bdrv_make_empty        = qcow_make_empty,
-    .bdrv_write_compressed  = qcow_write_compressed,
-    .bdrv_get_info          = qcow_get_info,
-
-    .create_options = qcow_create_options,
-};
-
-static void bdrv_qcow_init(void)
-{
-    bdrv_register(&bdrv_qcow);
-}
-
-block_init(bdrv_qcow_init);
diff --git a/contrib/qemu/block/qcow2-cache.c b/contrib/qemu/block/qcow2-cache.c
deleted file mode 100644
index 2f3114ecc24..00000000000
--- a/contrib/qemu/block/qcow2-cache.c
+++ /dev/null
@@ -1,323 +0,0 @@
-/*
- * L2/refcount table cache for the QCOW2 format
- *
- * Copyright (c) 2010 Kevin Wolf <kwolf@redhat.com>
- *
- * Permission is hereby granted, free of charge, to any person obtaining a copy
- * of this software and associated documentation files (the "Software"), to deal
- * in the Software without restriction, including without limitation the rights
- * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
- * copies of the Software, and to permit persons to whom the Software is
- * furnished to do so, subject to the following conditions:
- *
- * The above copyright notice and this permission notice shall be included in
- * all copies or substantial portions of the Software.
- *
- * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
- * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
- * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
- * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
- * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
- * THE SOFTWARE.
- */
-
-#include "block/block_int.h"
-#include "qemu-common.h"
-#include "qcow2.h"
-#include "trace.h"
-
-typedef struct Qcow2CachedTable {
-    void*   table;
-    int64_t offset;
-    bool    dirty;
-    int     cache_hits;
-    int     ref;
-} Qcow2CachedTable;
-
-struct Qcow2Cache {
-    Qcow2CachedTable*       entries;
-    struct Qcow2Cache*      depends;
-    int                     size;
-    bool                    depends_on_flush;
-};
-
-Qcow2Cache *qcow2_cache_create(BlockDriverState *bs, int num_tables)
-{
-    BDRVQcowState *s = bs->opaque;
-    Qcow2Cache *c;
-    int i;
-
-    c = g_malloc0(sizeof(*c));
-    c->size = num_tables;
-    c->entries = g_malloc0(sizeof(*c->entries) * num_tables);
-
-    for (i = 0; i < c->size; i++) {
-        c->entries[i].table = qemu_blockalign(bs, s->cluster_size);
-    }
-
-    return c;
-}
-
-int qcow2_cache_destroy(BlockDriverState* bs, Qcow2Cache *c)
-{
-    int i;
-
-    for (i = 0; i < c->size; i++) {
-        assert(c->entries[i].ref == 0);
-        qemu_vfree(c->entries[i].table);
-    }
-
-    g_free(c->entries);
-    g_free(c);
-
-    return 0;
-}
-
-static int qcow2_cache_flush_dependency(BlockDriverState *bs, Qcow2Cache *c)
-{
-    int ret;
-
-    ret = qcow2_cache_flush(bs, c->depends);
-    if (ret < 0) {
-        return ret;
-    }
-
-    c->depends = NULL;
-    c->depends_on_flush = false;
-
-    return 0;
-}
-
-static int qcow2_cache_entry_flush(BlockDriverState *bs, Qcow2Cache *c, int i)
-{
-    BDRVQcowState *s = bs->opaque;
-    int ret = 0;
-
-    if (!c->entries[i].dirty || !c->entries[i].offset) {
-        return 0;
-    }
-
-    trace_qcow2_cache_entry_flush(qemu_coroutine_self(),
-                                  c == s->l2_table_cache, i);
-
-    if (c->depends) {
-        ret = qcow2_cache_flush_dependency(bs, c);
-    } else if (c->depends_on_flush) {
-        ret = bdrv_flush(bs->file);
-        if (ret >= 0) {
-            c->depends_on_flush = false;
-        }
-    }
-
-    if (ret < 0) {
-        return ret;
-    }
-
-    if (c == s->refcount_block_cache) {
-        BLKDBG_EVENT(bs->file, BLKDBG_REFBLOCK_UPDATE_PART);
-    } else if (c == s->l2_table_cache) {
-        BLKDBG_EVENT(bs->file, BLKDBG_L2_UPDATE);
-    }
-
-    ret = bdrv_pwrite(bs->file, c->entries[i].offset, c->entries[i].table,
-        s->cluster_size);
-    if (ret < 0) {
-        return ret;
-    }
-
-    c->entries[i].dirty = false;
-
-    return 0;
-}
-
-int qcow2_cache_flush(BlockDriverState *bs, Qcow2Cache *c)
-{
-    BDRVQcowState *s = bs->opaque;
-    int result = 0;
-    int ret;
-    int i;
-
-    trace_qcow2_cache_flush(qemu_coroutine_self(), c == s->l2_table_cache);
-
-    for (i = 0; i < c->size; i++) {
-        ret = qcow2_cache_entry_flush(bs, c, i);
-        if (ret < 0 && result != -ENOSPC) {
-            result = ret;
-        }
-    }
-
-    if (result == 0) {
-        ret = bdrv_flush(bs->file);
-        if (ret < 0) {
-            result = ret;
-        }
-    }
-
-    return result;
-}
-
-int qcow2_cache_set_dependency(BlockDriverState *bs, Qcow2Cache *c,
-    Qcow2Cache *dependency)
-{
-    int ret;
-
-    if (dependency->depends) {
-        ret = qcow2_cache_flush_dependency(bs, dependency);
-        if (ret < 0) {
-            return ret;
-        }
-    }
-
-    if (c->depends && (c->depends != dependency)) {
-        ret = qcow2_cache_flush_dependency(bs, c);
-        if (ret < 0) {
-            return ret;
-        }
-    }
-
-    c->depends = dependency;
-    return 0;
-}
-
-void qcow2_cache_depends_on_flush(Qcow2Cache *c)
-{
-    c->depends_on_flush = true;
-}
-
-static int qcow2_cache_find_entry_to_replace(Qcow2Cache *c)
-{
-    int i;
-    int min_count = INT_MAX;
-    int min_index = -1;
-
-
-    for (i = 0; i < c->size; i++) {
-        if (c->entries[i].ref) {
-            continue;
-        }
-
-        if (c->entries[i].cache_hits < min_count) {
-            min_index = i;
-            min_count = c->entries[i].cache_hits;
-        }
-
-        /* Give newer hits priority */
-        /* TODO Check how to optimize the replacement strategy */
-        c->entries[i].cache_hits /= 2;
-    }
-
-    if (min_index == -1) {
-        /* This can't happen in current synchronous code, but leave the check
-         * here as a reminder for whoever starts using AIO with the cache */
-        abort();
-    }
-    return min_index;
-}
-
-static int qcow2_cache_do_get(BlockDriverState *bs, Qcow2Cache *c,
-    uint64_t offset, void **table, bool read_from_disk)
-{
-    BDRVQcowState *s = bs->opaque;
-    int i;
-    int ret;
-
-    trace_qcow2_cache_get(qemu_coroutine_self(), c == s->l2_table_cache,
-                          offset, read_from_disk);
-
-    /* Check if the table is already cached */
-    for (i = 0; i < c->size; i++) {
-        if (c->entries[i].offset == offset) {
-            goto found;
-        }
-    }
-
-    /* If not, write a table back and replace it */
-    i = qcow2_cache_find_entry_to_replace(c);
-    trace_qcow2_cache_get_replace_entry(qemu_coroutine_self(),
-                                        c == s->l2_table_cache, i);
-    if (i < 0) {
-        return i;
-    }
-
-    ret = qcow2_cache_entry_flush(bs, c, i);
-    if (ret < 0) {
-        return ret;
-    }
-
-    trace_qcow2_cache_get_read(qemu_coroutine_self(),
-                               c == s->l2_table_cache, i);
-    c->entries[i].offset = 0;
-    if (read_from_disk) {
-        if (c == s->l2_table_cache) {
-            BLKDBG_EVENT(bs->file, BLKDBG_L2_LOAD);
-        }
-
-        ret = bdrv_pread(bs->file, offset, c->entries[i].table, s->cluster_size);
-        if (ret < 0) {
-            return ret;
-        }
-    }
-
-    /* Give the table some hits for the start so that it won't be replaced
-     * immediately. The number 32 is completely arbitrary. */
-    c->entries[i].cache_hits = 32;
-    c->entries[i].offset = offset;
-
-    /* And return the right table */
-found:
-    c->entries[i].cache_hits++;
-    c->entries[i].ref++;
-    *table = c->entries[i].table;
-
-    trace_qcow2_cache_get_done(qemu_coroutine_self(),
-                               c == s->l2_table_cache, i);
-
-    return 0;
-}
-
-int qcow2_cache_get(BlockDriverState *bs, Qcow2Cache *c, uint64_t offset,
-    void **table)
-{
-    return qcow2_cache_do_get(bs, c, offset, table, true);
-}
-
-int qcow2_cache_get_empty(BlockDriverState *bs, Qcow2Cache *c, uint64_t offset,
-    void **table)
-{
-    return qcow2_cache_do_get(bs, c, offset, table, false);
-}
-
-int qcow2_cache_put(BlockDriverState *bs, Qcow2Cache *c, void **table)
-{
-    int i;
-
-    for (i = 0; i < c->size; i++) {
-        if (c->entries[i].table == *table) {
-            goto found;
-        }
-    }
-    return -ENOENT;
-
-found:
-    c->entries[i].ref--;
-    *table = NULL;
-
-    assert(c->entries[i].ref >= 0);
-    return 0;
-}
-
-void qcow2_cache_entry_mark_dirty(Qcow2Cache *c, void *table)
-{
-    int i;
-
-    for (i = 0; i < c->size; i++) {
-        if (c->entries[i].table == table) {
-            goto found;
-        }
-    }
-    abort();
-
-found:
-    c->entries[i].dirty = true;
-}
diff --git a/contrib/qemu/block/qcow2-cluster.c b/contrib/qemu/block/qcow2-cluster.c
deleted file mode 100644
index cca76d4fcdd..00000000000
--- a/contrib/qemu/block/qcow2-cluster.c
+++ /dev/null
@@ -1,1478 +0,0 @@
-/*
- * Block driver for the QCOW version 2 format
- *
- * Copyright (c) 2004-2006 Fabrice Bellard
- *
- * Permission is hereby granted, free of charge, to any person obtaining a copy
- * of this software and associated documentation files (the "Software"), to deal
- * in the Software without restriction, including without limitation the rights
- * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
- * copies of the Software, and to permit persons to whom the Software is
- * furnished to do so, subject to the following conditions:
- *
- * The above copyright notice and this permission notice shall be included in
- * all copies or substantial portions of the Software.
- *
- * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
- * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
- * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
- * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
- * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
- * THE SOFTWARE.
- */
-
-#include <zlib.h>
-
-#include "qemu-common.h"
-#include "block/block_int.h"
-#include "block/qcow2.h"
-#include "trace.h"
-
-int qcow2_grow_l1_table(BlockDriverState *bs, uint64_t min_size,
-                        bool exact_size)
-{
-    BDRVQcowState *s = bs->opaque;
-    int new_l1_size2, ret, i;
-    uint64_t *new_l1_table;
-    int64_t new_l1_table_offset, new_l1_size;
-    uint8_t data[12];
-
-    if (min_size <= s->l1_size)
-        return 0;
-
-    if (exact_size) {
-        new_l1_size = min_size;
-    } else {
-        /* Bump size up to reduce the number of times we have to grow */
-        new_l1_size = s->l1_size;
-        if (new_l1_size == 0) {
-            new_l1_size = 1;
-        }
-        while (min_size > new_l1_size) {
-            new_l1_size = (new_l1_size * 3 + 1) / 2;
-        }
-    }
-
-    if (new_l1_size > INT_MAX) {
-        return -EFBIG;
-    }
-
-#ifdef DEBUG_ALLOC2
-    fprintf(stderr, "grow l1_table from %d to %" PRId64 "\n",
-            s->l1_size, new_l1_size);
-#endif
-
-    new_l1_size2 = sizeof(uint64_t) * new_l1_size;
-    new_l1_table = g_malloc0(align_offset(new_l1_size2, 512));
-    memcpy(new_l1_table, s->l1_table, s->l1_size * sizeof(uint64_t));
-
-    /* write new table (align to cluster) */
-    BLKDBG_EVENT(bs->file, BLKDBG_L1_GROW_ALLOC_TABLE);
-    new_l1_table_offset = qcow2_alloc_clusters(bs, new_l1_size2);
-    if (new_l1_table_offset < 0) {
-        g_free(new_l1_table);
-        return new_l1_table_offset;
-    }
-
-    ret = qcow2_cache_flush(bs, s->refcount_block_cache);
-    if (ret < 0) {
-        goto fail;
-    }
-
-    BLKDBG_EVENT(bs->file, BLKDBG_L1_GROW_WRITE_TABLE);
-    for(i = 0; i < s->l1_size; i++)
-        new_l1_table[i] = cpu_to_be64(new_l1_table[i]);
-    ret = bdrv_pwrite_sync(bs->file, new_l1_table_offset, new_l1_table, new_l1_size2);
-    if (ret < 0)
-        goto fail;
-    for(i = 0; i < s->l1_size; i++)
-        new_l1_table[i] = be64_to_cpu(new_l1_table[i]);
-
-    /* set new table */
-    BLKDBG_EVENT(bs->file, BLKDBG_L1_GROW_ACTIVATE_TABLE);
-    cpu_to_be32w((uint32_t*)data, new_l1_size);
-    cpu_to_be64wu((uint64_t*)(data + 4), new_l1_table_offset);
-    ret = bdrv_pwrite_sync(bs->file, offsetof(QCowHeader, l1_size), data,sizeof(data));
-    if (ret < 0) {
-        goto fail;
-    }
-    g_free(s->l1_table);
-    qcow2_free_clusters(bs, s->l1_table_offset, s->l1_size * sizeof(uint64_t),
-                        QCOW2_DISCARD_OTHER);
-    s->l1_table_offset = new_l1_table_offset;
-    s->l1_table = new_l1_table;
-    s->l1_size = new_l1_size;
-    return 0;
- fail:
-    g_free(new_l1_table);
-    qcow2_free_clusters(bs, new_l1_table_offset, new_l1_size2,
-                        QCOW2_DISCARD_OTHER);
-    return ret;
-}
-
-/*
- * l2_load
- *
- * Loads a L2 table into memory. If the table is in the cache, the cache
- * is used; otherwise the L2 table is loaded from the image file.
- *
- * Returns a pointer to the L2 table on success, or NULL if the read from
- * the image file failed.
- */
-
-static int l2_load(BlockDriverState *bs, uint64_t l2_offset,
-    uint64_t **l2_table)
-{
-    BDRVQcowState *s = bs->opaque;
-    int ret;
-
-    ret = qcow2_cache_get(bs, s->l2_table_cache, l2_offset, (void**) l2_table);
-
-    return ret;
-}
-
-/*
- * Writes one sector of the L1 table to the disk (can't update single entries
- * and we really don't want bdrv_pread to perform a read-modify-write)
- */
-#define L1_ENTRIES_PER_SECTOR (512 / 8)
-static int write_l1_entry(BlockDriverState *bs, int l1_index)
-{
-    BDRVQcowState *s = bs->opaque;
-    uint64_t buf[L1_ENTRIES_PER_SECTOR];
-    int l1_start_index;
-    int i, ret;
-
-    l1_start_index = l1_index & ~(L1_ENTRIES_PER_SECTOR - 1);
-    for (i = 0; i < L1_ENTRIES_PER_SECTOR; i++) {
-        buf[i] = cpu_to_be64(s->l1_table[l1_start_index + i]);
-    }
-
-    BLKDBG_EVENT(bs->file, BLKDBG_L1_UPDATE);
-    ret = bdrv_pwrite_sync(bs->file, s->l1_table_offset + 8 * l1_start_index,
-        buf, sizeof(buf));
-    if (ret < 0) {
-        return ret;
-    }
-
-    return 0;
-}
-
-/*
- * l2_allocate
- *
- * Allocate a new l2 entry in the file. If l1_index points to an already
- * used entry in the L2 table (i.e. we are doing a copy on write for the L2
- * table) copy the contents of the old L2 table into the newly allocated one.
- * Otherwise the new table is initialized with zeros.
- *
- */
-
-static int l2_allocate(BlockDriverState *bs, int l1_index, uint64_t **table)
-{
-    BDRVQcowState *s = bs->opaque;
-    uint64_t old_l2_offset;
-    uint64_t *l2_table;
-    int64_t l2_offset;
-    int ret;
-
-    old_l2_offset = s->l1_table[l1_index];
-
-    trace_qcow2_l2_allocate(bs, l1_index);
-
-    /* allocate a new l2 entry */
-
-    l2_offset = qcow2_alloc_clusters(bs, s->l2_size * sizeof(uint64_t));
-    if (l2_offset < 0) {
-        return l2_offset;
-    }
-
-    ret = qcow2_cache_flush(bs, s->refcount_block_cache);
-    if (ret < 0) {
-        goto fail;
-    }
-
-    /* allocate a new entry in the l2 cache */
-
-    trace_qcow2_l2_allocate_get_empty(bs, l1_index);
-    ret = qcow2_cache_get_empty(bs, s->l2_table_cache, l2_offset, (void**) table);
-    if (ret < 0) {
-        return ret;
-    }
-
-    l2_table = *table;
-
-    if ((old_l2_offset & L1E_OFFSET_MASK) == 0) {
-        /* if there was no old l2 table, clear the new table */
-        memset(l2_table, 0, s->l2_size * sizeof(uint64_t));
-    } else {
-        uint64_t* old_table;
-
-        /* if there was an old l2 table, read it from the disk */
-        BLKDBG_EVENT(bs->file, BLKDBG_L2_ALLOC_COW_READ);
-        ret = qcow2_cache_get(bs, s->l2_table_cache,
-            old_l2_offset & L1E_OFFSET_MASK,
-            (void**) &old_table);
-        if (ret < 0) {
-            goto fail;
-        }
-
-        memcpy(l2_table, old_table, s->cluster_size);
-
-        ret = qcow2_cache_put(bs, s->l2_table_cache, (void**) &old_table);
-        if (ret < 0) {
-            goto fail;
-        }
-    }
-
-    /* write the l2 table to the file */
-    BLKDBG_EVENT(bs->file, BLKDBG_L2_ALLOC_WRITE);
-
-    trace_qcow2_l2_allocate_write_l2(bs, l1_index);
-    qcow2_cache_entry_mark_dirty(s->l2_table_cache, l2_table);
-    ret = qcow2_cache_flush(bs, s->l2_table_cache);
-    if (ret < 0) {
-        goto fail;
-    }
-
-    /* update the L1 entry */
-    trace_qcow2_l2_allocate_write_l1(bs, l1_index);
-    s->l1_table[l1_index] = l2_offset | QCOW_OFLAG_COPIED;
-    ret = write_l1_entry(bs, l1_index);
-    if (ret < 0) {
-        goto fail;
-    }
-
-    *table = l2_table;
-    trace_qcow2_l2_allocate_done(bs, l1_index, 0);
-    return 0;
-
-fail:
-    trace_qcow2_l2_allocate_done(bs, l1_index, ret);
-    qcow2_cache_put(bs, s->l2_table_cache, (void**) table);
-    s->l1_table[l1_index] = old_l2_offset;
-    return ret;
-}
-
-/*
- * Checks how many clusters in a given L2 table are contiguous in the image
- * file. As soon as one of the flags in the bitmask stop_flags changes compared
- * to the first cluster, the search is stopped and the cluster is not counted
- * as contiguous. (This allows it, for example, to stop at the first compressed
- * cluster which may require a different handling)
- */
-static int count_contiguous_clusters(uint64_t nb_clusters, int cluster_size,
-        uint64_t *l2_table, uint64_t start, uint64_t stop_flags)
-{
-    int i;
-    uint64_t mask = stop_flags | L2E_OFFSET_MASK;
-    uint64_t offset = be64_to_cpu(l2_table[0]) & mask;
-
-    if (!offset)
-        return 0;
-
-    for (i = start; i < start + nb_clusters; i++) {
-        uint64_t l2_entry = be64_to_cpu(l2_table[i]) & mask;
-        if (offset + (uint64_t) i * cluster_size != l2_entry) {
-            break;
-        }
-    }
-
-	return (i - start);
-}
-
-static int count_contiguous_free_clusters(uint64_t nb_clusters, uint64_t *l2_table)
-{
-    int i;
-
-    for (i = 0; i < nb_clusters; i++) {
-        int type = qcow2_get_cluster_type(be64_to_cpu(l2_table[i]));
-
-        if (type != QCOW2_CLUSTER_UNALLOCATED) {
-            break;
-        }
-    }
-
-    return i;
-}
-
-/* The crypt function is compatible with the linux cryptoloop
-   algorithm for < 4 GB images. NOTE: out_buf == in_buf is
-   supported */
-void qcow2_encrypt_sectors(BDRVQcowState *s, int64_t sector_num,
-                           uint8_t *out_buf, const uint8_t *in_buf,
-                           int nb_sectors, int enc,
-                           const AES_KEY *key)
-{
-    union {
-        uint64_t ll[2];
-        uint8_t b[16];
-    } ivec;
-    int i;
-
-    for(i = 0; i < nb_sectors; i++) {
-        ivec.ll[0] = cpu_to_le64(sector_num);
-        ivec.ll[1] = 0;
-        AES_cbc_encrypt(in_buf, out_buf, 512, key,
-                        ivec.b, enc);
-        sector_num++;
-        in_buf += 512;
-        out_buf += 512;
-    }
-}
-
-static int coroutine_fn copy_sectors(BlockDriverState *bs,
-                                     uint64_t start_sect,
-                                     uint64_t cluster_offset,
-                                     int n_start, int n_end)
-{
-    BDRVQcowState *s = bs->opaque;
-    QEMUIOVector qiov;
-    struct iovec iov;
-    int n, ret;
-
-    /*
-     * If this is the last cluster and it is only partially used, we must only
-     * copy until the end of the image, or bdrv_check_request will fail for the
-     * bdrv_read/write calls below.
-     */
-    if (start_sect + n_end > bs->total_sectors) {
-        n_end = bs->total_sectors - start_sect;
-    }
-
-    n = n_end - n_start;
-    if (n <= 0) {
-        return 0;
-    }
-
-    iov.iov_len = n * BDRV_SECTOR_SIZE;
-    iov.iov_base = qemu_blockalign(bs, iov.iov_len);
-
-    qemu_iovec_init_external(&qiov, &iov, 1);
-
-    BLKDBG_EVENT(bs->file, BLKDBG_COW_READ);
-
-    /* Call .bdrv_co_readv() directly instead of using the public block-layer
-     * interface.  This avoids double I/O throttling and request tracking,
-     * which can lead to deadlock when block layer copy-on-read is enabled.
-     */
-    ret = bs->drv->bdrv_co_readv(bs, start_sect + n_start, n, &qiov);
-    if (ret < 0) {
-        goto out;
-    }
-
-    if (s->crypt_method) {
-        qcow2_encrypt_sectors(s, start_sect + n_start,
-                        iov.iov_base, iov.iov_base, n, 1,
-                        &s->aes_encrypt_key);
-    }
-
-    BLKDBG_EVENT(bs->file, BLKDBG_COW_WRITE);
-    ret = bdrv_co_writev(bs->file, (cluster_offset >> 9) + n_start, n, &qiov);
-    if (ret < 0) {
-        goto out;
-    }
-
-    ret = 0;
-out:
-    qemu_vfree(iov.iov_base);
-    return ret;
-}
-
-
-/*
- * get_cluster_offset
- *
- * For a given offset of the disk image, find the cluster offset in
- * qcow2 file. The offset is stored in *cluster_offset.
- *
- * on entry, *num is the number of contiguous sectors we'd like to
- * access following offset.
- *
- * on exit, *num is the number of contiguous sectors we can read.
- *
- * Returns the cluster type (QCOW2_CLUSTER_*) on success, -errno in error
- * cases.
- */
-int qcow2_get_cluster_offset(BlockDriverState *bs, uint64_t offset,
-    int *num, uint64_t *cluster_offset)
-{
-    BDRVQcowState *s = bs->opaque;
-    unsigned int l2_index;
-    uint64_t l1_index, l2_offset, *l2_table;
-    int l1_bits, c;
-    unsigned int index_in_cluster, nb_clusters;
-    uint64_t nb_available, nb_needed;
-    int ret;
-
-    index_in_cluster = (offset >> 9) & (s->cluster_sectors - 1);
-    nb_needed = *num + index_in_cluster;
-
-    l1_bits = s->l2_bits + s->cluster_bits;
-
-    /* compute how many bytes there are between the offset and
-     * the end of the l1 entry
-     */
-
-    nb_available = (1ULL << l1_bits) - (offset & ((1ULL << l1_bits) - 1));
-
-    /* compute the number of available sectors */
-
-    nb_available = (nb_available >> 9) + index_in_cluster;
-
-    if (nb_needed > nb_available) {
-        nb_needed = nb_available;
-    }
-
-    *cluster_offset = 0;
-
-    /* seek the the l2 offset in the l1 table */
-
-    l1_index = offset >> l1_bits;
-    if (l1_index >= s->l1_size) {
-        ret = QCOW2_CLUSTER_UNALLOCATED;
-        goto out;
-    }
-
-    l2_offset = s->l1_table[l1_index] & L1E_OFFSET_MASK;
-    if (!l2_offset) {
-        ret = QCOW2_CLUSTER_UNALLOCATED;
-        goto out;
-    }
-
-    /* load the l2 table in memory */
-
-    ret = l2_load(bs, l2_offset, &l2_table);
-    if (ret < 0) {
-        return ret;
-    }
-
-    /* find the cluster offset for the given disk offset */
-
-    l2_index = (offset >> s->cluster_bits) & (s->l2_size - 1);
-    *cluster_offset = be64_to_cpu(l2_table[l2_index]);
-    nb_clusters = size_to_clusters(s, nb_needed << 9);
-
-    ret = qcow2_get_cluster_type(*cluster_offset);
-    switch (ret) {
-    case QCOW2_CLUSTER_COMPRESSED:
-        /* Compressed clusters can only be processed one by one */
-        c = 1;
-        *cluster_offset &= L2E_COMPRESSED_OFFSET_SIZE_MASK;
-        break;
-    case QCOW2_CLUSTER_ZERO:
-        if (s->qcow_version < 3) {
-            return -EIO;
-        }
-        c = count_contiguous_clusters(nb_clusters, s->cluster_size,
-                &l2_table[l2_index], 0,
-                QCOW_OFLAG_COMPRESSED | QCOW_OFLAG_ZERO);
-        *cluster_offset = 0;
-        break;
-    case QCOW2_CLUSTER_UNALLOCATED:
-        /* how many empty clusters ? */
-        c = count_contiguous_free_clusters(nb_clusters, &l2_table[l2_index]);
-        *cluster_offset = 0;
-        break;
-    case QCOW2_CLUSTER_NORMAL:
-        /* how many allocated clusters ? */
-        c = count_contiguous_clusters(nb_clusters, s->cluster_size,
-                &l2_table[l2_index], 0,
-                QCOW_OFLAG_COMPRESSED | QCOW_OFLAG_ZERO);
-        *cluster_offset &= L2E_OFFSET_MASK;
-        break;
-    default:
-        abort();
-    }
-
-    qcow2_cache_put(bs, s->l2_table_cache, (void**) &l2_table);
-
-    nb_available = (c * s->cluster_sectors);
-
-out:
-    if (nb_available > nb_needed)
-        nb_available = nb_needed;
-
-    *num = nb_available - index_in_cluster;
-
-    return ret;
-}
-
-/*
- * get_cluster_table
- *
- * for a given disk offset, load (and allocate if needed)
- * the l2 table.
- *
- * the l2 table offset in the qcow2 file and the cluster index
- * in the l2 table are given to the caller.
- *
- * Returns 0 on success, -errno in failure case
- */
-static int get_cluster_table(BlockDriverState *bs, uint64_t offset,
-                             uint64_t **new_l2_table,
-                             int *new_l2_index)
-{
-    BDRVQcowState *s = bs->opaque;
-    unsigned int l2_index;
-    uint64_t l1_index, l2_offset;
-    uint64_t *l2_table = NULL;
-    int ret;
-
-    /* seek the the l2 offset in the l1 table */
-
-    l1_index = offset >> (s->l2_bits + s->cluster_bits);
-    if (l1_index >= s->l1_size) {
-        ret = qcow2_grow_l1_table(bs, l1_index + 1, false);
-        if (ret < 0) {
-            return ret;
-        }
-    }
-
-    assert(l1_index < s->l1_size);
-    l2_offset = s->l1_table[l1_index] & L1E_OFFSET_MASK;
-
-    /* seek the l2 table of the given l2 offset */
-
-    if (s->l1_table[l1_index] & QCOW_OFLAG_COPIED) {
-        /* load the l2 table in memory */
-        ret = l2_load(bs, l2_offset, &l2_table);
-        if (ret < 0) {
-            return ret;
-        }
-    } else {
-        /* First allocate a new L2 table (and do COW if needed) */
-        ret = l2_allocate(bs, l1_index, &l2_table);
-        if (ret < 0) {
-            return ret;
-        }
-
-        /* Then decrease the refcount of the old table */
-        if (l2_offset) {
-            qcow2_free_clusters(bs, l2_offset, s->l2_size * sizeof(uint64_t),
-                                QCOW2_DISCARD_OTHER);
-        }
-    }
-
-    /* find the cluster offset for the given disk offset */
-
-    l2_index = (offset >> s->cluster_bits) & (s->l2_size - 1);
-
-    *new_l2_table = l2_table;
-    *new_l2_index = l2_index;
-
-    return 0;
-}
-
-/*
- * alloc_compressed_cluster_offset
- *
- * For a given offset of the disk image, return cluster offset in
- * qcow2 file.
- *
- * If the offset is not found, allocate a new compressed cluster.
- *
- * Return the cluster offset if successful,
- * Return 0, otherwise.
- *
- */
-
-uint64_t qcow2_alloc_compressed_cluster_offset(BlockDriverState *bs,
-                                               uint64_t offset,
-                                               int compressed_size)
-{
-    BDRVQcowState *s = bs->opaque;
-    int l2_index, ret;
-    uint64_t *l2_table;
-    int64_t cluster_offset;
-    int nb_csectors;
-
-    ret = get_cluster_table(bs, offset, &l2_table, &l2_index);
-    if (ret < 0) {
-        return 0;
-    }
-
-    /* Compression can't overwrite anything. Fail if the cluster was already
-     * allocated. */
-    cluster_offset = be64_to_cpu(l2_table[l2_index]);
-    if (cluster_offset & L2E_OFFSET_MASK) {
-        qcow2_cache_put(bs, s->l2_table_cache, (void**) &l2_table);
-        return 0;
-    }
-
-    cluster_offset = qcow2_alloc_bytes(bs, compressed_size);
-    if (cluster_offset < 0) {
-        qcow2_cache_put(bs, s->l2_table_cache, (void**) &l2_table);
-        return 0;
-    }
-
-    nb_csectors = ((cluster_offset + compressed_size - 1) >> 9) -
-                  (cluster_offset >> 9);
-
-    cluster_offset |= QCOW_OFLAG_COMPRESSED |
-                      ((uint64_t)nb_csectors << s->csize_shift);
-
-    /* update L2 table */
-
-    /* compressed clusters never have the copied flag */
-
-    BLKDBG_EVENT(bs->file, BLKDBG_L2_UPDATE_COMPRESSED);
-    qcow2_cache_entry_mark_dirty(s->l2_table_cache, l2_table);
-    l2_table[l2_index] = cpu_to_be64(cluster_offset);
-    ret = qcow2_cache_put(bs, s->l2_table_cache, (void**) &l2_table);
-    if (ret < 0) {
-        return 0;
-    }
-
-    return cluster_offset;
-}
-
-static int perform_cow(BlockDriverState *bs, QCowL2Meta *m, Qcow2COWRegion *r)
-{
-    BDRVQcowState *s = bs->opaque;
-    int ret;
-
-    if (r->nb_sectors == 0) {
-        return 0;
-    }
-
-    qemu_co_mutex_unlock(&s->lock);
-    ret = copy_sectors(bs, m->offset / BDRV_SECTOR_SIZE, m->alloc_offset,
-                       r->offset / BDRV_SECTOR_SIZE,
-                       r->offset / BDRV_SECTOR_SIZE + r->nb_sectors);
-    qemu_co_mutex_lock(&s->lock);
-
-    if (ret < 0) {
-        return ret;
-    }
-
-    /*
-     * Before we update the L2 table to actually point to the new cluster, we
-     * need to be sure that the refcounts have been increased and COW was
-     * handled.
-     */
-    qcow2_cache_depends_on_flush(s->l2_table_cache);
-
-    return 0;
-}
-
-int qcow2_alloc_cluster_link_l2(BlockDriverState *bs, QCowL2Meta *m)
-{
-    BDRVQcowState *s = bs->opaque;
-    int i, j = 0, l2_index, ret;
-    uint64_t *old_cluster, *l2_table;
-    uint64_t cluster_offset = m->alloc_offset;
-
-    trace_qcow2_cluster_link_l2(qemu_coroutine_self(), m->nb_clusters);
-    assert(m->nb_clusters > 0);
-
-    old_cluster = g_malloc(m->nb_clusters * sizeof(uint64_t));
-
-    /* copy content of unmodified sectors */
-    ret = perform_cow(bs, m, &m->cow_start);
-    if (ret < 0) {
-        goto err;
-    }
-
-    ret = perform_cow(bs, m, &m->cow_end);
-    if (ret < 0) {
-        goto err;
-    }
-
-    /* Update L2 table. */
-    if (s->use_lazy_refcounts) {
-        qcow2_mark_dirty(bs);
-    }
-    if (qcow2_need_accurate_refcounts(s)) {
-        qcow2_cache_set_dependency(bs, s->l2_table_cache,
-                                   s->refcount_block_cache);
-    }
-
-    ret = get_cluster_table(bs, m->offset, &l2_table, &l2_index);
-    if (ret < 0) {
-        goto err;
-    }
-    qcow2_cache_entry_mark_dirty(s->l2_table_cache, l2_table);
-
-    for (i = 0; i < m->nb_clusters; i++) {
-        /* if two concurrent writes happen to the same unallocated cluster
-	 * each write allocates separate cluster and writes data concurrently.
-	 * The first one to complete updates l2 table with pointer to its
-	 * cluster the second one has to do RMW (which is done above by
-	 * copy_sectors()), update l2 table with its cluster pointer and free
-	 * old cluster. This is what this loop does */
-        if(l2_table[l2_index + i] != 0)
-            old_cluster[j++] = l2_table[l2_index + i];
-
-        l2_table[l2_index + i] = cpu_to_be64((cluster_offset +
-                    (i << s->cluster_bits)) | QCOW_OFLAG_COPIED);
-     }
-
-
-    ret = qcow2_cache_put(bs, s->l2_table_cache, (void**) &l2_table);
-    if (ret < 0) {
-        goto err;
-    }
-
-    /*
-     * If this was a COW, we need to decrease the refcount of the old cluster.
-     * Also flush bs->file to get the right order for L2 and refcount update.
-     *
-     * Don't discard clusters that reach a refcount of 0 (e.g. compressed
-     * clusters), the next write will reuse them anyway.
-     */
-    if (j != 0) {
-        for (i = 0; i < j; i++) {
-            qcow2_free_any_clusters(bs, be64_to_cpu(old_cluster[i]), 1,
-                                    QCOW2_DISCARD_NEVER);
-        }
-    }
-
-    ret = 0;
-err:
-    g_free(old_cluster);
-    return ret;
- }
-
-/*
- * Returns the number of contiguous clusters that can be used for an allocating
- * write, but require COW to be performed (this includes yet unallocated space,
- * which must copy from the backing file)
- */
-static int count_cow_clusters(BDRVQcowState *s, int nb_clusters,
-    uint64_t *l2_table, int l2_index)
-{
-    int i;
-
-    for (i = 0; i < nb_clusters; i++) {
-        uint64_t l2_entry = be64_to_cpu(l2_table[l2_index + i]);
-        int cluster_type = qcow2_get_cluster_type(l2_entry);
-
-        switch(cluster_type) {
-        case QCOW2_CLUSTER_NORMAL:
-            if (l2_entry & QCOW_OFLAG_COPIED) {
-                goto out;
-            }
-            break;
-        case QCOW2_CLUSTER_UNALLOCATED:
-        case QCOW2_CLUSTER_COMPRESSED:
-        case QCOW2_CLUSTER_ZERO:
-            break;
-        default:
-            abort();
-        }
-    }
-
-out:
-    assert(i <= nb_clusters);
-    return i;
-}
-
-/*
- * Check if there already is an AIO write request in flight which allocates
- * the same cluster. In this case we need to wait until the previous
- * request has completed and updated the L2 table accordingly.
- *
- * Returns:
- *   0       if there was no dependency. *cur_bytes indicates the number of
- *           bytes from guest_offset that can be read before the next
- *           dependency must be processed (or the request is complete)
- *
- *   -EAGAIN if we had to wait for another request, previously gathered
- *           information on cluster allocation may be invalid now. The caller
- *           must start over anyway, so consider *cur_bytes undefined.
- */
-static int handle_dependencies(BlockDriverState *bs, uint64_t guest_offset,
-    uint64_t *cur_bytes, QCowL2Meta **m)
-{
-    BDRVQcowState *s = bs->opaque;
-    QCowL2Meta *old_alloc;
-    uint64_t bytes = *cur_bytes;
-
-    QLIST_FOREACH(old_alloc, &s->cluster_allocs, next_in_flight) {
-
-        uint64_t start = guest_offset;
-        uint64_t end = start + bytes;
-        uint64_t old_start = l2meta_cow_start(old_alloc);
-        uint64_t old_end = l2meta_cow_end(old_alloc);
-
-        if (end <= old_start || start >= old_end) {
-            /* No intersection */
-        } else {
-            if (start < old_start) {
-                /* Stop at the start of a running allocation */
-                bytes = old_start - start;
-            } else {
-                bytes = 0;
-            }
-
-            /* Stop if already an l2meta exists. After yielding, it wouldn't
-             * be valid any more, so we'd have to clean up the old L2Metas
-             * and deal with requests depending on them before starting to
-             * gather new ones. Not worth the trouble. */
-            if (bytes == 0 && *m) {
-                *cur_bytes = 0;
-                return 0;
-            }
-
-            if (bytes == 0) {
-                /* Wait for the dependency to complete. We need to recheck
-                 * the free/allocated clusters when we continue. */
-                qemu_co_mutex_unlock(&s->lock);
-                qemu_co_queue_wait(&old_alloc->dependent_requests);
-                qemu_co_mutex_lock(&s->lock);
-                return -EAGAIN;
-            }
-        }
-    }
-
-    /* Make sure that existing clusters and new allocations are only used up to
-     * the next dependency if we shortened the request above */
-    *cur_bytes = bytes;
-
-    return 0;
-}
-
-/*
- * Checks how many already allocated clusters that don't require a copy on
- * write there are at the given guest_offset (up to *bytes). If
- * *host_offset is not zero, only physically contiguous clusters beginning at
- * this host offset are counted.
- *
- * Note that guest_offset may not be cluster aligned. In this case, the
- * returned *host_offset points to exact byte referenced by guest_offset and
- * therefore isn't cluster aligned as well.
- *
- * Returns:
- *   0:     if no allocated clusters are available at the given offset.
- *          *bytes is normally unchanged. It is set to 0 if the cluster
- *          is allocated and doesn't need COW, but doesn't have the right
- *          physical offset.
- *
- *   1:     if allocated clusters that don't require a COW are available at
- *          the requested offset. *bytes may have decreased and describes
- *          the length of the area that can be written to.
- *
- *  -errno: in error cases
- */
-static int handle_copied(BlockDriverState *bs, uint64_t guest_offset,
-    uint64_t *host_offset, uint64_t *bytes, QCowL2Meta **m)
-{
-    BDRVQcowState *s = bs->opaque;
-    int l2_index;
-    uint64_t cluster_offset;
-    uint64_t *l2_table;
-    unsigned int nb_clusters;
-    unsigned int keep_clusters;
-    int ret, pret;
-
-    trace_qcow2_handle_copied(qemu_coroutine_self(), guest_offset, *host_offset,
-                              *bytes);
-
-    assert(*host_offset == 0 ||    offset_into_cluster(s, guest_offset)
-                                == offset_into_cluster(s, *host_offset));
-
-    /*
-     * Calculate the number of clusters to look for. We stop at L2 table
-     * boundaries to keep things simple.
-     */
-    nb_clusters =
-        size_to_clusters(s, offset_into_cluster(s, guest_offset) + *bytes);
-
-    l2_index = offset_to_l2_index(s, guest_offset);
-    nb_clusters = MIN(nb_clusters, s->l2_size - l2_index);
-
-    /* Find L2 entry for the first involved cluster */
-    ret = get_cluster_table(bs, guest_offset, &l2_table, &l2_index);
-    if (ret < 0) {
-        return ret;
-    }
-
-    cluster_offset = be64_to_cpu(l2_table[l2_index]);
-
-    /* Check how many clusters are already allocated and don't need COW */
-    if (qcow2_get_cluster_type(cluster_offset) == QCOW2_CLUSTER_NORMAL
-        && (cluster_offset & QCOW_OFLAG_COPIED))
-    {
-        /* If a specific host_offset is required, check it */
-        bool offset_matches =
-            (cluster_offset & L2E_OFFSET_MASK) == *host_offset;
-
-        if (*host_offset != 0 && !offset_matches) {
-            *bytes = 0;
-            ret = 0;
-            goto out;
-        }
-
-        /* We keep all QCOW_OFLAG_COPIED clusters */
-        keep_clusters =
-            count_contiguous_clusters(nb_clusters, s->cluster_size,
-                                      &l2_table[l2_index], 0,
-                                      QCOW_OFLAG_COPIED | QCOW_OFLAG_ZERO);
-        assert(keep_clusters <= nb_clusters);
-
-        *bytes = MIN(*bytes,
-                 keep_clusters * s->cluster_size
-                 - offset_into_cluster(s, guest_offset));
-
-        ret = 1;
-    } else {
-        ret = 0;
-    }
-
-    /* Cleanup */
-out:
-    pret = qcow2_cache_put(bs, s->l2_table_cache, (void**) &l2_table);
-    if (pret < 0) {
-        return pret;
-    }
-
-    /* Only return a host offset if we actually made progress. Otherwise we
-     * would make requirements for handle_alloc() that it can't fulfill */
-    if (ret) {
-        *host_offset = (cluster_offset & L2E_OFFSET_MASK)
-                     + offset_into_cluster(s, guest_offset);
-    }
-
-    return ret;
-}
-
-/*
- * Allocates new clusters for the given guest_offset.
- *
- * At most *nb_clusters are allocated, and on return *nb_clusters is updated to
- * contain the number of clusters that have been allocated and are contiguous
- * in the image file.
- *
- * If *host_offset is non-zero, it specifies the offset in the image file at
- * which the new clusters must start. *nb_clusters can be 0 on return in this
- * case if the cluster at host_offset is already in use. If *host_offset is
- * zero, the clusters can be allocated anywhere in the image file.
- *
- * *host_offset is updated to contain the offset into the image file at which
- * the first allocated cluster starts.
- *
- * Return 0 on success and -errno in error cases. -EAGAIN means that the
- * function has been waiting for another request and the allocation must be
- * restarted, but the whole request should not be failed.
- */
-static int do_alloc_cluster_offset(BlockDriverState *bs, uint64_t guest_offset,
-    uint64_t *host_offset, unsigned int *nb_clusters)
-{
-    BDRVQcowState *s = bs->opaque;
-
-    trace_qcow2_do_alloc_clusters_offset(qemu_coroutine_self(), guest_offset,
-                                         *host_offset, *nb_clusters);
-
-    /* Allocate new clusters */
-    trace_qcow2_cluster_alloc_phys(qemu_coroutine_self());
-    if (*host_offset == 0) {
-        int64_t cluster_offset =
-            qcow2_alloc_clusters(bs, *nb_clusters * s->cluster_size);
-        if (cluster_offset < 0) {
-            return cluster_offset;
-        }
-        *host_offset = cluster_offset;
-        return 0;
-    } else {
-        int ret = qcow2_alloc_clusters_at(bs, *host_offset, *nb_clusters);
-        if (ret < 0) {
-            return ret;
-        }
-        *nb_clusters = ret;
-        return 0;
-    }
-}
-
-/*
- * Allocates new clusters for an area that either is yet unallocated or needs a
- * copy on write. If *host_offset is non-zero, clusters are only allocated if
- * the new allocation can match the specified host offset.
- *
- * Note that guest_offset may not be cluster aligned. In this case, the
- * returned *host_offset points to exact byte referenced by guest_offset and
- * therefore isn't cluster aligned as well.
- *
- * Returns:
- *   0:     if no clusters could be allocated. *bytes is set to 0,
- *          *host_offset is left unchanged.
- *
- *   1:     if new clusters were allocated. *bytes may be decreased if the
- *          new allocation doesn't cover all of the requested area.
- *          *host_offset is updated to contain the host offset of the first
- *          newly allocated cluster.
- *
- *  -errno: in error cases
- */
-static int handle_alloc(BlockDriverState *bs, uint64_t guest_offset,
-    uint64_t *host_offset, uint64_t *bytes, QCowL2Meta **m)
-{
-    BDRVQcowState *s = bs->opaque;
-    int l2_index;
-    uint64_t *l2_table;
-    uint64_t entry;
-    unsigned int nb_clusters;
-    int ret;
-
-    uint64_t alloc_cluster_offset;
-
-    trace_qcow2_handle_alloc(qemu_coroutine_self(), guest_offset, *host_offset,
-                             *bytes);
-    assert(*bytes > 0);
-
-    /*
-     * Calculate the number of clusters to look for. We stop at L2 table
-     * boundaries to keep things simple.
-     */
-    nb_clusters =
-        size_to_clusters(s, offset_into_cluster(s, guest_offset) + *bytes);
-
-    l2_index = offset_to_l2_index(s, guest_offset);
-    nb_clusters = MIN(nb_clusters, s->l2_size - l2_index);
-
-    /* Find L2 entry for the first involved cluster */
-    ret = get_cluster_table(bs, guest_offset, &l2_table, &l2_index);
-    if (ret < 0) {
-        return ret;
-    }
-
-    entry = be64_to_cpu(l2_table[l2_index]);
-
-    /* For the moment, overwrite compressed clusters one by one */
-    if (entry & QCOW_OFLAG_COMPRESSED) {
-        nb_clusters = 1;
-    } else {
-        nb_clusters = count_cow_clusters(s, nb_clusters, l2_table, l2_index);
-    }
-
-    /* This function is only called when there were no non-COW clusters, so if
-     * we can't find any unallocated or COW clusters either, something is
-     * wrong with our code. */
-    assert(nb_clusters > 0);
-
-    ret = qcow2_cache_put(bs, s->l2_table_cache, (void**) &l2_table);
-    if (ret < 0) {
-        return ret;
-    }
-
-    /* Allocate, if necessary at a given offset in the image file */
-    alloc_cluster_offset = start_of_cluster(s, *host_offset);
-    ret = do_alloc_cluster_offset(bs, guest_offset, &alloc_cluster_offset,
-                                  &nb_clusters);
-    if (ret < 0) {
-        goto fail;
-    }
-
-    /* Can't extend contiguous allocation */
-    if (nb_clusters == 0) {
-        *bytes = 0;
-        return 0;
-    }
-
-    /*
-     * Save info needed for meta data update.
-     *
-     * requested_sectors: Number of sectors from the start of the first
-     * newly allocated cluster to the end of the (possibly shortened
-     * before) write request.
-     *
-     * avail_sectors: Number of sectors from the start of the first
-     * newly allocated to the end of the last newly allocated cluster.
-     *
-     * nb_sectors: The number of sectors from the start of the first
-     * newly allocated cluster to the end of the area that the write
-     * request actually writes to (excluding COW at the end)
-     */
-    int requested_sectors =
-        (*bytes + offset_into_cluster(s, guest_offset))
-        >> BDRV_SECTOR_BITS;
-    int avail_sectors = nb_clusters
-                        << (s->cluster_bits - BDRV_SECTOR_BITS);
-    int alloc_n_start = offset_into_cluster(s, guest_offset)
-                        >> BDRV_SECTOR_BITS;
-    int nb_sectors = MIN(requested_sectors, avail_sectors);
-    QCowL2Meta *old_m = *m;
-
-    *m = g_malloc0(sizeof(**m));
-
-    **m = (QCowL2Meta) {
-        .next           = old_m,
-
-        .alloc_offset   = alloc_cluster_offset,
-        .offset         = start_of_cluster(s, guest_offset),
-        .nb_clusters    = nb_clusters,
-        .nb_available   = nb_sectors,
-
-        .cow_start = {
-            .offset     = 0,
-            .nb_sectors = alloc_n_start,
-        },
-        .cow_end = {
-            .offset     = nb_sectors * BDRV_SECTOR_SIZE,
-            .nb_sectors = avail_sectors - nb_sectors,
-        },
-    };
-    qemu_co_queue_init(&(*m)->dependent_requests);
-    QLIST_INSERT_HEAD(&s->cluster_allocs, *m, next_in_flight);
-
-    *host_offset = alloc_cluster_offset + offset_into_cluster(s, guest_offset);
-    *bytes = MIN(*bytes, (nb_sectors * BDRV_SECTOR_SIZE)
-                         - offset_into_cluster(s, guest_offset));
-    assert(*bytes != 0);
-
-    return 1;
-
-fail:
-    if (*m && (*m)->nb_clusters > 0) {
-        QLIST_REMOVE(*m, next_in_flight);
-    }
-    return ret;
-}
-
-/*
- * alloc_cluster_offset
- *
- * For a given offset on the virtual disk, find the cluster offset in qcow2
- * file. If the offset is not found, allocate a new cluster.
- *
- * If the cluster was already allocated, m->nb_clusters is set to 0 and
- * other fields in m are meaningless.
- *
- * If the cluster is newly allocated, m->nb_clusters is set to the number of
- * contiguous clusters that have been allocated. In this case, the other
- * fields of m are valid and contain information about the first allocated
- * cluster.
- *
- * If the request conflicts with another write request in flight, the coroutine
- * is queued and will be reentered when the dependency has completed.
- *
- * Return 0 on success and -errno in error cases
- */
-int qcow2_alloc_cluster_offset(BlockDriverState *bs, uint64_t offset,
-    int n_start, int n_end, int *num, uint64_t *host_offset, QCowL2Meta **m)
-{
-    BDRVQcowState *s = bs->opaque;
-    uint64_t start, remaining;
-    uint64_t cluster_offset;
-    uint64_t cur_bytes;
-    int ret;
-
-    trace_qcow2_alloc_clusters_offset(qemu_coroutine_self(), offset,
-                                      n_start, n_end);
-
-    assert(n_start * BDRV_SECTOR_SIZE == offset_into_cluster(s, offset));
-    offset = start_of_cluster(s, offset);
-
-again:
-    start = offset + (n_start << BDRV_SECTOR_BITS);
-    remaining = (n_end - n_start) << BDRV_SECTOR_BITS;
-    cluster_offset = 0;
-    *host_offset = 0;
-    cur_bytes = 0;
-    *m = NULL;
-
-    while (true) {
-
-        if (!*host_offset) {
-            *host_offset = start_of_cluster(s, cluster_offset);
-        }
-
-        assert(remaining >= cur_bytes);
-
-        start           += cur_bytes;
-        remaining       -= cur_bytes;
-        cluster_offset  += cur_bytes;
-
-        if (remaining == 0) {
-            break;
-        }
-
-        cur_bytes = remaining;
-
-        /*
-         * Now start gathering as many contiguous clusters as possible:
-         *
-         * 1. Check for overlaps with in-flight allocations
-         *
-         *      a) Overlap not in the first cluster -> shorten this request and
-         *         let the caller handle the rest in its next loop iteration.
-         *
-         *      b) Real overlaps of two requests. Yield and restart the search
-         *         for contiguous clusters (the situation could have changed
-         *         while we were sleeping)
-         *
-         *      c) TODO: Request starts in the same cluster as the in-flight
-         *         allocation ends. Shorten the COW of the in-fight allocation,
-         *         set cluster_offset to write to the same cluster and set up
-         *         the right synchronisation between the in-flight request and
-         *         the new one.
-         */
-        ret = handle_dependencies(bs, start, &cur_bytes, m);
-        if (ret == -EAGAIN) {
-            /* Currently handle_dependencies() doesn't yield if we already had
-             * an allocation. If it did, we would have to clean up the L2Meta
-             * structs before starting over. */
-            assert(*m == NULL);
-            goto again;
-        } else if (ret < 0) {
-            return ret;
-        } else if (cur_bytes == 0) {
-            break;
-        } else {
-            /* handle_dependencies() may have decreased cur_bytes (shortened
-             * the allocations below) so that the next dependency is processed
-             * correctly during the next loop iteration. */
-        }
-
-        /*
-         * 2. Count contiguous COPIED clusters.
-         */
-        ret = handle_copied(bs, start, &cluster_offset, &cur_bytes, m);
-        if (ret < 0) {
-            return ret;
-        } else if (ret) {
-            continue;
-        } else if (cur_bytes == 0) {
-            break;
-        }
-
-        /*
-         * 3. If the request still hasn't completed, allocate new clusters,
-         *    considering any cluster_offset of steps 1c or 2.
-         */
-        ret = handle_alloc(bs, start, &cluster_offset, &cur_bytes, m);
-        if (ret < 0) {
-            return ret;
-        } else if (ret) {
-            continue;
-        } else {
-            assert(cur_bytes == 0);
-            break;
-        }
-    }
-
-    *num = (n_end - n_start) - (remaining >> BDRV_SECTOR_BITS);
-    assert(*num > 0);
-    assert(*host_offset != 0);
-
-    return 0;
-}
-
-static int decompress_buffer(uint8_t *out_buf, int out_buf_size,
-                             const uint8_t *buf, int buf_size)
-{
-    z_stream strm1, *strm = &strm1;
-    int ret, out_len;
-
-    memset(strm, 0, sizeof(*strm));
-
-    strm->next_in = (uint8_t *)buf;
-    strm->avail_in = buf_size;
-    strm->next_out = out_buf;
-    strm->avail_out = out_buf_size;
-
-    ret = inflateInit2(strm, -12);
-    if (ret != Z_OK)
-        return -1;
-    ret = inflate(strm, Z_FINISH);
-    out_len = strm->next_out - out_buf;
-    if ((ret != Z_STREAM_END && ret != Z_BUF_ERROR) ||
-        out_len != out_buf_size) {
-        inflateEnd(strm);
-        return -1;
-    }
-    inflateEnd(strm);
-    return 0;
-}
-
-int qcow2_decompress_cluster(BlockDriverState *bs, uint64_t cluster_offset)
-{
-    BDRVQcowState *s = bs->opaque;
-    int ret, csize, nb_csectors, sector_offset;
-    uint64_t coffset;
-
-    coffset = cluster_offset & s->cluster_offset_mask;
-    if (s->cluster_cache_offset != coffset) {
-        nb_csectors = ((cluster_offset >> s->csize_shift) & s->csize_mask) + 1;
-        sector_offset = coffset & 511;
-        csize = nb_csectors * 512 - sector_offset;
-        BLKDBG_EVENT(bs->file, BLKDBG_READ_COMPRESSED);
-        ret = bdrv_read(bs->file, coffset >> 9, s->cluster_data, nb_csectors);
-        if (ret < 0) {
-            return ret;
-        }
-        if (decompress_buffer(s->cluster_cache, s->cluster_size,
-                              s->cluster_data + sector_offset, csize) < 0) {
-            return -EIO;
-        }
-        s->cluster_cache_offset = coffset;
-    }
-    return 0;
-}
-
-/*
- * This discards as many clusters of nb_clusters as possible at once (i.e.
- * all clusters in the same L2 table) and returns the number of discarded
- * clusters.
- */
-static int discard_single_l2(BlockDriverState *bs, uint64_t offset,
-    unsigned int nb_clusters)
-{
-    BDRVQcowState *s = bs->opaque;
-    uint64_t *l2_table;
-    int l2_index;
-    int ret;
-    int i;
-
-    ret = get_cluster_table(bs, offset, &l2_table, &l2_index);
-    if (ret < 0) {
-        return ret;
-    }
-
-    /* Limit nb_clusters to one L2 table */
-    nb_clusters = MIN(nb_clusters, s->l2_size - l2_index);
-
-    for (i = 0; i < nb_clusters; i++) {
-        uint64_t old_offset;
-
-        old_offset = be64_to_cpu(l2_table[l2_index + i]);
-        if ((old_offset & L2E_OFFSET_MASK) == 0) {
-            continue;
-        }
-
-        /* First remove L2 entries */
-        qcow2_cache_entry_mark_dirty(s->l2_table_cache, l2_table);
-        l2_table[l2_index + i] = cpu_to_be64(0);
-
-        /* Then decrease the refcount */
-        qcow2_free_any_clusters(bs, old_offset, 1, QCOW2_DISCARD_REQUEST);
-    }
-
-    ret = qcow2_cache_put(bs, s->l2_table_cache, (void**) &l2_table);
-    if (ret < 0) {
-        return ret;
-    }
-
-    return nb_clusters;
-}
-
-int qcow2_discard_clusters(BlockDriverState *bs, uint64_t offset,
-    int nb_sectors)
-{
-    BDRVQcowState *s = bs->opaque;
-    uint64_t end_offset;
-    unsigned int nb_clusters;
-    int ret;
-
-    end_offset = offset + (nb_sectors << BDRV_SECTOR_BITS);
-
-    /* Round start up and end down */
-    offset = align_offset(offset, s->cluster_size);
-    end_offset &= ~(s->cluster_size - 1);
-
-    if (offset > end_offset) {
-        return 0;
-    }
-
-    nb_clusters = size_to_clusters(s, end_offset - offset);
-
-    s->cache_discards = true;
-
-    /* Each L2 table is handled by its own loop iteration */
-    while (nb_clusters > 0) {
-        ret = discard_single_l2(bs, offset, nb_clusters);
-        if (ret < 0) {
-            goto fail;
-        }
-
-        nb_clusters -= ret;
-        offset += (ret * s->cluster_size);
-    }
-
-    ret = 0;
-fail:
-    s->cache_discards = false;
-    qcow2_process_discards(bs, ret);
-
-    return ret;
-}
-
-/*
- * This zeroes as many clusters of nb_clusters as possible at once (i.e.
- * all clusters in the same L2 table) and returns the number of zeroed
- * clusters.
- */
-static int zero_single_l2(BlockDriverState *bs, uint64_t offset,
-    unsigned int nb_clusters)
-{
-    BDRVQcowState *s = bs->opaque;
-    uint64_t *l2_table;
-    int l2_index;
-    int ret;
-    int i;
-
-    ret = get_cluster_table(bs, offset, &l2_table, &l2_index);
-    if (ret < 0) {
-        return ret;
-    }
-
-    /* Limit nb_clusters to one L2 table */
-    nb_clusters = MIN(nb_clusters, s->l2_size - l2_index);
-
-    for (i = 0; i < nb_clusters; i++) {
-        uint64_t old_offset;
-
-        old_offset = be64_to_cpu(l2_table[l2_index + i]);
-
-        /* Update L2 entries */
-        qcow2_cache_entry_mark_dirty(s->l2_table_cache, l2_table);
-        if (old_offset & QCOW_OFLAG_COMPRESSED) {
-            l2_table[l2_index + i] = cpu_to_be64(QCOW_OFLAG_ZERO);
-            qcow2_free_any_clusters(bs, old_offset, 1, QCOW2_DISCARD_REQUEST);
-        } else {
-            l2_table[l2_index + i] |= cpu_to_be64(QCOW_OFLAG_ZERO);
-        }
-    }
-
-    ret = qcow2_cache_put(bs, s->l2_table_cache, (void**) &l2_table);
-    if (ret < 0) {
-        return ret;
-    }
-
-    return nb_clusters;
-}
-
-int qcow2_zero_clusters(BlockDriverState *bs, uint64_t offset, int nb_sectors)
-{
-    BDRVQcowState *s = bs->opaque;
-    unsigned int nb_clusters;
-    int ret;
-
-    /* The zero flag is only supported by version 3 and newer */
-    if (s->qcow_version < 3) {
-        return -ENOTSUP;
-    }
-
-    /* Each L2 table is handled by its own loop iteration */
-    nb_clusters = size_to_clusters(s, nb_sectors << BDRV_SECTOR_BITS);
-
-    s->cache_discards = true;
-
-    while (nb_clusters > 0) {
-        ret = zero_single_l2(bs, offset, nb_clusters);
-        if (ret < 0) {
-            goto fail;
-        }
-
-        nb_clusters -= ret;
-        offset += (ret * s->cluster_size);
-    }
-
-    ret = 0;
-fail:
-    s->cache_discards = false;
-    qcow2_process_discards(bs, ret);
-
-    return ret;
-}
diff --git a/contrib/qemu/block/qcow2-refcount.c b/contrib/qemu/block/qcow2-refcount.c
deleted file mode 100644
index 1244693f39e..00000000000
--- a/contrib/qemu/block/qcow2-refcount.c
+++ /dev/null
@@ -1,1374 +0,0 @@
-/*
- * Block driver for the QCOW version 2 format
- *
- * Copyright (c) 2004-2006 Fabrice Bellard
- *
- * Permission is hereby granted, free of charge, to any person obtaining a copy
- * of this software and associated documentation files (the "Software"), to deal
- * in the Software without restriction, including without limitation the rights
- * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
- * copies of the Software, and to permit persons to whom the Software is
- * furnished to do so, subject to the following conditions:
- *
- * The above copyright notice and this permission notice shall be included in
- * all copies or substantial portions of the Software.
- *
- * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
- * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
- * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
- * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
- * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
- * THE SOFTWARE.
- */
-
-#include "qemu-common.h"
-#include "block/block_int.h"
-#include "block/qcow2.h"
-
-static int64_t alloc_clusters_noref(BlockDriverState *bs, int64_t size);
-static int QEMU_WARN_UNUSED_RESULT update_refcount(BlockDriverState *bs,
-                            int64_t offset, int64_t length,
-                            int addend, enum qcow2_discard_type type);
-
-
-/*********************************************************/
-/* refcount handling */
-
-int qcow2_refcount_init(BlockDriverState *bs)
-{
-    BDRVQcowState *s = bs->opaque;
-    int ret, refcount_table_size2, i;
-
-    refcount_table_size2 = s->refcount_table_size * sizeof(uint64_t);
-    s->refcount_table = g_malloc(refcount_table_size2);
-    if (s->refcount_table_size > 0) {
-        BLKDBG_EVENT(bs->file, BLKDBG_REFTABLE_LOAD);
-        ret = bdrv_pread(bs->file, s->refcount_table_offset,
-                         s->refcount_table, refcount_table_size2);
-        if (ret != refcount_table_size2)
-            goto fail;
-        for(i = 0; i < s->refcount_table_size; i++)
-            be64_to_cpus(&s->refcount_table[i]);
-    }
-    return 0;
- fail:
-    return -ENOMEM;
-}
-
-void qcow2_refcount_close(BlockDriverState *bs)
-{
-    BDRVQcowState *s = bs->opaque;
-    g_free(s->refcount_table);
-}
-
-
-static int load_refcount_block(BlockDriverState *bs,
-                               int64_t refcount_block_offset,
-                               void **refcount_block)
-{
-    BDRVQcowState *s = bs->opaque;
-    int ret;
-
-    BLKDBG_EVENT(bs->file, BLKDBG_REFBLOCK_LOAD);
-    ret = qcow2_cache_get(bs, s->refcount_block_cache, refcount_block_offset,
-        refcount_block);
-
-    return ret;
-}
-
-/*
- * Returns the refcount of the cluster given by its index. Any non-negative
- * return value is the refcount of the cluster, negative values are -errno
- * and indicate an error.
- */
-static int get_refcount(BlockDriverState *bs, int64_t cluster_index)
-{
-    BDRVQcowState *s = bs->opaque;
-    int refcount_table_index, block_index;
-    int64_t refcount_block_offset;
-    int ret;
-    uint16_t *refcount_block;
-    uint16_t refcount;
-
-    refcount_table_index = cluster_index >> (s->cluster_bits - REFCOUNT_SHIFT);
-    if (refcount_table_index >= s->refcount_table_size)
-        return 0;
-    refcount_block_offset = s->refcount_table[refcount_table_index];
-    if (!refcount_block_offset)
-        return 0;
-
-    ret = qcow2_cache_get(bs, s->refcount_block_cache, refcount_block_offset,
-        (void**) &refcount_block);
-    if (ret < 0) {
-        return ret;
-    }
-
-    block_index = cluster_index &
-        ((1 << (s->cluster_bits - REFCOUNT_SHIFT)) - 1);
-    refcount = be16_to_cpu(refcount_block[block_index]);
-
-    ret = qcow2_cache_put(bs, s->refcount_block_cache,
-        (void**) &refcount_block);
-    if (ret < 0) {
-        return ret;
-    }
-
-    return refcount;
-}
-
-/*
- * Rounds the refcount table size up to avoid growing the table for each single
- * refcount block that is allocated.
- */
-static unsigned int next_refcount_table_size(BDRVQcowState *s,
-    unsigned int min_size)
-{
-    unsigned int min_clusters = (min_size >> (s->cluster_bits - 3)) + 1;
-    unsigned int refcount_table_clusters =
-        MAX(1, s->refcount_table_size >> (s->cluster_bits - 3));
-
-    while (min_clusters > refcount_table_clusters) {
-        refcount_table_clusters = (refcount_table_clusters * 3 + 1) / 2;
-    }
-
-    return refcount_table_clusters << (s->cluster_bits - 3);
-}
-
-
-/* Checks if two offsets are described by the same refcount block */
-static int in_same_refcount_block(BDRVQcowState *s, uint64_t offset_a,
-    uint64_t offset_b)
-{
-    uint64_t block_a = offset_a >> (2 * s->cluster_bits - REFCOUNT_SHIFT);
-    uint64_t block_b = offset_b >> (2 * s->cluster_bits - REFCOUNT_SHIFT);
-
-    return (block_a == block_b);
-}
-
-/*
- * Loads a refcount block. If it doesn't exist yet, it is allocated first
- * (including growing the refcount table if needed).
- *
- * Returns 0 on success or -errno in error case
- */
-static int alloc_refcount_block(BlockDriverState *bs,
-    int64_t cluster_index, uint16_t **refcount_block)
-{
-    BDRVQcowState *s = bs->opaque;
-    unsigned int refcount_table_index;
-    int ret;
-
-    BLKDBG_EVENT(bs->file, BLKDBG_REFBLOCK_ALLOC);
-
-    /* Find the refcount block for the given cluster */
-    refcount_table_index = cluster_index >> (s->cluster_bits - REFCOUNT_SHIFT);
-
-    if (refcount_table_index < s->refcount_table_size) {
-
-        uint64_t refcount_block_offset =
-            s->refcount_table[refcount_table_index] & REFT_OFFSET_MASK;
-
-        /* If it's already there, we're done */
-        if (refcount_block_offset) {
-             return load_refcount_block(bs, refcount_block_offset,
-                 (void**) refcount_block);
-        }
-    }
-
-    /*
-     * If we came here, we need to allocate something. Something is at least
-     * a cluster for the new refcount block. It may also include a new refcount
-     * table if the old refcount table is too small.
-     *
-     * Note that allocating clusters here needs some special care:
-     *
-     * - We can't use the normal qcow2_alloc_clusters(), it would try to
-     *   increase the refcount and very likely we would end up with an endless
-     *   recursion. Instead we must place the refcount blocks in a way that
-     *   they can describe them themselves.
-     *
-     * - We need to consider that at this point we are inside update_refcounts
-     *   and doing the initial refcount increase. This means that some clusters
-     *   have already been allocated by the caller, but their refcount isn't
-     *   accurate yet. free_cluster_index tells us where this allocation ends
-     *   as long as we don't overwrite it by freeing clusters.
-     *
-     * - alloc_clusters_noref and qcow2_free_clusters may load a different
-     *   refcount block into the cache
-     */
-
-    *refcount_block = NULL;
-
-    /* We write to the refcount table, so we might depend on L2 tables */
-    ret = qcow2_cache_flush(bs, s->l2_table_cache);
-    if (ret < 0) {
-        return ret;
-    }
-
-    /* Allocate the refcount block itself and mark it as used */
-    int64_t new_block = alloc_clusters_noref(bs, s->cluster_size);
-    if (new_block < 0) {
-        return new_block;
-    }
-
-#ifdef DEBUG_ALLOC2
-    fprintf(stderr, "qcow2: Allocate refcount block %d for %" PRIx64
-        " at %" PRIx64 "\n",
-        refcount_table_index, cluster_index << s->cluster_bits, new_block);
-#endif
-
-    if (in_same_refcount_block(s, new_block, cluster_index << s->cluster_bits)) {
-        /* Zero the new refcount block before updating it */
-        ret = qcow2_cache_get_empty(bs, s->refcount_block_cache, new_block,
-            (void**) refcount_block);
-        if (ret < 0) {
-            goto fail_block;
-        }
-
-        memset(*refcount_block, 0, s->cluster_size);
-
-        /* The block describes itself, need to update the cache */
-        int block_index = (new_block >> s->cluster_bits) &
-            ((1 << (s->cluster_bits - REFCOUNT_SHIFT)) - 1);
-        (*refcount_block)[block_index] = cpu_to_be16(1);
-    } else {
-        /* Described somewhere else. This can recurse at most twice before we
-         * arrive at a block that describes itself. */
-        ret = update_refcount(bs, new_block, s->cluster_size, 1,
-                              QCOW2_DISCARD_NEVER);
-        if (ret < 0) {
-            goto fail_block;
-        }
-
-        ret = qcow2_cache_flush(bs, s->refcount_block_cache);
-        if (ret < 0) {
-            goto fail_block;
-        }
-
-        /* Initialize the new refcount block only after updating its refcount,
-         * update_refcount uses the refcount cache itself */
-        ret = qcow2_cache_get_empty(bs, s->refcount_block_cache, new_block,
-            (void**) refcount_block);
-        if (ret < 0) {
-            goto fail_block;
-        }
-
-        memset(*refcount_block, 0, s->cluster_size);
-    }
-
-    /* Now the new refcount block needs to be written to disk */
-    BLKDBG_EVENT(bs->file, BLKDBG_REFBLOCK_ALLOC_WRITE);
-    qcow2_cache_entry_mark_dirty(s->refcount_block_cache, *refcount_block);
-    ret = qcow2_cache_flush(bs, s->refcount_block_cache);
-    if (ret < 0) {
-        goto fail_block;
-    }
-
-    /* If the refcount table is big enough, just hook the block up there */
-    if (refcount_table_index < s->refcount_table_size) {
-        uint64_t data64 = cpu_to_be64(new_block);
-        BLKDBG_EVENT(bs->file, BLKDBG_REFBLOCK_ALLOC_HOOKUP);
-        ret = bdrv_pwrite_sync(bs->file,
-            s->refcount_table_offset + refcount_table_index * sizeof(uint64_t),
-            &data64, sizeof(data64));
-        if (ret < 0) {
-            goto fail_block;
-        }
-
-        s->refcount_table[refcount_table_index] = new_block;
-        return 0;
-    }
-
-    ret = qcow2_cache_put(bs, s->refcount_block_cache, (void**) refcount_block);
-    if (ret < 0) {
-        goto fail_block;
-    }
-
-    /*
-     * If we come here, we need to grow the refcount table. Again, a new
-     * refcount table needs some space and we can't simply allocate to avoid
-     * endless recursion.
-     *
-     * Therefore let's grab new refcount blocks at the end of the image, which
-     * will describe themselves and the new refcount table. This way we can
-     * reference them only in the new table and do the switch to the new
-     * refcount table at once without producing an inconsistent state in
-     * between.
-     */
-    BLKDBG_EVENT(bs->file, BLKDBG_REFTABLE_GROW);
-
-    /* Calculate the number of refcount blocks needed so far */
-    uint64_t refcount_block_clusters = 1 << (s->cluster_bits - REFCOUNT_SHIFT);
-    uint64_t blocks_used = (s->free_cluster_index +
-        refcount_block_clusters - 1) / refcount_block_clusters;
-
-    /* And now we need at least one block more for the new metadata */
-    uint64_t table_size = next_refcount_table_size(s, blocks_used + 1);
-    uint64_t last_table_size;
-    uint64_t blocks_clusters;
-    do {
-        uint64_t table_clusters =
-            size_to_clusters(s, table_size * sizeof(uint64_t));
-        blocks_clusters = 1 +
-            ((table_clusters + refcount_block_clusters - 1)
-            / refcount_block_clusters);
-        uint64_t meta_clusters = table_clusters + blocks_clusters;
-
-        last_table_size = table_size;
-        table_size = next_refcount_table_size(s, blocks_used +
-            ((meta_clusters + refcount_block_clusters - 1)
-            / refcount_block_clusters));
-
-    } while (last_table_size != table_size);
-
-#ifdef DEBUG_ALLOC2
-    fprintf(stderr, "qcow2: Grow refcount table %" PRId32 " => %" PRId64 "\n",
-        s->refcount_table_size, table_size);
-#endif
-
-    /* Create the new refcount table and blocks */
-    uint64_t meta_offset = (blocks_used * refcount_block_clusters) *
-        s->cluster_size;
-    uint64_t table_offset = meta_offset + blocks_clusters * s->cluster_size;
-    uint16_t *new_blocks = g_malloc0(blocks_clusters * s->cluster_size);
-    uint64_t *new_table = g_malloc0(table_size * sizeof(uint64_t));
-
-    assert(meta_offset >= (s->free_cluster_index * s->cluster_size));
-
-    /* Fill the new refcount table */
-    memcpy(new_table, s->refcount_table,
-        s->refcount_table_size * sizeof(uint64_t));
-    new_table[refcount_table_index] = new_block;
-
-    int i;
-    for (i = 0; i < blocks_clusters; i++) {
-        new_table[blocks_used + i] = meta_offset + (i * s->cluster_size);
-    }
-
-    /* Fill the refcount blocks */
-    uint64_t table_clusters = size_to_clusters(s, table_size * sizeof(uint64_t));
-    int block = 0;
-    for (i = 0; i < table_clusters + blocks_clusters; i++) {
-        new_blocks[block++] = cpu_to_be16(1);
-    }
-
-    /* Write refcount blocks to disk */
-    BLKDBG_EVENT(bs->file, BLKDBG_REFBLOCK_ALLOC_WRITE_BLOCKS);
-    ret = bdrv_pwrite_sync(bs->file, meta_offset, new_blocks,
-        blocks_clusters * s->cluster_size);
-    g_free(new_blocks);
-    if (ret < 0) {
-        goto fail_table;
-    }
-
-    /* Write refcount table to disk */
-    for(i = 0; i < table_size; i++) {
-        cpu_to_be64s(&new_table[i]);
-    }
-
-    BLKDBG_EVENT(bs->file, BLKDBG_REFBLOCK_ALLOC_WRITE_TABLE);
-    ret = bdrv_pwrite_sync(bs->file, table_offset, new_table,
-        table_size * sizeof(uint64_t));
-    if (ret < 0) {
-        goto fail_table;
-    }
-
-    for(i = 0; i < table_size; i++) {
-        be64_to_cpus(&new_table[i]);
-    }
-
-    /* Hook up the new refcount table in the qcow2 header */
-    uint8_t data[12];
-    cpu_to_be64w((uint64_t*)data, table_offset);
-    cpu_to_be32w((uint32_t*)(data + 8), table_clusters);
-    BLKDBG_EVENT(bs->file, BLKDBG_REFBLOCK_ALLOC_SWITCH_TABLE);
-    ret = bdrv_pwrite_sync(bs->file, offsetof(QCowHeader, refcount_table_offset),
-        data, sizeof(data));
-    if (ret < 0) {
-        goto fail_table;
-    }
-
-    /* And switch it in memory */
-    uint64_t old_table_offset = s->refcount_table_offset;
-    uint64_t old_table_size = s->refcount_table_size;
-
-    g_free(s->refcount_table);
-    s->refcount_table = new_table;
-    s->refcount_table_size = table_size;
-    s->refcount_table_offset = table_offset;
-
-    /* Free old table. Remember, we must not change free_cluster_index */
-    uint64_t old_free_cluster_index = s->free_cluster_index;
-    qcow2_free_clusters(bs, old_table_offset, old_table_size * sizeof(uint64_t),
-                        QCOW2_DISCARD_OTHER);
-    s->free_cluster_index = old_free_cluster_index;
-
-    ret = load_refcount_block(bs, new_block, (void**) refcount_block);
-    if (ret < 0) {
-        return ret;
-    }
-
-    return 0;
-
-fail_table:
-    g_free(new_table);
-fail_block:
-    if (*refcount_block != NULL) {
-        qcow2_cache_put(bs, s->refcount_block_cache, (void**) refcount_block);
-    }
-    return ret;
-}
-
-void qcow2_process_discards(BlockDriverState *bs, int ret)
-{
-    BDRVQcowState *s = bs->opaque;
-    Qcow2DiscardRegion *d, *next;
-
-    QTAILQ_FOREACH_SAFE(d, &s->discards, next, next) {
-        QTAILQ_REMOVE(&s->discards, d, next);
-
-        /* Discard is optional, ignore the return value */
-        if (ret >= 0) {
-            bdrv_discard(bs->file,
-                         d->offset >> BDRV_SECTOR_BITS,
-                         d->bytes >> BDRV_SECTOR_BITS);
-        }
-
-        g_free(d);
-    }
-}
-
-static void update_refcount_discard(BlockDriverState *bs,
-                                    uint64_t offset, uint64_t length)
-{
-    BDRVQcowState *s = bs->opaque;
-    Qcow2DiscardRegion *d, *p, *next;
-
-    QTAILQ_FOREACH(d, &s->discards, next) {
-        uint64_t new_start = MIN(offset, d->offset);
-        uint64_t new_end = MAX(offset + length, d->offset + d->bytes);
-
-        if (new_end - new_start <= length + d->bytes) {
-            /* There can't be any overlap, areas ending up here have no
-             * references any more and therefore shouldn't get freed another
-             * time. */
-            assert(d->bytes + length == new_end - new_start);
-            d->offset = new_start;
-            d->bytes = new_end - new_start;
-            goto found;
-        }
-    }
-
-    d = g_malloc(sizeof(*d));
-    *d = (Qcow2DiscardRegion) {
-        .bs     = bs,
-        .offset = offset,
-        .bytes  = length,
-    };
-    QTAILQ_INSERT_TAIL(&s->discards, d, next);
-
-found:
-    /* Merge discard requests if they are adjacent now */
-    QTAILQ_FOREACH_SAFE(p, &s->discards, next, next) {
-        if (p == d
-            || p->offset > d->offset + d->bytes
-            || d->offset > p->offset + p->bytes)
-        {
-            continue;
-        }
-
-        /* Still no overlap possible */
-        assert(p->offset == d->offset + d->bytes
-            || d->offset == p->offset + p->bytes);
-
-        QTAILQ_REMOVE(&s->discards, p, next);
-        d->offset = MIN(d->offset, p->offset);
-        d->bytes += p->bytes;
-    }
-}
-
-/* XXX: cache several refcount block clusters ? */
-static int QEMU_WARN_UNUSED_RESULT update_refcount(BlockDriverState *bs,
-    int64_t offset, int64_t length, int addend, enum qcow2_discard_type type)
-{
-    BDRVQcowState *s = bs->opaque;
-    int64_t start, last, cluster_offset;
-    uint16_t *refcount_block = NULL;
-    int64_t old_table_index = -1;
-    int ret;
-
-#ifdef DEBUG_ALLOC2
-    fprintf(stderr, "update_refcount: offset=%" PRId64 " size=%" PRId64 " addend=%d\n",
-           offset, length, addend);
-#endif
-    if (length < 0) {
-        return -EINVAL;
-    } else if (length == 0) {
-        return 0;
-    }
-
-    if (addend < 0) {
-        qcow2_cache_set_dependency(bs, s->refcount_block_cache,
-            s->l2_table_cache);
-    }
-
-    start = offset & ~(s->cluster_size - 1);
-    last = (offset + length - 1) & ~(s->cluster_size - 1);
-    for(cluster_offset = start; cluster_offset <= last;
-        cluster_offset += s->cluster_size)
-    {
-        int block_index, refcount;
-        int64_t cluster_index = cluster_offset >> s->cluster_bits;
-        int64_t table_index =
-            cluster_index >> (s->cluster_bits - REFCOUNT_SHIFT);
-
-        /* Load the refcount block and allocate it if needed */
-        if (table_index != old_table_index) {
-            if (refcount_block) {
-                ret = qcow2_cache_put(bs, s->refcount_block_cache,
-                    (void**) &refcount_block);
-                if (ret < 0) {
-                    goto fail;
-                }
-            }
-
-            ret = alloc_refcount_block(bs, cluster_index, &refcount_block);
-            if (ret < 0) {
-                goto fail;
-            }
-        }
-        old_table_index = table_index;
-
-        qcow2_cache_entry_mark_dirty(s->refcount_block_cache, refcount_block);
-
-        /* we can update the count and save it */
-        block_index = cluster_index &
-            ((1 << (s->cluster_bits - REFCOUNT_SHIFT)) - 1);
-
-        refcount = be16_to_cpu(refcount_block[block_index]);
-        refcount += addend;
-        if (refcount < 0 || refcount > 0xffff) {
-            ret = -EINVAL;
-            goto fail;
-        }
-        if (refcount == 0 && cluster_index < s->free_cluster_index) {
-            s->free_cluster_index = cluster_index;
-        }
-        refcount_block[block_index] = cpu_to_be16(refcount);
-
-        if (refcount == 0 && s->discard_passthrough[type]) {
-            update_refcount_discard(bs, cluster_offset, s->cluster_size);
-        }
-    }
-
-    ret = 0;
-fail:
-    if (!s->cache_discards) {
-        qcow2_process_discards(bs, ret);
-    }
-
-    /* Write last changed block to disk */
-    if (refcount_block) {
-        int wret;
-        wret = qcow2_cache_put(bs, s->refcount_block_cache,
-            (void**) &refcount_block);
-        if (wret < 0) {
-            return ret < 0 ? ret : wret;
-        }
-    }
-
-    /*
-     * Try do undo any updates if an error is returned (This may succeed in
-     * some cases like ENOSPC for allocating a new refcount block)
-     */
-    if (ret < 0) {
-        int dummy;
-        dummy = update_refcount(bs, offset, cluster_offset - offset, -addend,
-                                QCOW2_DISCARD_NEVER);
-        (void)dummy;
-    }
-
-    return ret;
-}
-
-/*
- * Increases or decreases the refcount of a given cluster by one.
- * addend must be 1 or -1.
- *
- * If the return value is non-negative, it is the new refcount of the cluster.
- * If it is negative, it is -errno and indicates an error.
- */
-static int update_cluster_refcount(BlockDriverState *bs,
-                                   int64_t cluster_index,
-                                   int addend,
-                                   enum qcow2_discard_type type)
-{
-    BDRVQcowState *s = bs->opaque;
-    int ret;
-
-    ret = update_refcount(bs, cluster_index << s->cluster_bits, 1, addend,
-                          type);
-    if (ret < 0) {
-        return ret;
-    }
-
-    return get_refcount(bs, cluster_index);
-}
-
-
-
-/*********************************************************/
-/* cluster allocation functions */
-
-
-
-/* return < 0 if error */
-static int64_t alloc_clusters_noref(BlockDriverState *bs, int64_t size)
-{
-    BDRVQcowState *s = bs->opaque;
-    int i, nb_clusters, refcount;
-
-    nb_clusters = size_to_clusters(s, size);
-retry:
-    for(i = 0; i < nb_clusters; i++) {
-        int64_t next_cluster_index = s->free_cluster_index++;
-        refcount = get_refcount(bs, next_cluster_index);
-
-        if (refcount < 0) {
-            return refcount;
-        } else if (refcount != 0) {
-            goto retry;
-        }
-    }
-#ifdef DEBUG_ALLOC2
-    fprintf(stderr, "alloc_clusters: size=%" PRId64 " -> %" PRId64 "\n",
-            size,
-            (s->free_cluster_index - nb_clusters) << s->cluster_bits);
-#endif
-    return (s->free_cluster_index - nb_clusters) << s->cluster_bits;
-}
-
-int64_t qcow2_alloc_clusters(BlockDriverState *bs, int64_t size)
-{
-    int64_t offset;
-    int ret;
-
-    BLKDBG_EVENT(bs->file, BLKDBG_CLUSTER_ALLOC);
-    offset = alloc_clusters_noref(bs, size);
-    if (offset < 0) {
-        return offset;
-    }
-
-    ret = update_refcount(bs, offset, size, 1, QCOW2_DISCARD_NEVER);
-    if (ret < 0) {
-        return ret;
-    }
-
-    return offset;
-}
-
-int qcow2_alloc_clusters_at(BlockDriverState *bs, uint64_t offset,
-    int nb_clusters)
-{
-    BDRVQcowState *s = bs->opaque;
-    uint64_t cluster_index;
-    uint64_t old_free_cluster_index;
-    int i, refcount, ret;
-
-    /* Check how many clusters there are free */
-    cluster_index = offset >> s->cluster_bits;
-    for(i = 0; i < nb_clusters; i++) {
-        refcount = get_refcount(bs, cluster_index++);
-
-        if (refcount < 0) {
-            return refcount;
-        } else if (refcount != 0) {
-            break;
-        }
-    }
-
-    /* And then allocate them */
-    old_free_cluster_index = s->free_cluster_index;
-    s->free_cluster_index = cluster_index + i;
-
-    ret = update_refcount(bs, offset, i << s->cluster_bits, 1,
-                          QCOW2_DISCARD_NEVER);
-    if (ret < 0) {
-        return ret;
-    }
-
-    s->free_cluster_index = old_free_cluster_index;
-
-    return i;
-}
-
-/* only used to allocate compressed sectors. We try to allocate
-   contiguous sectors. size must be <= cluster_size */
-int64_t qcow2_alloc_bytes(BlockDriverState *bs, int size)
-{
-    BDRVQcowState *s = bs->opaque;
-    int64_t offset, cluster_offset;
-    int free_in_cluster;
-
-    BLKDBG_EVENT(bs->file, BLKDBG_CLUSTER_ALLOC_BYTES);
-    assert(size > 0 && size <= s->cluster_size);
-    if (s->free_byte_offset == 0) {
-        offset = qcow2_alloc_clusters(bs, s->cluster_size);
-        if (offset < 0) {
-            return offset;
-        }
-        s->free_byte_offset = offset;
-    }
- redo:
-    free_in_cluster = s->cluster_size -
-        (s->free_byte_offset & (s->cluster_size - 1));
-    if (size <= free_in_cluster) {
-        /* enough space in current cluster */
-        offset = s->free_byte_offset;
-        s->free_byte_offset += size;
-        free_in_cluster -= size;
-        if (free_in_cluster == 0)
-            s->free_byte_offset = 0;
-        if ((offset & (s->cluster_size - 1)) != 0)
-            update_cluster_refcount(bs, offset >> s->cluster_bits, 1,
-                                    QCOW2_DISCARD_NEVER);
-    } else {
-        offset = qcow2_alloc_clusters(bs, s->cluster_size);
-        if (offset < 0) {
-            return offset;
-        }
-        cluster_offset = s->free_byte_offset & ~(s->cluster_size - 1);
-        if ((cluster_offset + s->cluster_size) == offset) {
-            /* we are lucky: contiguous data */
-            offset = s->free_byte_offset;
-            update_cluster_refcount(bs, offset >> s->cluster_bits, 1,
-                                    QCOW2_DISCARD_NEVER);
-            s->free_byte_offset += size;
-        } else {
-            s->free_byte_offset = offset;
-            goto redo;
-        }
-    }
-
-    /* The cluster refcount was incremented, either by qcow2_alloc_clusters()
-     * or explicitly by update_cluster_refcount().  Refcount blocks must be
-     * flushed before the caller's L2 table updates.
-     */
-    qcow2_cache_set_dependency(bs, s->l2_table_cache, s->refcount_block_cache);
-    return offset;
-}
-
-void qcow2_free_clusters(BlockDriverState *bs,
-                          int64_t offset, int64_t size,
-                          enum qcow2_discard_type type)
-{
-    int ret;
-
-    BLKDBG_EVENT(bs->file, BLKDBG_CLUSTER_FREE);
-    ret = update_refcount(bs, offset, size, -1, type);
-    if (ret < 0) {
-        fprintf(stderr, "qcow2_free_clusters failed: %s\n", strerror(-ret));
-        /* TODO Remember the clusters to free them later and avoid leaking */
-    }
-}
-
-/*
- * Free a cluster using its L2 entry (handles clusters of all types, e.g.
- * normal cluster, compressed cluster, etc.)
- */
-void qcow2_free_any_clusters(BlockDriverState *bs, uint64_t l2_entry,
-                             int nb_clusters, enum qcow2_discard_type type)
-{
-    BDRVQcowState *s = bs->opaque;
-
-    switch (qcow2_get_cluster_type(l2_entry)) {
-    case QCOW2_CLUSTER_COMPRESSED:
-        {
-            int nb_csectors;
-            nb_csectors = ((l2_entry >> s->csize_shift) &
-                           s->csize_mask) + 1;
-            qcow2_free_clusters(bs,
-                (l2_entry & s->cluster_offset_mask) & ~511,
-                nb_csectors * 512, type);
-        }
-        break;
-    case QCOW2_CLUSTER_NORMAL:
-        qcow2_free_clusters(bs, l2_entry & L2E_OFFSET_MASK,
-                            nb_clusters << s->cluster_bits, type);
-        break;
-    case QCOW2_CLUSTER_UNALLOCATED:
-    case QCOW2_CLUSTER_ZERO:
-        break;
-    default:
-        abort();
-    }
-}
-
-
-
-/*********************************************************/
-/* snapshots and image creation */
-
-
-
-/* update the refcounts of snapshots and the copied flag */
-int qcow2_update_snapshot_refcount(BlockDriverState *bs,
-    int64_t l1_table_offset, int l1_size, int addend)
-{
-    BDRVQcowState *s = bs->opaque;
-    uint64_t *l1_table, *l2_table, l2_offset, offset, l1_size2, l1_allocated;
-    int64_t old_offset, old_l2_offset;
-    int i, j, l1_modified = 0, nb_csectors, refcount;
-    int ret;
-
-    l2_table = NULL;
-    l1_table = NULL;
-    l1_size2 = l1_size * sizeof(uint64_t);
-
-    s->cache_discards = true;
-
-    /* WARNING: qcow2_snapshot_goto relies on this function not using the
-     * l1_table_offset when it is the current s->l1_table_offset! Be careful
-     * when changing this! */
-    if (l1_table_offset != s->l1_table_offset) {
-        l1_table = g_malloc0(align_offset(l1_size2, 512));
-        l1_allocated = 1;
-
-        ret = bdrv_pread(bs->file, l1_table_offset, l1_table, l1_size2);
-        if (ret < 0) {
-            goto fail;
-        }
-
-        for(i = 0;i < l1_size; i++)
-            be64_to_cpus(&l1_table[i]);
-    } else {
-        assert(l1_size == s->l1_size);
-        l1_table = s->l1_table;
-        l1_allocated = 0;
-    }
-
-    for(i = 0; i < l1_size; i++) {
-        l2_offset = l1_table[i];
-        if (l2_offset) {
-            old_l2_offset = l2_offset;
-            l2_offset &= L1E_OFFSET_MASK;
-
-            ret = qcow2_cache_get(bs, s->l2_table_cache, l2_offset,
-                (void**) &l2_table);
-            if (ret < 0) {
-                goto fail;
-            }
-
-            for(j = 0; j < s->l2_size; j++) {
-                offset = be64_to_cpu(l2_table[j]);
-                if (offset != 0) {
-                    old_offset = offset;
-                    offset &= ~QCOW_OFLAG_COPIED;
-                    if (offset & QCOW_OFLAG_COMPRESSED) {
-                        nb_csectors = ((offset >> s->csize_shift) &
-                                       s->csize_mask) + 1;
-                        if (addend != 0) {
-                            int ret;
-                            ret = update_refcount(bs,
-                                (offset & s->cluster_offset_mask) & ~511,
-                                nb_csectors * 512, addend,
-                                QCOW2_DISCARD_SNAPSHOT);
-                            if (ret < 0) {
-                                goto fail;
-                            }
-                        }
-                        /* compressed clusters are never modified */
-                        refcount = 2;
-                    } else {
-                        uint64_t cluster_index = (offset & L2E_OFFSET_MASK) >> s->cluster_bits;
-                        if (addend != 0) {
-                            refcount = update_cluster_refcount(bs, cluster_index, addend,
-                                                               QCOW2_DISCARD_SNAPSHOT);
-                        } else {
-                            refcount = get_refcount(bs, cluster_index);
-                        }
-
-                        if (refcount < 0) {
-                            ret = refcount;
-                            goto fail;
-                        }
-                    }
-
-                    if (refcount == 1) {
-                        offset |= QCOW_OFLAG_COPIED;
-                    }
-                    if (offset != old_offset) {
-                        if (addend > 0) {
-                            qcow2_cache_set_dependency(bs, s->l2_table_cache,
-                                s->refcount_block_cache);
-                        }
-                        l2_table[j] = cpu_to_be64(offset);
-                        qcow2_cache_entry_mark_dirty(s->l2_table_cache, l2_table);
-                    }
-                }
-            }
-
-            ret = qcow2_cache_put(bs, s->l2_table_cache, (void**) &l2_table);
-            if (ret < 0) {
-                goto fail;
-            }
-
-
-            if (addend != 0) {
-                refcount = update_cluster_refcount(bs, l2_offset >> s->cluster_bits, addend,
-                                                   QCOW2_DISCARD_SNAPSHOT);
-            } else {
-                refcount = get_refcount(bs, l2_offset >> s->cluster_bits);
-            }
-            if (refcount < 0) {
-                ret = refcount;
-                goto fail;
-            } else if (refcount == 1) {
-                l2_offset |= QCOW_OFLAG_COPIED;
-            }
-            if (l2_offset != old_l2_offset) {
-                l1_table[i] = l2_offset;
-                l1_modified = 1;
-            }
-        }
-    }
-
-    ret = bdrv_flush(bs);
-fail:
-    if (l2_table) {
-        qcow2_cache_put(bs, s->l2_table_cache, (void**) &l2_table);
-    }
-
-    s->cache_discards = false;
-    qcow2_process_discards(bs, ret);
-
-    /* Update L1 only if it isn't deleted anyway (addend = -1) */
-    if (ret == 0 && addend >= 0 && l1_modified) {
-        for (i = 0; i < l1_size; i++) {
-            cpu_to_be64s(&l1_table[i]);
-        }
-
-        ret = bdrv_pwrite_sync(bs->file, l1_table_offset, l1_table, l1_size2);
-
-        for (i = 0; i < l1_size; i++) {
-            be64_to_cpus(&l1_table[i]);
-        }
-    }
-    if (l1_allocated)
-        g_free(l1_table);
-    return ret;
-}
-
-
-
-
-/*********************************************************/
-/* refcount checking functions */
-
-
-
-/*
- * Increases the refcount for a range of clusters in a given refcount table.
- * This is used to construct a temporary refcount table out of L1 and L2 tables
- * which can be compared the the refcount table saved in the image.
- *
- * Modifies the number of errors in res.
- */
-static void inc_refcounts(BlockDriverState *bs,
-                          BdrvCheckResult *res,
-                          uint16_t *refcount_table,
-                          int refcount_table_size,
-                          int64_t offset, int64_t size)
-{
-    BDRVQcowState *s = bs->opaque;
-    int64_t start, last, cluster_offset;
-    int k;
-
-    if (size <= 0)
-        return;
-
-    start = offset & ~(s->cluster_size - 1);
-    last = (offset + size - 1) & ~(s->cluster_size - 1);
-    for(cluster_offset = start; cluster_offset <= last;
-        cluster_offset += s->cluster_size) {
-        k = cluster_offset >> s->cluster_bits;
-        if (k < 0) {
-            fprintf(stderr, "ERROR: invalid cluster offset=0x%" PRIx64 "\n",
-                cluster_offset);
-            res->corruptions++;
-        } else if (k >= refcount_table_size) {
-            fprintf(stderr, "Warning: cluster offset=0x%" PRIx64 " is after "
-                "the end of the image file, can't properly check refcounts.\n",
-                cluster_offset);
-            res->check_errors++;
-        } else {
-            if (++refcount_table[k] == 0) {
-                fprintf(stderr, "ERROR: overflow cluster offset=0x%" PRIx64
-                    "\n", cluster_offset);
-                res->corruptions++;
-            }
-        }
-    }
-}
-
-/* Flags for check_refcounts_l1() and check_refcounts_l2() */
-enum {
-    CHECK_OFLAG_COPIED = 0x1,   /* check QCOW_OFLAG_COPIED matches refcount */
-    CHECK_FRAG_INFO = 0x2,      /* update BlockFragInfo counters */
-};
-
-/*
- * Increases the refcount in the given refcount table for the all clusters
- * referenced in the L2 table. While doing so, performs some checks on L2
- * entries.
- *
- * Returns the number of errors found by the checks or -errno if an internal
- * error occurred.
- */
-static int check_refcounts_l2(BlockDriverState *bs, BdrvCheckResult *res,
-    uint16_t *refcount_table, int refcount_table_size, int64_t l2_offset,
-    int flags)
-{
-    BDRVQcowState *s = bs->opaque;
-    uint64_t *l2_table, l2_entry;
-    uint64_t next_contiguous_offset = 0;
-    int i, l2_size, nb_csectors, refcount;
-
-    /* Read L2 table from disk */
-    l2_size = s->l2_size * sizeof(uint64_t);
-    l2_table = g_malloc(l2_size);
-
-    if (bdrv_pread(bs->file, l2_offset, l2_table, l2_size) != l2_size)
-        goto fail;
-
-    /* Do the actual checks */
-    for(i = 0; i < s->l2_size; i++) {
-        l2_entry = be64_to_cpu(l2_table[i]);
-
-        switch (qcow2_get_cluster_type(l2_entry)) {
-        case QCOW2_CLUSTER_COMPRESSED:
-            /* Compressed clusters don't have QCOW_OFLAG_COPIED */
-            if (l2_entry & QCOW_OFLAG_COPIED) {
-                fprintf(stderr, "ERROR: cluster %" PRId64 ": "
-                    "copied flag must never be set for compressed "
-                    "clusters\n", l2_entry >> s->cluster_bits);
-                l2_entry &= ~QCOW_OFLAG_COPIED;
-                res->corruptions++;
-            }
-
-            /* Mark cluster as used */
-            nb_csectors = ((l2_entry >> s->csize_shift) &
-                           s->csize_mask) + 1;
-            l2_entry &= s->cluster_offset_mask;
-            inc_refcounts(bs, res, refcount_table, refcount_table_size,
-                l2_entry & ~511, nb_csectors * 512);
-
-            if (flags & CHECK_FRAG_INFO) {
-                res->bfi.allocated_clusters++;
-                res->bfi.compressed_clusters++;
-
-                /* Compressed clusters are fragmented by nature.  Since they
-                 * take up sub-sector space but we only have sector granularity
-                 * I/O we need to re-read the same sectors even for adjacent
-                 * compressed clusters.
-                 */
-                res->bfi.fragmented_clusters++;
-            }
-            break;
-
-        case QCOW2_CLUSTER_ZERO:
-            if ((l2_entry & L2E_OFFSET_MASK) == 0) {
-                break;
-            }
-            /* fall through */
-
-        case QCOW2_CLUSTER_NORMAL:
-        {
-            /* QCOW_OFLAG_COPIED must be set iff refcount == 1 */
-            uint64_t offset = l2_entry & L2E_OFFSET_MASK;
-
-            if (flags & CHECK_OFLAG_COPIED) {
-                refcount = get_refcount(bs, offset >> s->cluster_bits);
-                if (refcount < 0) {
-                    fprintf(stderr, "Can't get refcount for offset %"
-                        PRIx64 ": %s\n", l2_entry, strerror(-refcount));
-                    goto fail;
-                }
-                if ((refcount == 1) != ((l2_entry & QCOW_OFLAG_COPIED) != 0)) {
-                    fprintf(stderr, "ERROR OFLAG_COPIED: offset=%"
-                        PRIx64 " refcount=%d\n", l2_entry, refcount);
-                    res->corruptions++;
-                }
-            }
-
-            if (flags & CHECK_FRAG_INFO) {
-                res->bfi.allocated_clusters++;
-                if (next_contiguous_offset &&
-                    offset != next_contiguous_offset) {
-                    res->bfi.fragmented_clusters++;
-                }
-                next_contiguous_offset = offset + s->cluster_size;
-            }
-
-            /* Mark cluster as used */
-            inc_refcounts(bs, res, refcount_table,refcount_table_size,
-                offset, s->cluster_size);
-
-            /* Correct offsets are cluster aligned */
-            if (offset & (s->cluster_size - 1)) {
-                fprintf(stderr, "ERROR offset=%" PRIx64 ": Cluster is not "
-                    "properly aligned; L2 entry corrupted.\n", offset);
-                res->corruptions++;
-            }
-            break;
-        }
-
-        case QCOW2_CLUSTER_UNALLOCATED:
-            break;
-
-        default:
-            abort();
-        }
-    }
-
-    g_free(l2_table);
-    return 0;
-
-fail:
-    fprintf(stderr, "ERROR: I/O error in check_refcounts_l2\n");
-    g_free(l2_table);
-    return -EIO;
-}
-
-/*
- * Increases the refcount for the L1 table, its L2 tables and all referenced
- * clusters in the given refcount table. While doing so, performs some checks
- * on L1 and L2 entries.
- *
- * Returns the number of errors found by the checks or -errno if an internal
- * error occurred.
- */
-static int check_refcounts_l1(BlockDriverState *bs,
-                              BdrvCheckResult *res,
-                              uint16_t *refcount_table,
-                              int refcount_table_size,
-                              int64_t l1_table_offset, int l1_size,
-                              int flags)
-{
-    BDRVQcowState *s = bs->opaque;
-    uint64_t *l1_table, l2_offset, l1_size2;
-    int i, refcount, ret;
-
-    l1_size2 = l1_size * sizeof(uint64_t);
-
-    /* Mark L1 table as used */
-    inc_refcounts(bs, res, refcount_table, refcount_table_size,
-        l1_table_offset, l1_size2);
-
-    /* Read L1 table entries from disk */
-    if (l1_size2 == 0) {
-        l1_table = NULL;
-    } else {
-        l1_table = g_malloc(l1_size2);
-        if (bdrv_pread(bs->file, l1_table_offset,
-                       l1_table, l1_size2) != l1_size2)
-            goto fail;
-        for(i = 0;i < l1_size; i++)
-            be64_to_cpus(&l1_table[i]);
-    }
-
-    /* Do the actual checks */
-    for(i = 0; i < l1_size; i++) {
-        l2_offset = l1_table[i];
-        if (l2_offset) {
-            /* QCOW_OFLAG_COPIED must be set iff refcount == 1 */
-            if (flags & CHECK_OFLAG_COPIED) {
-                refcount = get_refcount(bs, (l2_offset & ~QCOW_OFLAG_COPIED)
-                    >> s->cluster_bits);
-                if (refcount < 0) {
-                    fprintf(stderr, "Can't get refcount for l2_offset %"
-                        PRIx64 ": %s\n", l2_offset, strerror(-refcount));
-                    goto fail;
-                }
-                if ((refcount == 1) != ((l2_offset & QCOW_OFLAG_COPIED) != 0)) {
-                    fprintf(stderr, "ERROR OFLAG_COPIED: l2_offset=%" PRIx64
-                        " refcount=%d\n", l2_offset, refcount);
-                    res->corruptions++;
-                }
-            }
-
-            /* Mark L2 table as used */
-            l2_offset &= L1E_OFFSET_MASK;
-            inc_refcounts(bs, res, refcount_table, refcount_table_size,
-                l2_offset, s->cluster_size);
-
-            /* L2 tables are cluster aligned */
-            if (l2_offset & (s->cluster_size - 1)) {
-                fprintf(stderr, "ERROR l2_offset=%" PRIx64 ": Table is not "
-                    "cluster aligned; L1 entry corrupted\n", l2_offset);
-                res->corruptions++;
-            }
-
-            /* Process and check L2 entries */
-            ret = check_refcounts_l2(bs, res, refcount_table,
-                                     refcount_table_size, l2_offset, flags);
-            if (ret < 0) {
-                goto fail;
-            }
-        }
-    }
-    g_free(l1_table);
-    return 0;
-
-fail:
-    fprintf(stderr, "ERROR: I/O error in check_refcounts_l1\n");
-    res->check_errors++;
-    g_free(l1_table);
-    return -EIO;
-}
-
-/*
- * Checks an image for refcount consistency.
- *
- * Returns 0 if no errors are found, the number of errors in case the image is
- * detected as corrupted, and -errno when an internal error occurred.
- */
-int qcow2_check_refcounts(BlockDriverState *bs, BdrvCheckResult *res,
-                          BdrvCheckMode fix)
-{
-    BDRVQcowState *s = bs->opaque;
-    int64_t size, i, highest_cluster;
-    int nb_clusters, refcount1, refcount2;
-    QCowSnapshot *sn;
-    uint16_t *refcount_table;
-    int ret;
-
-    size = bdrv_getlength(bs->file);
-    nb_clusters = size_to_clusters(s, size);
-    refcount_table = g_malloc0(nb_clusters * sizeof(uint16_t));
-
-    res->bfi.total_clusters =
-        size_to_clusters(s, bs->total_sectors * BDRV_SECTOR_SIZE);
-
-    /* header */
-    inc_refcounts(bs, res, refcount_table, nb_clusters,
-        0, s->cluster_size);
-
-    /* current L1 table */
-    ret = check_refcounts_l1(bs, res, refcount_table, nb_clusters,
-                             s->l1_table_offset, s->l1_size,
-                             CHECK_OFLAG_COPIED | CHECK_FRAG_INFO);
-    if (ret < 0) {
-        goto fail;
-    }
-
-    /* snapshots */
-    for(i = 0; i < s->nb_snapshots; i++) {
-        sn = s->snapshots + i;
-        ret = check_refcounts_l1(bs, res, refcount_table, nb_clusters,
-            sn->l1_table_offset, sn->l1_size, 0);
-        if (ret < 0) {
-            goto fail;
-        }
-    }
-    inc_refcounts(bs, res, refcount_table, nb_clusters,
-        s->snapshots_offset, s->snapshots_size);
-
-    /* refcount data */
-    inc_refcounts(bs, res, refcount_table, nb_clusters,
-        s->refcount_table_offset,
-        s->refcount_table_size * sizeof(uint64_t));
-
-    for(i = 0; i < s->refcount_table_size; i++) {
-        uint64_t offset, cluster;
-        offset = s->refcount_table[i];
-        cluster = offset >> s->cluster_bits;
-
-        /* Refcount blocks are cluster aligned */
-        if (offset & (s->cluster_size - 1)) {
-            fprintf(stderr, "ERROR refcount block %" PRId64 " is not "
-                "cluster aligned; refcount table entry corrupted\n", i);
-            res->corruptions++;
-            continue;
-        }
-
-        if (cluster >= nb_clusters) {
-            fprintf(stderr, "ERROR refcount block %" PRId64
-                    " is outside image\n", i);
-            res->corruptions++;
-            continue;
-        }
-
-        if (offset != 0) {
-            inc_refcounts(bs, res, refcount_table, nb_clusters,
-                offset, s->cluster_size);
-            if (refcount_table[cluster] != 1) {
-                fprintf(stderr, "ERROR refcount block %" PRId64
-                    " refcount=%d\n",
-                    i, refcount_table[cluster]);
-                res->corruptions++;
-            }
-        }
-    }
-
-    /* compare ref counts */
-    for (i = 0, highest_cluster = 0; i < nb_clusters; i++) {
-        refcount1 = get_refcount(bs, i);
-        if (refcount1 < 0) {
-            fprintf(stderr, "Can't get refcount for cluster %" PRId64 ": %s\n",
-                i, strerror(-refcount1));
-            res->check_errors++;
-            continue;
-        }
-
-        refcount2 = refcount_table[i];
-
-        if (refcount1 > 0 || refcount2 > 0) {
-            highest_cluster = i;
-        }
-
-        if (refcount1 != refcount2) {
-
-            /* Check if we're allowed to fix the mismatch */
-            int *num_fixed = NULL;
-            if (refcount1 > refcount2 && (fix & BDRV_FIX_LEAKS)) {
-                num_fixed = &res->leaks_fixed;
-            } else if (refcount1 < refcount2 && (fix & BDRV_FIX_ERRORS)) {
-                num_fixed = &res->corruptions_fixed;
-            }
-
-            fprintf(stderr, "%s cluster %" PRId64 " refcount=%d reference=%d\n",
-                   num_fixed != NULL     ? "Repairing" :
-                   refcount1 < refcount2 ? "ERROR" :
-                                           "Leaked",
-                   i, refcount1, refcount2);
-
-            if (num_fixed) {
-                ret = update_refcount(bs, i << s->cluster_bits, 1,
-                                      refcount2 - refcount1,
-                                      QCOW2_DISCARD_ALWAYS);
-                if (ret >= 0) {
-                    (*num_fixed)++;
-                    continue;
-                }
-            }
-
-            /* And if we couldn't, print an error */
-            if (refcount1 < refcount2) {
-                res->corruptions++;
-            } else {
-                res->leaks++;
-            }
-        }
-    }
-
-    res->image_end_offset = (highest_cluster + 1) * s->cluster_size;
-    ret = 0;
-
-fail:
-    g_free(refcount_table);
-
-    return ret;
-}
-
diff --git a/contrib/qemu/block/qcow2-snapshot.c b/contrib/qemu/block/qcow2-snapshot.c
deleted file mode 100644
index 0caac9055f8..00000000000
--- a/contrib/qemu/block/qcow2-snapshot.c
+++ /dev/null
@@ -1,660 +0,0 @@
-/*
- * Block driver for the QCOW version 2 format
- *
- * Copyright (c) 2004-2006 Fabrice Bellard
- *
- * Permission is hereby granted, free of charge, to any person obtaining a copy
- * of this software and associated documentation files (the "Software"), to deal
- * in the Software without restriction, including without limitation the rights
- * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
- * copies of the Software, and to permit persons to whom the Software is
- * furnished to do so, subject to the following conditions:
- *
- * The above copyright notice and this permission notice shall be included in
- * all copies or substantial portions of the Software.
- *
- * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
- * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
- * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
- * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
- * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
- * THE SOFTWARE.
- */
-
-#include "qemu-common.h"
-#include "block/block_int.h"
-#include "block/qcow2.h"
-
-typedef struct QEMU_PACKED QCowSnapshotHeader {
-    /* header is 8 byte aligned */
-    uint64_t l1_table_offset;
-
-    uint32_t l1_size;
-    uint16_t id_str_size;
-    uint16_t name_size;
-
-    uint32_t date_sec;
-    uint32_t date_nsec;
-
-    uint64_t vm_clock_nsec;
-
-    uint32_t vm_state_size;
-    uint32_t extra_data_size; /* for extension */
-    /* extra data follows */
-    /* id_str follows */
-    /* name follows  */
-} QCowSnapshotHeader;
-
-typedef struct QEMU_PACKED QCowSnapshotExtraData {
-    uint64_t vm_state_size_large;
-    uint64_t disk_size;
-} QCowSnapshotExtraData;
-
-void qcow2_free_snapshots(BlockDriverState *bs)
-{
-    BDRVQcowState *s = bs->opaque;
-    int i;
-
-    for(i = 0; i < s->nb_snapshots; i++) {
-        g_free(s->snapshots[i].name);
-        g_free(s->snapshots[i].id_str);
-    }
-    g_free(s->snapshots);
-    s->snapshots = NULL;
-    s->nb_snapshots = 0;
-}
-
-int qcow2_read_snapshots(BlockDriverState *bs)
-{
-    BDRVQcowState *s = bs->opaque;
-    QCowSnapshotHeader h;
-    QCowSnapshotExtraData extra;
-    QCowSnapshot *sn;
-    int i, id_str_size, name_size;
-    int64_t offset;
-    uint32_t extra_data_size;
-    int ret;
-
-    if (!s->nb_snapshots) {
-        s->snapshots = NULL;
-        s->snapshots_size = 0;
-        return 0;
-    }
-
-    offset = s->snapshots_offset;
-    s->snapshots = g_malloc0(s->nb_snapshots * sizeof(QCowSnapshot));
-
-    for(i = 0; i < s->nb_snapshots; i++) {
-        /* Read statically sized part of the snapshot header */
-        offset = align_offset(offset, 8);
-        ret = bdrv_pread(bs->file, offset, &h, sizeof(h));
-        if (ret < 0) {
-            goto fail;
-        }
-
-        offset += sizeof(h);
-        sn = s->snapshots + i;
-        sn->l1_table_offset = be64_to_cpu(h.l1_table_offset);
-        sn->l1_size = be32_to_cpu(h.l1_size);
-        sn->vm_state_size = be32_to_cpu(h.vm_state_size);
-        sn->date_sec = be32_to_cpu(h.date_sec);
-        sn->date_nsec = be32_to_cpu(h.date_nsec);
-        sn->vm_clock_nsec = be64_to_cpu(h.vm_clock_nsec);
-        extra_data_size = be32_to_cpu(h.extra_data_size);
-
-        id_str_size = be16_to_cpu(h.id_str_size);
-        name_size = be16_to_cpu(h.name_size);
-
-        /* Read extra data */
-        ret = bdrv_pread(bs->file, offset, &extra,
-                         MIN(sizeof(extra), extra_data_size));
-        if (ret < 0) {
-            goto fail;
-        }
-        offset += extra_data_size;
-
-        if (extra_data_size >= 8) {
-            sn->vm_state_size = be64_to_cpu(extra.vm_state_size_large);
-        }
-
-        if (extra_data_size >= 16) {
-            sn->disk_size = be64_to_cpu(extra.disk_size);
-        } else {
-            sn->disk_size = bs->total_sectors * BDRV_SECTOR_SIZE;
-        }
-
-        /* Read snapshot ID */
-        sn->id_str = g_malloc(id_str_size + 1);
-        ret = bdrv_pread(bs->file, offset, sn->id_str, id_str_size);
-        if (ret < 0) {
-            goto fail;
-        }
-        offset += id_str_size;
-        sn->id_str[id_str_size] = '\0';
-
-        /* Read snapshot name */
-        sn->name = g_malloc(name_size + 1);
-        ret = bdrv_pread(bs->file, offset, sn->name, name_size);
-        if (ret < 0) {
-            goto fail;
-        }
-        offset += name_size;
-        sn->name[name_size] = '\0';
-    }
-
-    s->snapshots_size = offset - s->snapshots_offset;
-    return 0;
-
-fail:
-    qcow2_free_snapshots(bs);
-    return ret;
-}
-
-/* add at the end of the file a new list of snapshots */
-static int qcow2_write_snapshots(BlockDriverState *bs)
-{
-    BDRVQcowState *s = bs->opaque;
-    QCowSnapshot *sn;
-    QCowSnapshotHeader h;
-    QCowSnapshotExtraData extra;
-    int i, name_size, id_str_size, snapshots_size;
-    struct {
-        uint32_t nb_snapshots;
-        uint64_t snapshots_offset;
-    } QEMU_PACKED header_data;
-    int64_t offset, snapshots_offset;
-    int ret;
-
-    /* compute the size of the snapshots */
-    offset = 0;
-    for(i = 0; i < s->nb_snapshots; i++) {
-        sn = s->snapshots + i;
-        offset = align_offset(offset, 8);
-        offset += sizeof(h);
-        offset += sizeof(extra);
-        offset += strlen(sn->id_str);
-        offset += strlen(sn->name);
-    }
-    snapshots_size = offset;
-
-    /* Allocate space for the new snapshot list */
-    snapshots_offset = qcow2_alloc_clusters(bs, snapshots_size);
-    offset = snapshots_offset;
-    if (offset < 0) {
-        return offset;
-    }
-    ret = bdrv_flush(bs);
-    if (ret < 0) {
-        return ret;
-    }
-
-    /* Write all snapshots to the new list */
-    for(i = 0; i < s->nb_snapshots; i++) {
-        sn = s->snapshots + i;
-        memset(&h, 0, sizeof(h));
-        h.l1_table_offset = cpu_to_be64(sn->l1_table_offset);
-        h.l1_size = cpu_to_be32(sn->l1_size);
-        /* If it doesn't fit in 32 bit, older implementations should treat it
-         * as a disk-only snapshot rather than truncate the VM state */
-        if (sn->vm_state_size <= 0xffffffff) {
-            h.vm_state_size = cpu_to_be32(sn->vm_state_size);
-        }
-        h.date_sec = cpu_to_be32(sn->date_sec);
-        h.date_nsec = cpu_to_be32(sn->date_nsec);
-        h.vm_clock_nsec = cpu_to_be64(sn->vm_clock_nsec);
-        h.extra_data_size = cpu_to_be32(sizeof(extra));
-
-        memset(&extra, 0, sizeof(extra));
-        extra.vm_state_size_large = cpu_to_be64(sn->vm_state_size);
-        extra.disk_size = cpu_to_be64(sn->disk_size);
-
-        id_str_size = strlen(sn->id_str);
-        name_size = strlen(sn->name);
-        h.id_str_size = cpu_to_be16(id_str_size);
-        h.name_size = cpu_to_be16(name_size);
-        offset = align_offset(offset, 8);
-
-        ret = bdrv_pwrite(bs->file, offset, &h, sizeof(h));
-        if (ret < 0) {
-            goto fail;
-        }
-        offset += sizeof(h);
-
-        ret = bdrv_pwrite(bs->file, offset, &extra, sizeof(extra));
-        if (ret < 0) {
-            goto fail;
-        }
-        offset += sizeof(extra);
-
-        ret = bdrv_pwrite(bs->file, offset, sn->id_str, id_str_size);
-        if (ret < 0) {
-            goto fail;
-        }
-        offset += id_str_size;
-
-        ret = bdrv_pwrite(bs->file, offset, sn->name, name_size);
-        if (ret < 0) {
-            goto fail;
-        }
-        offset += name_size;
-    }
-
-    /*
-     * Update the header to point to the new snapshot table. This requires the
-     * new table and its refcounts to be stable on disk.
-     */
-    ret = bdrv_flush(bs);
-    if (ret < 0) {
-        goto fail;
-    }
-
-    QEMU_BUILD_BUG_ON(offsetof(QCowHeader, snapshots_offset) !=
-        offsetof(QCowHeader, nb_snapshots) + sizeof(header_data.nb_snapshots));
-
-    header_data.nb_snapshots        = cpu_to_be32(s->nb_snapshots);
-    header_data.snapshots_offset    = cpu_to_be64(snapshots_offset);
-
-    ret = bdrv_pwrite_sync(bs->file, offsetof(QCowHeader, nb_snapshots),
-                           &header_data, sizeof(header_data));
-    if (ret < 0) {
-        goto fail;
-    }
-
-    /* free the old snapshot table */
-    qcow2_free_clusters(bs, s->snapshots_offset, s->snapshots_size,
-                        QCOW2_DISCARD_SNAPSHOT);
-    s->snapshots_offset = snapshots_offset;
-    s->snapshots_size = snapshots_size;
-    return 0;
-
-fail:
-    return ret;
-}
-
-static void find_new_snapshot_id(BlockDriverState *bs,
-                                 char *id_str, int id_str_size)
-{
-    BDRVQcowState *s = bs->opaque;
-    QCowSnapshot *sn;
-    int i, id, id_max = 0;
-
-    for(i = 0; i < s->nb_snapshots; i++) {
-        sn = s->snapshots + i;
-        id = strtoul(sn->id_str, NULL, 10);
-        if (id > id_max)
-            id_max = id;
-    }
-    snprintf(id_str, id_str_size, "%d", id_max + 1);
-}
-
-static int find_snapshot_by_id(BlockDriverState *bs, const char *id_str)
-{
-    BDRVQcowState *s = bs->opaque;
-    int i;
-
-    for(i = 0; i < s->nb_snapshots; i++) {
-        if (!strcmp(s->snapshots[i].id_str, id_str))
-            return i;
-    }
-    return -1;
-}
-
-static int find_snapshot_by_id_or_name(BlockDriverState *bs, const char *name)
-{
-    BDRVQcowState *s = bs->opaque;
-    int i, ret;
-
-    ret = find_snapshot_by_id(bs, name);
-    if (ret >= 0)
-        return ret;
-    for(i = 0; i < s->nb_snapshots; i++) {
-        if (!strcmp(s->snapshots[i].name, name))
-            return i;
-    }
-    return -1;
-}
-
-/* if no id is provided, a new one is constructed */
-int qcow2_snapshot_create(BlockDriverState *bs, QEMUSnapshotInfo *sn_info)
-{
-    BDRVQcowState *s = bs->opaque;
-    QCowSnapshot *new_snapshot_list = NULL;
-    QCowSnapshot *old_snapshot_list = NULL;
-    QCowSnapshot sn1, *sn = &sn1;
-    int i, ret;
-    uint64_t *l1_table = NULL;
-    int64_t l1_table_offset;
-
-    memset(sn, 0, sizeof(*sn));
-
-    /* Generate an ID if it wasn't passed */
-    if (sn_info->id_str[0] == '\0') {
-        find_new_snapshot_id(bs, sn_info->id_str, sizeof(sn_info->id_str));
-    }
-
-    /* Check that the ID is unique */
-    if (find_snapshot_by_id(bs, sn_info->id_str) >= 0) {
-        return -EEXIST;
-    }
-
-    /* Populate sn with passed data */
-    sn->id_str = g_strdup(sn_info->id_str);
-    sn->name = g_strdup(sn_info->name);
-
-    sn->disk_size = bs->total_sectors * BDRV_SECTOR_SIZE;
-    sn->vm_state_size = sn_info->vm_state_size;
-    sn->date_sec = sn_info->date_sec;
-    sn->date_nsec = sn_info->date_nsec;
-    sn->vm_clock_nsec = sn_info->vm_clock_nsec;
-
-    /* Allocate the L1 table of the snapshot and copy the current one there. */
-    l1_table_offset = qcow2_alloc_clusters(bs, s->l1_size * sizeof(uint64_t));
-    if (l1_table_offset < 0) {
-        ret = l1_table_offset;
-        goto fail;
-    }
-
-    sn->l1_table_offset = l1_table_offset;
-    sn->l1_size = s->l1_size;
-
-    l1_table = g_malloc(s->l1_size * sizeof(uint64_t));
-    for(i = 0; i < s->l1_size; i++) {
-        l1_table[i] = cpu_to_be64(s->l1_table[i]);
-    }
-
-    ret = bdrv_pwrite(bs->file, sn->l1_table_offset, l1_table,
-                      s->l1_size * sizeof(uint64_t));
-    if (ret < 0) {
-        goto fail;
-    }
-
-    g_free(l1_table);
-    l1_table = NULL;
-
-    /*
-     * Increase the refcounts of all clusters and make sure everything is
-     * stable on disk before updating the snapshot table to contain a pointer
-     * to the new L1 table.
-     */
-    ret = qcow2_update_snapshot_refcount(bs, s->l1_table_offset, s->l1_size, 1);
-    if (ret < 0) {
-        goto fail;
-    }
-
-    /* Append the new snapshot to the snapshot list */
-    new_snapshot_list = g_malloc((s->nb_snapshots + 1) * sizeof(QCowSnapshot));
-    if (s->snapshots) {
-        memcpy(new_snapshot_list, s->snapshots,
-               s->nb_snapshots * sizeof(QCowSnapshot));
-        old_snapshot_list = s->snapshots;
-    }
-    s->snapshots = new_snapshot_list;
-    s->snapshots[s->nb_snapshots++] = *sn;
-
-    ret = qcow2_write_snapshots(bs);
-    if (ret < 0) {
-        g_free(s->snapshots);
-        s->snapshots = old_snapshot_list;
-        goto fail;
-    }
-
-    g_free(old_snapshot_list);
-
-#ifdef DEBUG_ALLOC
-    {
-      BdrvCheckResult result = {0};
-      qcow2_check_refcounts(bs, &result, 0);
-    }
-#endif
-    return 0;
-
-fail:
-    g_free(sn->id_str);
-    g_free(sn->name);
-    g_free(l1_table);
-
-    return ret;
-}
-
-/* copy the snapshot 'snapshot_name' into the current disk image */
-int qcow2_snapshot_goto(BlockDriverState *bs, const char *snapshot_id)
-{
-    BDRVQcowState *s = bs->opaque;
-    QCowSnapshot *sn;
-    int i, snapshot_index;
-    int cur_l1_bytes, sn_l1_bytes;
-    int ret;
-    uint64_t *sn_l1_table = NULL;
-
-    /* Search the snapshot */
-    snapshot_index = find_snapshot_by_id_or_name(bs, snapshot_id);
-    if (snapshot_index < 0) {
-        return -ENOENT;
-    }
-    sn = &s->snapshots[snapshot_index];
-
-    if (sn->disk_size != bs->total_sectors * BDRV_SECTOR_SIZE) {
-        error_report("qcow2: Loading snapshots with different disk "
-            "size is not implemented");
-        ret = -ENOTSUP;
-        goto fail;
-    }
-
-    /*
-     * Make sure that the current L1 table is big enough to contain the whole
-     * L1 table of the snapshot. If the snapshot L1 table is smaller, the
-     * current one must be padded with zeros.
-     */
-    ret = qcow2_grow_l1_table(bs, sn->l1_size, true);
-    if (ret < 0) {
-        goto fail;
-    }
-
-    cur_l1_bytes = s->l1_size * sizeof(uint64_t);
-    sn_l1_bytes = sn->l1_size * sizeof(uint64_t);
-
-    /*
-     * Copy the snapshot L1 table to the current L1 table.
-     *
-     * Before overwriting the old current L1 table on disk, make sure to
-     * increase all refcounts for the clusters referenced by the new one.
-     * Decrease the refcount referenced by the old one only when the L1
-     * table is overwritten.
-     */
-    sn_l1_table = g_malloc0(cur_l1_bytes);
-
-    ret = bdrv_pread(bs->file, sn->l1_table_offset, sn_l1_table, sn_l1_bytes);
-    if (ret < 0) {
-        goto fail;
-    }
-
-    ret = qcow2_update_snapshot_refcount(bs, sn->l1_table_offset,
-                                         sn->l1_size, 1);
-    if (ret < 0) {
-        goto fail;
-    }
-
-    ret = bdrv_pwrite_sync(bs->file, s->l1_table_offset, sn_l1_table,
-                           cur_l1_bytes);
-    if (ret < 0) {
-        goto fail;
-    }
-
-    /*
-     * Decrease refcount of clusters of current L1 table.
-     *
-     * At this point, the in-memory s->l1_table points to the old L1 table,
-     * whereas on disk we already have the new one.
-     *
-     * qcow2_update_snapshot_refcount special cases the current L1 table to use
-     * the in-memory data instead of really using the offset to load a new one,
-     * which is why this works.
-     */
-    ret = qcow2_update_snapshot_refcount(bs, s->l1_table_offset,
-                                         s->l1_size, -1);
-
-    /*
-     * Now update the in-memory L1 table to be in sync with the on-disk one. We
-     * need to do this even if updating refcounts failed.
-     */
-    for(i = 0;i < s->l1_size; i++) {
-        s->l1_table[i] = be64_to_cpu(sn_l1_table[i]);
-    }
-
-    if (ret < 0) {
-        goto fail;
-    }
-
-    g_free(sn_l1_table);
-    sn_l1_table = NULL;
-
-    /*
-     * Update QCOW_OFLAG_COPIED in the active L1 table (it may have changed
-     * when we decreased the refcount of the old snapshot.
-     */
-    ret = qcow2_update_snapshot_refcount(bs, s->l1_table_offset, s->l1_size, 0);
-    if (ret < 0) {
-        goto fail;
-    }
-
-#ifdef DEBUG_ALLOC
-    {
-        BdrvCheckResult result = {0};
-        qcow2_check_refcounts(bs, &result, 0);
-    }
-#endif
-    return 0;
-
-fail:
-    g_free(sn_l1_table);
-    return ret;
-}
-
-int qcow2_snapshot_delete(BlockDriverState *bs, const char *snapshot_id)
-{
-    BDRVQcowState *s = bs->opaque;
-    QCowSnapshot sn;
-    int snapshot_index, ret;
-
-    /* Search the snapshot */
-    snapshot_index = find_snapshot_by_id_or_name(bs, snapshot_id);
-    if (snapshot_index < 0) {
-        return -ENOENT;
-    }
-    sn = s->snapshots[snapshot_index];
-
-    /* Remove it from the snapshot list */
-    memmove(s->snapshots + snapshot_index,
-            s->snapshots + snapshot_index + 1,
-            (s->nb_snapshots - snapshot_index - 1) * sizeof(sn));
-    s->nb_snapshots--;
-    ret = qcow2_write_snapshots(bs);
-    if (ret < 0) {
-        return ret;
-    }
-
-    /*
-     * The snapshot is now unused, clean up. If we fail after this point, we
-     * won't recover but just leak clusters.
-     */
-    g_free(sn.id_str);
-    g_free(sn.name);
-
-    /*
-     * Now decrease the refcounts of clusters referenced by the snapshot and
-     * free the L1 table.
-     */
-    ret = qcow2_update_snapshot_refcount(bs, sn.l1_table_offset,
-                                         sn.l1_size, -1);
-    if (ret < 0) {
-        return ret;
-    }
-    qcow2_free_clusters(bs, sn.l1_table_offset, sn.l1_size * sizeof(uint64_t),
-                        QCOW2_DISCARD_SNAPSHOT);
-
-    /* must update the copied flag on the current cluster offsets */
-    ret = qcow2_update_snapshot_refcount(bs, s->l1_table_offset, s->l1_size, 0);
-    if (ret < 0) {
-        return ret;
-    }
-
-#ifdef DEBUG_ALLOC
-    {
-        BdrvCheckResult result = {0};
-        qcow2_check_refcounts(bs, &result, 0);
-    }
-#endif
-    return 0;
-}
-
-int qcow2_snapshot_list(BlockDriverState *bs, QEMUSnapshotInfo **psn_tab)
-{
-    BDRVQcowState *s = bs->opaque;
-    QEMUSnapshotInfo *sn_tab, *sn_info;
-    QCowSnapshot *sn;
-    int i;
-
-    if (!s->nb_snapshots) {
-        *psn_tab = NULL;
-        return s->nb_snapshots;
-    }
-
-    sn_tab = g_malloc0(s->nb_snapshots * sizeof(QEMUSnapshotInfo));
-    for(i = 0; i < s->nb_snapshots; i++) {
-        sn_info = sn_tab + i;
-        sn = s->snapshots + i;
-        pstrcpy(sn_info->id_str, sizeof(sn_info->id_str),
-                sn->id_str);
-        pstrcpy(sn_info->name, sizeof(sn_info->name),
-                sn->name);
-        sn_info->vm_state_size = sn->vm_state_size;
-        sn_info->date_sec = sn->date_sec;
-        sn_info->date_nsec = sn->date_nsec;
-        sn_info->vm_clock_nsec = sn->vm_clock_nsec;
-    }
-    *psn_tab = sn_tab;
-    return s->nb_snapshots;
-}
-
-int qcow2_snapshot_load_tmp(BlockDriverState *bs, const char *snapshot_name)
-{
-    int i, snapshot_index;
-    BDRVQcowState *s = bs->opaque;
-    QCowSnapshot *sn;
-    uint64_t *new_l1_table;
-    int new_l1_bytes;
-    int ret;
-
-    assert(bs->read_only);
-
-    /* Search the snapshot */
-    snapshot_index = find_snapshot_by_id_or_name(bs, snapshot_name);
-    if (snapshot_index < 0) {
-        return -ENOENT;
-    }
-    sn = &s->snapshots[snapshot_index];
-
-    /* Allocate and read in the snapshot's L1 table */
-    new_l1_bytes = s->l1_size * sizeof(uint64_t);
-    new_l1_table = g_malloc0(align_offset(new_l1_bytes, 512));
-
-    ret = bdrv_pread(bs->file, sn->l1_table_offset, new_l1_table, new_l1_bytes);
-    if (ret < 0) {
-        g_free(new_l1_table);
-        return ret;
-    }
-
-    /* Switch the L1 table */
-    g_free(s->l1_table);
-
-    s->l1_size = sn->l1_size;
-    s->l1_table_offset = sn->l1_table_offset;
-    s->l1_table = new_l1_table;
-
-    for(i = 0;i < s->l1_size; i++) {
-        be64_to_cpus(&s->l1_table[i]);
-    }
-
-    return 0;
-}
diff --git a/contrib/qemu/block/qcow2.c b/contrib/qemu/block/qcow2.c
deleted file mode 100644
index 0eceefe2cd9..00000000000
--- a/contrib/qemu/block/qcow2.c
+++ /dev/null
@@ -1,1825 +0,0 @@
-/*
- * Block driver for the QCOW version 2 format
- *
- * Copyright (c) 2004-2006 Fabrice Bellard
- *
- * Permission is hereby granted, free of charge, to any person obtaining a copy
- * of this software and associated documentation files (the "Software"), to deal
- * in the Software without restriction, including without limitation the rights
- * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
- * copies of the Software, and to permit persons to whom the Software is
- * furnished to do so, subject to the following conditions:
- *
- * The above copyright notice and this permission notice shall be included in
- * all copies or substantial portions of the Software.
- *
- * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
- * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
- * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
- * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
- * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
- * THE SOFTWARE.
- */
-#include "qemu-common.h"
-#include "block/block_int.h"
-#include "qemu/module.h"
-#include <zlib.h>
-#include "qemu/aes.h"
-#include "block/qcow2.h"
-#include "qemu/error-report.h"
-#include "qapi/qmp/qerror.h"
-#include "qapi/qmp/qbool.h"
-#include "trace.h"
-
-/*
-  Differences with QCOW:
-
-  - Support for multiple incremental snapshots.
-  - Memory management by reference counts.
-  - Clusters which have a reference count of one have the bit
-    QCOW_OFLAG_COPIED to optimize write performance.
-  - Size of compressed clusters is stored in sectors to reduce bit usage
-    in the cluster offsets.
-  - Support for storing additional data (such as the VM state) in the
-    snapshots.
-  - If a backing store is used, the cluster size is not constrained
-    (could be backported to QCOW).
-  - L2 tables have always a size of one cluster.
-*/
-
-
-typedef struct {
-    uint32_t magic;
-    uint32_t len;
-} QCowExtension;
-
-#define  QCOW2_EXT_MAGIC_END 0
-#define  QCOW2_EXT_MAGIC_BACKING_FORMAT 0xE2792ACA
-#define  QCOW2_EXT_MAGIC_FEATURE_TABLE 0x6803f857
-
-static int qcow2_probe(const uint8_t *buf, int buf_size, const char *filename)
-{
-    const QCowHeader *cow_header = (const void *)buf;
-
-    if (buf_size >= sizeof(QCowHeader) &&
-        be32_to_cpu(cow_header->magic) == QCOW_MAGIC &&
-        be32_to_cpu(cow_header->version) >= 2)
-        return 100;
-    else
-        return 0;
-}
-
-
-/* 
- * read qcow2 extension and fill bs
- * start reading from start_offset
- * finish reading upon magic of value 0 or when end_offset reached
- * unknown magic is skipped (future extension this version knows nothing about)
- * return 0 upon success, non-0 otherwise
- */
-static int qcow2_read_extensions(BlockDriverState *bs, uint64_t start_offset,
-                                 uint64_t end_offset, void **p_feature_table)
-{
-    BDRVQcowState *s = bs->opaque;
-    QCowExtension ext;
-    uint64_t offset;
-    int ret;
-
-#ifdef DEBUG_EXT
-    printf("qcow2_read_extensions: start=%ld end=%ld\n", start_offset, end_offset);
-#endif
-    offset = start_offset;
-    while (offset < end_offset) {
-
-#ifdef DEBUG_EXT
-        /* Sanity check */
-        if (offset > s->cluster_size)
-            printf("qcow2_read_extension: suspicious offset %lu\n", offset);
-
-        printf("attempting to read extended header in offset %lu\n", offset);
-#endif
-
-        if (bdrv_pread(bs->file, offset, &ext, sizeof(ext)) != sizeof(ext)) {
-            fprintf(stderr, "qcow2_read_extension: ERROR: "
-                    "pread fail from offset %" PRIu64 "\n",
-                    offset);
-            return 1;
-        }
-        be32_to_cpus(&ext.magic);
-        be32_to_cpus(&ext.len);
-        offset += sizeof(ext);
-#ifdef DEBUG_EXT
-        printf("ext.magic = 0x%x\n", ext.magic);
-#endif
-        if (ext.len > end_offset - offset) {
-            error_report("Header extension too large");
-            return -EINVAL;
-        }
-
-        switch (ext.magic) {
-        case QCOW2_EXT_MAGIC_END:
-            return 0;
-
-        case QCOW2_EXT_MAGIC_BACKING_FORMAT:
-            if (ext.len >= sizeof(bs->backing_format)) {
-                fprintf(stderr, "ERROR: ext_backing_format: len=%u too large"
-                        " (>=%zu)\n",
-                        ext.len, sizeof(bs->backing_format));
-                return 2;
-            }
-            if (bdrv_pread(bs->file, offset , bs->backing_format,
-                           ext.len) != ext.len)
-                return 3;
-            bs->backing_format[ext.len] = '\0';
-#ifdef DEBUG_EXT
-            printf("Qcow2: Got format extension %s\n", bs->backing_format);
-#endif
-            break;
-
-        case QCOW2_EXT_MAGIC_FEATURE_TABLE:
-            if (p_feature_table != NULL) {
-                void* feature_table = g_malloc0(ext.len + 2 * sizeof(Qcow2Feature));
-                ret = bdrv_pread(bs->file, offset , feature_table, ext.len);
-                if (ret < 0) {
-                    return ret;
-                }
-
-                *p_feature_table = feature_table;
-            }
-            break;
-
-        default:
-            /* unknown magic - save it in case we need to rewrite the header */
-            {
-                Qcow2UnknownHeaderExtension *uext;
-
-                uext = g_malloc0(sizeof(*uext)  + ext.len);
-                uext->magic = ext.magic;
-                uext->len = ext.len;
-                QLIST_INSERT_HEAD(&s->unknown_header_ext, uext, next);
-
-                ret = bdrv_pread(bs->file, offset , uext->data, uext->len);
-                if (ret < 0) {
-                    return ret;
-                }
-            }
-            break;
-        }
-
-        offset += ((ext.len + 7) & ~7);
-    }
-
-    return 0;
-}
-
-static void cleanup_unknown_header_ext(BlockDriverState *bs)
-{
-    BDRVQcowState *s = bs->opaque;
-    Qcow2UnknownHeaderExtension *uext, *next;
-
-    QLIST_FOREACH_SAFE(uext, &s->unknown_header_ext, next, next) {
-        QLIST_REMOVE(uext, next);
-        g_free(uext);
-    }
-}
-
-static void GCC_FMT_ATTR(2, 3) report_unsupported(BlockDriverState *bs,
-    const char *fmt, ...)
-{
-    char msg[64];
-    va_list ap;
-
-    va_start(ap, fmt);
-    vsnprintf(msg, sizeof(msg), fmt, ap);
-    va_end(ap);
-
-    qerror_report(QERR_UNKNOWN_BLOCK_FORMAT_FEATURE,
-        bs->device_name, "qcow2", msg);
-}
-
-static void report_unsupported_feature(BlockDriverState *bs,
-    Qcow2Feature *table, uint64_t mask)
-{
-    while (table && table->name[0] != '\0') {
-        if (table->type == QCOW2_FEAT_TYPE_INCOMPATIBLE) {
-            if (mask & (1 << table->bit)) {
-                report_unsupported(bs, "%.46s",table->name);
-                mask &= ~(1 << table->bit);
-            }
-        }
-        table++;
-    }
-
-    if (mask) {
-        report_unsupported(bs, "Unknown incompatible feature: %" PRIx64, mask);
-    }
-}
-
-/*
- * Sets the dirty bit and flushes afterwards if necessary.
- *
- * The incompatible_features bit is only set if the image file header was
- * updated successfully.  Therefore it is not required to check the return
- * value of this function.
- */
-int qcow2_mark_dirty(BlockDriverState *bs)
-{
-    BDRVQcowState *s = bs->opaque;
-    uint64_t val;
-    int ret;
-
-    assert(s->qcow_version >= 3);
-
-    if (s->incompatible_features & QCOW2_INCOMPAT_DIRTY) {
-        return 0; /* already dirty */
-    }
-
-    val = cpu_to_be64(s->incompatible_features | QCOW2_INCOMPAT_DIRTY);
-    ret = bdrv_pwrite(bs->file, offsetof(QCowHeader, incompatible_features),
-                      &val, sizeof(val));
-    if (ret < 0) {
-        return ret;
-    }
-    ret = bdrv_flush(bs->file);
-    if (ret < 0) {
-        return ret;
-    }
-
-    /* Only treat image as dirty if the header was updated successfully */
-    s->incompatible_features |= QCOW2_INCOMPAT_DIRTY;
-    return 0;
-}
-
-/*
- * Clears the dirty bit and flushes before if necessary.  Only call this
- * function when there are no pending requests, it does not guard against
- * concurrent requests dirtying the image.
- */
-static int qcow2_mark_clean(BlockDriverState *bs)
-{
-    BDRVQcowState *s = bs->opaque;
-
-    if (s->incompatible_features & QCOW2_INCOMPAT_DIRTY) {
-        int ret = bdrv_flush(bs);
-        if (ret < 0) {
-            return ret;
-        }
-
-        s->incompatible_features &= ~QCOW2_INCOMPAT_DIRTY;
-        return qcow2_update_header(bs);
-    }
-    return 0;
-}
-
-static int qcow2_check(BlockDriverState *bs, BdrvCheckResult *result,
-                       BdrvCheckMode fix)
-{
-    int ret = qcow2_check_refcounts(bs, result, fix);
-    if (ret < 0) {
-        return ret;
-    }
-
-    if (fix && result->check_errors == 0 && result->corruptions == 0) {
-        return qcow2_mark_clean(bs);
-    }
-    return ret;
-}
-
-static QemuOptsList qcow2_runtime_opts = {
-    .name = "qcow2",
-    .head = QTAILQ_HEAD_INITIALIZER(qcow2_runtime_opts.head),
-    .desc = {
-        {
-            .name = "lazy_refcounts",
-            .type = QEMU_OPT_BOOL,
-            .help = "Postpone refcount updates",
-        },
-        {
-            .name = QCOW2_OPT_DISCARD_REQUEST,
-            .type = QEMU_OPT_BOOL,
-            .help = "Pass guest discard requests to the layer below",
-        },
-        {
-            .name = QCOW2_OPT_DISCARD_SNAPSHOT,
-            .type = QEMU_OPT_BOOL,
-            .help = "Generate discard requests when snapshot related space "
-                    "is freed",
-        },
-        {
-            .name = QCOW2_OPT_DISCARD_OTHER,
-            .type = QEMU_OPT_BOOL,
-            .help = "Generate discard requests when other clusters are freed",
-        },
-        { /* end of list */ }
-    },
-};
-
-static int qcow2_open(BlockDriverState *bs, QDict *options, int flags)
-{
-    BDRVQcowState *s = bs->opaque;
-    int len, i, ret = 0;
-    QCowHeader header;
-    QemuOpts *opts;
-    Error *local_err = NULL;
-    uint64_t ext_end;
-    uint64_t l1_vm_state_index;
-
-    ret = bdrv_pread(bs->file, 0, &header, sizeof(header));
-    if (ret < 0) {
-        goto fail;
-    }
-    be32_to_cpus(&header.magic);
-    be32_to_cpus(&header.version);
-    be64_to_cpus(&header.backing_file_offset);
-    be32_to_cpus(&header.backing_file_size);
-    be64_to_cpus(&header.size);
-    be32_to_cpus(&header.cluster_bits);
-    be32_to_cpus(&header.crypt_method);
-    be64_to_cpus(&header.l1_table_offset);
-    be32_to_cpus(&header.l1_size);
-    be64_to_cpus(&header.refcount_table_offset);
-    be32_to_cpus(&header.refcount_table_clusters);
-    be64_to_cpus(&header.snapshots_offset);
-    be32_to_cpus(&header.nb_snapshots);
-
-    if (header.magic != QCOW_MAGIC) {
-        ret = -EMEDIUMTYPE;
-        goto fail;
-    }
-    if (header.version < 2 || header.version > 3) {
-        report_unsupported(bs, "QCOW version %d", header.version);
-        ret = -ENOTSUP;
-        goto fail;
-    }
-
-    s->qcow_version = header.version;
-
-    /* Initialise version 3 header fields */
-    if (header.version == 2) {
-        header.incompatible_features    = 0;
-        header.compatible_features      = 0;
-        header.autoclear_features       = 0;
-        header.refcount_order           = 4;
-        header.header_length            = 72;
-    } else {
-        be64_to_cpus(&header.incompatible_features);
-        be64_to_cpus(&header.compatible_features);
-        be64_to_cpus(&header.autoclear_features);
-        be32_to_cpus(&header.refcount_order);
-        be32_to_cpus(&header.header_length);
-    }
-
-    if (header.header_length > sizeof(header)) {
-        s->unknown_header_fields_size = header.header_length - sizeof(header);
-        s->unknown_header_fields = g_malloc(s->unknown_header_fields_size);
-        ret = bdrv_pread(bs->file, sizeof(header), s->unknown_header_fields,
-                         s->unknown_header_fields_size);
-        if (ret < 0) {
-            goto fail;
-        }
-    }
-
-    if (header.backing_file_offset) {
-        ext_end = header.backing_file_offset;
-    } else {
-        ext_end = 1 << header.cluster_bits;
-    }
-
-    /* Handle feature bits */
-    s->incompatible_features    = header.incompatible_features;
-    s->compatible_features      = header.compatible_features;
-    s->autoclear_features       = header.autoclear_features;
-
-    if (s->incompatible_features & ~QCOW2_INCOMPAT_MASK) {
-        void *feature_table = NULL;
-        qcow2_read_extensions(bs, header.header_length, ext_end,
-                              &feature_table);
-        report_unsupported_feature(bs, feature_table,
-                                   s->incompatible_features &
-                                   ~QCOW2_INCOMPAT_MASK);
-        ret = -ENOTSUP;
-        goto fail;
-    }
-
-    /* Check support for various header values */
-    if (header.refcount_order != 4) {
-        report_unsupported(bs, "%d bit reference counts",
-                           1 << header.refcount_order);
-        ret = -ENOTSUP;
-        goto fail;
-    }
-
-    if (header.cluster_bits < MIN_CLUSTER_BITS ||
-        header.cluster_bits > MAX_CLUSTER_BITS) {
-        ret = -EINVAL;
-        goto fail;
-    }
-    if (header.crypt_method > QCOW_CRYPT_AES) {
-        ret = -EINVAL;
-        goto fail;
-    }
-    s->crypt_method_header = header.crypt_method;
-    if (s->crypt_method_header) {
-        bs->encrypted = 1;
-    }
-    s->cluster_bits = header.cluster_bits;
-    s->cluster_size = 1 << s->cluster_bits;
-    s->cluster_sectors = 1 << (s->cluster_bits - 9);
-    s->l2_bits = s->cluster_bits - 3; /* L2 is always one cluster */
-    s->l2_size = 1 << s->l2_bits;
-    bs->total_sectors = header.size / 512;
-    s->csize_shift = (62 - (s->cluster_bits - 8));
-    s->csize_mask = (1 << (s->cluster_bits - 8)) - 1;
-    s->cluster_offset_mask = (1LL << s->csize_shift) - 1;
-    s->refcount_table_offset = header.refcount_table_offset;
-    s->refcount_table_size =
-        header.refcount_table_clusters << (s->cluster_bits - 3);
-
-    s->snapshots_offset = header.snapshots_offset;
-    s->nb_snapshots = header.nb_snapshots;
-
-    /* read the level 1 table */
-    s->l1_size = header.l1_size;
-
-    l1_vm_state_index = size_to_l1(s, header.size);
-    if (l1_vm_state_index > INT_MAX) {
-        ret = -EFBIG;
-        goto fail;
-    }
-    s->l1_vm_state_index = l1_vm_state_index;
-
-    /* the L1 table must contain at least enough entries to put
-       header.size bytes */
-    if (s->l1_size < s->l1_vm_state_index) {
-        ret = -EINVAL;
-        goto fail;
-    }
-    s->l1_table_offset = header.l1_table_offset;
-    if (s->l1_size > 0) {
-        s->l1_table = g_malloc0(
-            align_offset(s->l1_size * sizeof(uint64_t), 512));
-        ret = bdrv_pread(bs->file, s->l1_table_offset, s->l1_table,
-                         s->l1_size * sizeof(uint64_t));
-        if (ret < 0) {
-            goto fail;
-        }
-        for(i = 0;i < s->l1_size; i++) {
-            be64_to_cpus(&s->l1_table[i]);
-        }
-    }
-
-    /* alloc L2 table/refcount block cache */
-    s->l2_table_cache = qcow2_cache_create(bs, L2_CACHE_SIZE);
-    s->refcount_block_cache = qcow2_cache_create(bs, REFCOUNT_CACHE_SIZE);
-
-    s->cluster_cache = g_malloc(s->cluster_size);
-    /* one more sector for decompressed data alignment */
-    s->cluster_data = qemu_blockalign(bs, QCOW_MAX_CRYPT_CLUSTERS * s->cluster_size
-                                  + 512);
-    s->cluster_cache_offset = -1;
-    s->flags = flags;
-
-    ret = qcow2_refcount_init(bs);
-    if (ret != 0) {
-        goto fail;
-    }
-
-    QLIST_INIT(&s->cluster_allocs);
-    QTAILQ_INIT(&s->discards);
-
-    /* read qcow2 extensions */
-    if (qcow2_read_extensions(bs, header.header_length, ext_end, NULL)) {
-        ret = -EINVAL;
-        goto fail;
-    }
-
-    /* read the backing file name */
-    if (header.backing_file_offset != 0) {
-        len = header.backing_file_size;
-        if (len > 1023) {
-            len = 1023;
-        }
-        ret = bdrv_pread(bs->file, header.backing_file_offset,
-                         bs->backing_file, len);
-        if (ret < 0) {
-            goto fail;
-        }
-        bs->backing_file[len] = '\0';
-    }
-
-    ret = qcow2_read_snapshots(bs);
-    if (ret < 0) {
-        goto fail;
-    }
-
-    /* Clear unknown autoclear feature bits */
-    if (!bs->read_only && s->autoclear_features != 0) {
-        s->autoclear_features = 0;
-        ret = qcow2_update_header(bs);
-        if (ret < 0) {
-            goto fail;
-        }
-    }
-
-    /* Initialise locks */
-    qemu_co_mutex_init(&s->lock);
-
-    /* Repair image if dirty */
-    if (!(flags & BDRV_O_CHECK) && !bs->read_only &&
-        (s->incompatible_features & QCOW2_INCOMPAT_DIRTY)) {
-        BdrvCheckResult result = {0};
-
-        ret = qcow2_check(bs, &result, BDRV_FIX_ERRORS);
-        if (ret < 0) {
-            goto fail;
-        }
-    }
-
-    /* Enable lazy_refcounts according to image and command line options */
-    opts = qemu_opts_create_nofail(&qcow2_runtime_opts);
-    qemu_opts_absorb_qdict(opts, options, &local_err);
-    if (error_is_set(&local_err)) {
-        qerror_report_err(local_err);
-        error_free(local_err);
-        ret = -EINVAL;
-        goto fail;
-    }
-
-    s->use_lazy_refcounts = qemu_opt_get_bool(opts, QCOW2_OPT_LAZY_REFCOUNTS,
-        (s->compatible_features & QCOW2_COMPAT_LAZY_REFCOUNTS));
-
-    s->discard_passthrough[QCOW2_DISCARD_NEVER] = false;
-    s->discard_passthrough[QCOW2_DISCARD_ALWAYS] = true;
-    s->discard_passthrough[QCOW2_DISCARD_REQUEST] =
-        qemu_opt_get_bool(opts, QCOW2_OPT_DISCARD_REQUEST,
-                          flags & BDRV_O_UNMAP);
-    s->discard_passthrough[QCOW2_DISCARD_SNAPSHOT] =
-        qemu_opt_get_bool(opts, QCOW2_OPT_DISCARD_SNAPSHOT, true);
-    s->discard_passthrough[QCOW2_DISCARD_OTHER] =
-        qemu_opt_get_bool(opts, QCOW2_OPT_DISCARD_OTHER, false);
-
-    qemu_opts_del(opts);
-
-    if (s->use_lazy_refcounts && s->qcow_version < 3) {
-        qerror_report(ERROR_CLASS_GENERIC_ERROR, "Lazy refcounts require "
-            "a qcow2 image with at least qemu 1.1 compatibility level");
-        ret = -EINVAL;
-        goto fail;
-    }
-
-#ifdef DEBUG_ALLOC
-    {
-        BdrvCheckResult result = {0};
-        qcow2_check_refcounts(bs, &result, 0);
-    }
-#endif
-    return ret;
-
- fail:
-    g_free(s->unknown_header_fields);
-    cleanup_unknown_header_ext(bs);
-    qcow2_free_snapshots(bs);
-    qcow2_refcount_close(bs);
-    g_free(s->l1_table);
-    if (s->l2_table_cache) {
-        qcow2_cache_destroy(bs, s->l2_table_cache);
-    }
-    g_free(s->cluster_cache);
-    qemu_vfree(s->cluster_data);
-    return ret;
-}
-
-static int qcow2_set_key(BlockDriverState *bs, const char *key)
-{
-    BDRVQcowState *s = bs->opaque;
-    uint8_t keybuf[16];
-    int len, i;
-
-    memset(keybuf, 0, 16);
-    len = strlen(key);
-    if (len > 16)
-        len = 16;
-    /* XXX: we could compress the chars to 7 bits to increase
-       entropy */
-    for(i = 0;i < len;i++) {
-        keybuf[i] = key[i];
-    }
-    s->crypt_method = s->crypt_method_header;
-
-    if (AES_set_encrypt_key(keybuf, 128, &s->aes_encrypt_key) != 0)
-        return -1;
-    if (AES_set_decrypt_key(keybuf, 128, &s->aes_decrypt_key) != 0)
-        return -1;
-#if 0
-    /* test */
-    {
-        uint8_t in[16];
-        uint8_t out[16];
-        uint8_t tmp[16];
-        for(i=0;i<16;i++)
-            in[i] = i;
-        AES_encrypt(in, tmp, &s->aes_encrypt_key);
-        AES_decrypt(tmp, out, &s->aes_decrypt_key);
-        for(i = 0; i < 16; i++)
-            printf(" %02x", tmp[i]);
-        printf("\n");
-        for(i = 0; i < 16; i++)
-            printf(" %02x", out[i]);
-        printf("\n");
-    }
-#endif
-    return 0;
-}
-
-/* We have nothing to do for QCOW2 reopen, stubs just return
- * success */
-static int qcow2_reopen_prepare(BDRVReopenState *state,
-                                BlockReopenQueue *queue, Error **errp)
-{
-    return 0;
-}
-
-static int coroutine_fn qcow2_co_is_allocated(BlockDriverState *bs,
-        int64_t sector_num, int nb_sectors, int *pnum)
-{
-    BDRVQcowState *s = bs->opaque;
-    uint64_t cluster_offset;
-    int ret;
-
-    *pnum = nb_sectors;
-    /* FIXME We can get errors here, but the bdrv_co_is_allocated interface
-     * can't pass them on today */
-    qemu_co_mutex_lock(&s->lock);
-    ret = qcow2_get_cluster_offset(bs, sector_num << 9, pnum, &cluster_offset);
-    qemu_co_mutex_unlock(&s->lock);
-    if (ret < 0) {
-        *pnum = 0;
-    }
-
-    return (cluster_offset != 0) || (ret == QCOW2_CLUSTER_ZERO);
-}
-
-/* handle reading after the end of the backing file */
-int qcow2_backing_read1(BlockDriverState *bs, QEMUIOVector *qiov,
-                  int64_t sector_num, int nb_sectors)
-{
-    int n1;
-    if ((sector_num + nb_sectors) <= bs->total_sectors)
-        return nb_sectors;
-    if (sector_num >= bs->total_sectors)
-        n1 = 0;
-    else
-        n1 = bs->total_sectors - sector_num;
-
-    qemu_iovec_memset(qiov, 512 * n1, 0, 512 * (nb_sectors - n1));
-
-    return n1;
-}
-
-static coroutine_fn int qcow2_co_readv(BlockDriverState *bs, int64_t sector_num,
-                          int remaining_sectors, QEMUIOVector *qiov)
-{
-    BDRVQcowState *s = bs->opaque;
-    int index_in_cluster, n1;
-    int ret;
-    int cur_nr_sectors; /* number of sectors in current iteration */
-    uint64_t cluster_offset = 0;
-    uint64_t bytes_done = 0;
-    QEMUIOVector hd_qiov;
-    uint8_t *cluster_data = NULL;
-
-    qemu_iovec_init(&hd_qiov, qiov->niov);
-
-    qemu_co_mutex_lock(&s->lock);
-
-    while (remaining_sectors != 0) {
-
-        /* prepare next request */
-        cur_nr_sectors = remaining_sectors;
-        if (s->crypt_method) {
-            cur_nr_sectors = MIN(cur_nr_sectors,
-                QCOW_MAX_CRYPT_CLUSTERS * s->cluster_sectors);
-        }
-
-        ret = qcow2_get_cluster_offset(bs, sector_num << 9,
-            &cur_nr_sectors, &cluster_offset);
-        if (ret < 0) {
-            goto fail;
-        }
-
-        index_in_cluster = sector_num & (s->cluster_sectors - 1);
-
-        qemu_iovec_reset(&hd_qiov);
-        qemu_iovec_concat(&hd_qiov, qiov, bytes_done,
-            cur_nr_sectors * 512);
-
-        switch (ret) {
-        case QCOW2_CLUSTER_UNALLOCATED:
-
-            if (bs->backing_hd) {
-                /* read from the base image */
-                n1 = qcow2_backing_read1(bs->backing_hd, &hd_qiov,
-                    sector_num, cur_nr_sectors);
-                if (n1 > 0) {
-                    BLKDBG_EVENT(bs->file, BLKDBG_READ_BACKING_AIO);
-                    qemu_co_mutex_unlock(&s->lock);
-                    ret = bdrv_co_readv(bs->backing_hd, sector_num,
-                                        n1, &hd_qiov);
-                    qemu_co_mutex_lock(&s->lock);
-                    if (ret < 0) {
-                        goto fail;
-                    }
-                }
-            } else {
-                /* Note: in this case, no need to wait */
-                qemu_iovec_memset(&hd_qiov, 0, 0, 512 * cur_nr_sectors);
-            }
-            break;
-
-        case QCOW2_CLUSTER_ZERO:
-            qemu_iovec_memset(&hd_qiov, 0, 0, 512 * cur_nr_sectors);
-            break;
-
-        case QCOW2_CLUSTER_COMPRESSED:
-            /* add AIO support for compressed blocks ? */
-            ret = qcow2_decompress_cluster(bs, cluster_offset);
-            if (ret < 0) {
-                goto fail;
-            }
-
-            qemu_iovec_from_buf(&hd_qiov, 0,
-                s->cluster_cache + index_in_cluster * 512,
-                512 * cur_nr_sectors);
-            break;
-
-        case QCOW2_CLUSTER_NORMAL:
-            if ((cluster_offset & 511) != 0) {
-                ret = -EIO;
-                goto fail;
-            }
-
-            if (s->crypt_method) {
-                /*
-                 * For encrypted images, read everything into a temporary
-                 * contiguous buffer on which the AES functions can work.
-                 */
-                if (!cluster_data) {
-                    cluster_data =
-                        qemu_blockalign(bs, QCOW_MAX_CRYPT_CLUSTERS * s->cluster_size);
-                }
-
-                assert(cur_nr_sectors <=
-                    QCOW_MAX_CRYPT_CLUSTERS * s->cluster_sectors);
-                qemu_iovec_reset(&hd_qiov);
-                qemu_iovec_add(&hd_qiov, cluster_data,
-                    512 * cur_nr_sectors);
-            }
-
-            BLKDBG_EVENT(bs->file, BLKDBG_READ_AIO);
-            qemu_co_mutex_unlock(&s->lock);
-            ret = bdrv_co_readv(bs->file,
-                                (cluster_offset >> 9) + index_in_cluster,
-                                cur_nr_sectors, &hd_qiov);
-            qemu_co_mutex_lock(&s->lock);
-            if (ret < 0) {
-                goto fail;
-            }
-            if (s->crypt_method) {
-                qcow2_encrypt_sectors(s, sector_num,  cluster_data,
-                    cluster_data, cur_nr_sectors, 0, &s->aes_decrypt_key);
-                qemu_iovec_from_buf(qiov, bytes_done,
-                    cluster_data, 512 * cur_nr_sectors);
-            }
-            break;
-
-        default:
-            g_assert_not_reached();
-            ret = -EIO;
-            goto fail;
-        }
-
-        remaining_sectors -= cur_nr_sectors;
-        sector_num += cur_nr_sectors;
-        bytes_done += cur_nr_sectors * 512;
-    }
-    ret = 0;
-
-fail:
-    qemu_co_mutex_unlock(&s->lock);
-
-    qemu_iovec_destroy(&hd_qiov);
-    qemu_vfree(cluster_data);
-
-    return ret;
-}
-
-static coroutine_fn int qcow2_co_writev(BlockDriverState *bs,
-                           int64_t sector_num,
-                           int remaining_sectors,
-                           QEMUIOVector *qiov)
-{
-    BDRVQcowState *s = bs->opaque;
-    int index_in_cluster;
-    int n_end;
-    int ret;
-    int cur_nr_sectors; /* number of sectors in current iteration */
-    uint64_t cluster_offset;
-    QEMUIOVector hd_qiov;
-    uint64_t bytes_done = 0;
-    uint8_t *cluster_data = NULL;
-    QCowL2Meta *l2meta = NULL;
-
-    trace_qcow2_writev_start_req(qemu_coroutine_self(), sector_num,
-                                 remaining_sectors);
-
-    qemu_iovec_init(&hd_qiov, qiov->niov);
-
-    s->cluster_cache_offset = -1; /* disable compressed cache */
-
-    qemu_co_mutex_lock(&s->lock);
-
-    while (remaining_sectors != 0) {
-
-        l2meta = NULL;
-
-        trace_qcow2_writev_start_part(qemu_coroutine_self());
-        index_in_cluster = sector_num & (s->cluster_sectors - 1);
-        n_end = index_in_cluster + remaining_sectors;
-        if (s->crypt_method &&
-            n_end > QCOW_MAX_CRYPT_CLUSTERS * s->cluster_sectors) {
-            n_end = QCOW_MAX_CRYPT_CLUSTERS * s->cluster_sectors;
-        }
-
-        ret = qcow2_alloc_cluster_offset(bs, sector_num << 9,
-            index_in_cluster, n_end, &cur_nr_sectors, &cluster_offset, &l2meta);
-        if (ret < 0) {
-            goto fail;
-        }
-
-        assert((cluster_offset & 511) == 0);
-
-        qemu_iovec_reset(&hd_qiov);
-        qemu_iovec_concat(&hd_qiov, qiov, bytes_done,
-            cur_nr_sectors * 512);
-
-        if (s->crypt_method) {
-            if (!cluster_data) {
-                cluster_data = qemu_blockalign(bs, QCOW_MAX_CRYPT_CLUSTERS *
-                                                 s->cluster_size);
-            }
-
-            assert(hd_qiov.size <=
-                   QCOW_MAX_CRYPT_CLUSTERS * s->cluster_size);
-            qemu_iovec_to_buf(&hd_qiov, 0, cluster_data, hd_qiov.size);
-
-            qcow2_encrypt_sectors(s, sector_num, cluster_data,
-                cluster_data, cur_nr_sectors, 1, &s->aes_encrypt_key);
-
-            qemu_iovec_reset(&hd_qiov);
-            qemu_iovec_add(&hd_qiov, cluster_data,
-                cur_nr_sectors * 512);
-        }
-
-        qemu_co_mutex_unlock(&s->lock);
-        BLKDBG_EVENT(bs->file, BLKDBG_WRITE_AIO);
-        trace_qcow2_writev_data(qemu_coroutine_self(),
-                                (cluster_offset >> 9) + index_in_cluster);
-        ret = bdrv_co_writev(bs->file,
-                             (cluster_offset >> 9) + index_in_cluster,
-                             cur_nr_sectors, &hd_qiov);
-        qemu_co_mutex_lock(&s->lock);
-        if (ret < 0) {
-            goto fail;
-        }
-
-        while (l2meta != NULL) {
-            QCowL2Meta *next;
-
-            ret = qcow2_alloc_cluster_link_l2(bs, l2meta);
-            if (ret < 0) {
-                goto fail;
-            }
-
-            /* Take the request off the list of running requests */
-            if (l2meta->nb_clusters != 0) {
-                QLIST_REMOVE(l2meta, next_in_flight);
-            }
-
-            qemu_co_queue_restart_all(&l2meta->dependent_requests);
-
-            next = l2meta->next;
-            g_free(l2meta);
-            l2meta = next;
-        }
-
-        remaining_sectors -= cur_nr_sectors;
-        sector_num += cur_nr_sectors;
-        bytes_done += cur_nr_sectors * 512;
-        trace_qcow2_writev_done_part(qemu_coroutine_self(), cur_nr_sectors);
-    }
-    ret = 0;
-
-fail:
-    qemu_co_mutex_unlock(&s->lock);
-
-    while (l2meta != NULL) {
-        QCowL2Meta *next;
-
-        if (l2meta->nb_clusters != 0) {
-            QLIST_REMOVE(l2meta, next_in_flight);
-        }
-        qemu_co_queue_restart_all(&l2meta->dependent_requests);
-
-        next = l2meta->next;
-        g_free(l2meta);
-        l2meta = next;
-    }
-
-    qemu_iovec_destroy(&hd_qiov);
-    qemu_vfree(cluster_data);
-    trace_qcow2_writev_done_req(qemu_coroutine_self(), ret);
-
-    return ret;
-}
-
-static void qcow2_close(BlockDriverState *bs)
-{
-    BDRVQcowState *s = bs->opaque;
-    g_free(s->l1_table);
-
-    qcow2_cache_flush(bs, s->l2_table_cache);
-    qcow2_cache_flush(bs, s->refcount_block_cache);
-
-    qcow2_mark_clean(bs);
-
-    qcow2_cache_destroy(bs, s->l2_table_cache);
-    qcow2_cache_destroy(bs, s->refcount_block_cache);
-
-    g_free(s->unknown_header_fields);
-    cleanup_unknown_header_ext(bs);
-
-    g_free(s->cluster_cache);
-    qemu_vfree(s->cluster_data);
-    qcow2_refcount_close(bs);
-    qcow2_free_snapshots(bs);
-}
-
-static void qcow2_invalidate_cache(BlockDriverState *bs)
-{
-    BDRVQcowState *s = bs->opaque;
-    int flags = s->flags;
-    AES_KEY aes_encrypt_key;
-    AES_KEY aes_decrypt_key;
-    uint32_t crypt_method = 0;
-    QDict *options;
-
-    /*
-     * Backing files are read-only which makes all of their metadata immutable,
-     * that means we don't have to worry about reopening them here.
-     */
-
-    if (s->crypt_method) {
-        crypt_method = s->crypt_method;
-        memcpy(&aes_encrypt_key, &s->aes_encrypt_key, sizeof(aes_encrypt_key));
-        memcpy(&aes_decrypt_key, &s->aes_decrypt_key, sizeof(aes_decrypt_key));
-    }
-
-    qcow2_close(bs);
-
-    options = qdict_new();
-    qdict_put(options, QCOW2_OPT_LAZY_REFCOUNTS,
-              qbool_from_int(s->use_lazy_refcounts));
-
-    memset(s, 0, sizeof(BDRVQcowState));
-    qcow2_open(bs, options, flags);
-
-    QDECREF(options);
-
-    if (crypt_method) {
-        s->crypt_method = crypt_method;
-        memcpy(&s->aes_encrypt_key, &aes_encrypt_key, sizeof(aes_encrypt_key));
-        memcpy(&s->aes_decrypt_key, &aes_decrypt_key, sizeof(aes_decrypt_key));
-    }
-}
-
-static size_t header_ext_add(char *buf, uint32_t magic, const void *s,
-    size_t len, size_t buflen)
-{
-    QCowExtension *ext_backing_fmt = (QCowExtension*) buf;
-    size_t ext_len = sizeof(QCowExtension) + ((len + 7) & ~7);
-
-    if (buflen < ext_len) {
-        return -ENOSPC;
-    }
-
-    *ext_backing_fmt = (QCowExtension) {
-        .magic  = cpu_to_be32(magic),
-        .len    = cpu_to_be32(len),
-    };
-    memcpy(buf + sizeof(QCowExtension), s, len);
-
-    return ext_len;
-}
-
-/*
- * Updates the qcow2 header, including the variable length parts of it, i.e.
- * the backing file name and all extensions. qcow2 was not designed to allow
- * such changes, so if we run out of space (we can only use the first cluster)
- * this function may fail.
- *
- * Returns 0 on success, -errno in error cases.
- */
-int qcow2_update_header(BlockDriverState *bs)
-{
-    BDRVQcowState *s = bs->opaque;
-    QCowHeader *header;
-    char *buf;
-    size_t buflen = s->cluster_size;
-    int ret;
-    uint64_t total_size;
-    uint32_t refcount_table_clusters;
-    size_t header_length;
-    Qcow2UnknownHeaderExtension *uext;
-
-    buf = qemu_blockalign(bs, buflen);
-
-    /* Header structure */
-    header = (QCowHeader*) buf;
-
-    if (buflen < sizeof(*header)) {
-        ret = -ENOSPC;
-        goto fail;
-    }
-
-    header_length = sizeof(*header) + s->unknown_header_fields_size;
-    total_size = bs->total_sectors * BDRV_SECTOR_SIZE;
-    refcount_table_clusters = s->refcount_table_size >> (s->cluster_bits - 3);
-
-    *header = (QCowHeader) {
-        /* Version 2 fields */
-        .magic                  = cpu_to_be32(QCOW_MAGIC),
-        .version                = cpu_to_be32(s->qcow_version),
-        .backing_file_offset    = 0,
-        .backing_file_size      = 0,
-        .cluster_bits           = cpu_to_be32(s->cluster_bits),
-        .size                   = cpu_to_be64(total_size),
-        .crypt_method           = cpu_to_be32(s->crypt_method_header),
-        .l1_size                = cpu_to_be32(s->l1_size),
-        .l1_table_offset        = cpu_to_be64(s->l1_table_offset),
-        .refcount_table_offset  = cpu_to_be64(s->refcount_table_offset),
-        .refcount_table_clusters = cpu_to_be32(refcount_table_clusters),
-        .nb_snapshots           = cpu_to_be32(s->nb_snapshots),
-        .snapshots_offset       = cpu_to_be64(s->snapshots_offset),
-
-        /* Version 3 fields */
-        .incompatible_features  = cpu_to_be64(s->incompatible_features),
-        .compatible_features    = cpu_to_be64(s->compatible_features),
-        .autoclear_features     = cpu_to_be64(s->autoclear_features),
-        .refcount_order         = cpu_to_be32(3 + REFCOUNT_SHIFT),
-        .header_length          = cpu_to_be32(header_length),
-    };
-
-    /* For older versions, write a shorter header */
-    switch (s->qcow_version) {
-    case 2:
-        ret = offsetof(QCowHeader, incompatible_features);
-        break;
-    case 3:
-        ret = sizeof(*header);
-        break;
-    default:
-        ret = -EINVAL;
-        goto fail;
-    }
-
-    buf += ret;
-    buflen -= ret;
-    memset(buf, 0, buflen);
-
-    /* Preserve any unknown field in the header */
-    if (s->unknown_header_fields_size) {
-        if (buflen < s->unknown_header_fields_size) {
-            ret = -ENOSPC;
-            goto fail;
-        }
-
-        memcpy(buf, s->unknown_header_fields, s->unknown_header_fields_size);
-        buf += s->unknown_header_fields_size;
-        buflen -= s->unknown_header_fields_size;
-    }
-
-    /* Backing file format header extension */
-    if (*bs->backing_format) {
-        ret = header_ext_add(buf, QCOW2_EXT_MAGIC_BACKING_FORMAT,
-                             bs->backing_format, strlen(bs->backing_format),
-                             buflen);
-        if (ret < 0) {
-            goto fail;
-        }
-
-        buf += ret;
-        buflen -= ret;
-    }
-
-    /* Feature table */
-    Qcow2Feature features[] = {
-        {
-            .type = QCOW2_FEAT_TYPE_INCOMPATIBLE,
-            .bit  = QCOW2_INCOMPAT_DIRTY_BITNR,
-            .name = "dirty bit",
-        },
-        {
-            .type = QCOW2_FEAT_TYPE_COMPATIBLE,
-            .bit  = QCOW2_COMPAT_LAZY_REFCOUNTS_BITNR,
-            .name = "lazy refcounts",
-        },
-    };
-
-    ret = header_ext_add(buf, QCOW2_EXT_MAGIC_FEATURE_TABLE,
-                         features, sizeof(features), buflen);
-    if (ret < 0) {
-        goto fail;
-    }
-    buf += ret;
-    buflen -= ret;
-
-    /* Keep unknown header extensions */
-    QLIST_FOREACH(uext, &s->unknown_header_ext, next) {
-        ret = header_ext_add(buf, uext->magic, uext->data, uext->len, buflen);
-        if (ret < 0) {
-            goto fail;
-        }
-
-        buf += ret;
-        buflen -= ret;
-    }
-
-    /* End of header extensions */
-    ret = header_ext_add(buf, QCOW2_EXT_MAGIC_END, NULL, 0, buflen);
-    if (ret < 0) {
-        goto fail;
-    }
-
-    buf += ret;
-    buflen -= ret;
-
-    /* Backing file name */
-    if (*bs->backing_file) {
-        size_t backing_file_len = strlen(bs->backing_file);
-
-        if (buflen < backing_file_len) {
-            ret = -ENOSPC;
-            goto fail;
-        }
-
-        /* Using strncpy is ok here, since buf is not NUL-terminated. */
-        strncpy(buf, bs->backing_file, buflen);
-
-        header->backing_file_offset = cpu_to_be64(buf - ((char*) header));
-        header->backing_file_size   = cpu_to_be32(backing_file_len);
-    }
-
-    /* Write the new header */
-    ret = bdrv_pwrite(bs->file, 0, header, s->cluster_size);
-    if (ret < 0) {
-        goto fail;
-    }
-
-    ret = 0;
-fail:
-    qemu_vfree(header);
-    return ret;
-}
-
-static int qcow2_change_backing_file(BlockDriverState *bs,
-    const char *backing_file, const char *backing_fmt)
-{
-    pstrcpy(bs->backing_file, sizeof(bs->backing_file), backing_file ?: "");
-    pstrcpy(bs->backing_format, sizeof(bs->backing_format), backing_fmt ?: "");
-
-    return qcow2_update_header(bs);
-}
-
-static int preallocate(BlockDriverState *bs)
-{
-    uint64_t nb_sectors;
-    uint64_t offset;
-    uint64_t host_offset = 0;
-    int num;
-    int ret;
-    QCowL2Meta *meta;
-
-    nb_sectors = bdrv_getlength(bs) >> 9;
-    offset = 0;
-
-    while (nb_sectors) {
-        num = MIN(nb_sectors, INT_MAX >> 9);
-        ret = qcow2_alloc_cluster_offset(bs, offset, 0, num, &num,
-                                         &host_offset, &meta);
-        if (ret < 0) {
-            return ret;
-        }
-
-        ret = qcow2_alloc_cluster_link_l2(bs, meta);
-        if (ret < 0) {
-            qcow2_free_any_clusters(bs, meta->alloc_offset, meta->nb_clusters,
-                                    QCOW2_DISCARD_NEVER);
-            return ret;
-        }
-
-        /* There are no dependent requests, but we need to remove our request
-         * from the list of in-flight requests */
-        if (meta != NULL) {
-            QLIST_REMOVE(meta, next_in_flight);
-        }
-
-        /* TODO Preallocate data if requested */
-
-        nb_sectors -= num;
-        offset += num << 9;
-    }
-
-    /*
-     * It is expected that the image file is large enough to actually contain
-     * all of the allocated clusters (otherwise we get failing reads after
-     * EOF). Extend the image to the last allocated sector.
-     */
-    if (host_offset != 0) {
-        uint8_t buf[512];
-        memset(buf, 0, 512);
-        ret = bdrv_write(bs->file, (host_offset >> 9) + num - 1, buf, 1);
-        if (ret < 0) {
-            return ret;
-        }
-    }
-
-    return 0;
-}
-
-static int qcow2_create2(const char *filename, int64_t total_size,
-                         const char *backing_file, const char *backing_format,
-                         int flags, size_t cluster_size, int prealloc,
-                         QEMUOptionParameter *options, int version)
-{
-    /* Calculate cluster_bits */
-    int cluster_bits;
-    cluster_bits = ffs(cluster_size) - 1;
-    if (cluster_bits < MIN_CLUSTER_BITS || cluster_bits > MAX_CLUSTER_BITS ||
-        (1 << cluster_bits) != cluster_size)
-    {
-        error_report(
-            "Cluster size must be a power of two between %d and %dk",
-            1 << MIN_CLUSTER_BITS, 1 << (MAX_CLUSTER_BITS - 10));
-        return -EINVAL;
-    }
-
-    /*
-     * Open the image file and write a minimal qcow2 header.
-     *
-     * We keep things simple and start with a zero-sized image. We also
-     * do without refcount blocks or a L1 table for now. We'll fix the
-     * inconsistency later.
-     *
-     * We do need a refcount table because growing the refcount table means
-     * allocating two new refcount blocks - the seconds of which would be at
-     * 2 GB for 64k clusters, and we don't want to have a 2 GB initial file
-     * size for any qcow2 image.
-     */
-    BlockDriverState* bs;
-    QCowHeader header;
-    uint8_t* refcount_table;
-    int ret;
-
-    ret = bdrv_create_file(filename, options);
-    if (ret < 0) {
-        return ret;
-    }
-
-    ret = bdrv_file_open(&bs, filename, NULL, BDRV_O_RDWR);
-    if (ret < 0) {
-        return ret;
-    }
-
-    /* Write the header */
-    memset(&header, 0, sizeof(header));
-    header.magic = cpu_to_be32(QCOW_MAGIC);
-    header.version = cpu_to_be32(version);
-    header.cluster_bits = cpu_to_be32(cluster_bits);
-    header.size = cpu_to_be64(0);
-    header.l1_table_offset = cpu_to_be64(0);
-    header.l1_size = cpu_to_be32(0);
-    header.refcount_table_offset = cpu_to_be64(cluster_size);
-    header.refcount_table_clusters = cpu_to_be32(1);
-    header.refcount_order = cpu_to_be32(3 + REFCOUNT_SHIFT);
-    header.header_length = cpu_to_be32(sizeof(header));
-
-    if (flags & BLOCK_FLAG_ENCRYPT) {
-        header.crypt_method = cpu_to_be32(QCOW_CRYPT_AES);
-    } else {
-        header.crypt_method = cpu_to_be32(QCOW_CRYPT_NONE);
-    }
-
-    if (flags & BLOCK_FLAG_LAZY_REFCOUNTS) {
-        header.compatible_features |=
-            cpu_to_be64(QCOW2_COMPAT_LAZY_REFCOUNTS);
-    }
-
-    ret = bdrv_pwrite(bs, 0, &header, sizeof(header));
-    if (ret < 0) {
-        goto out;
-    }
-
-    /* Write an empty refcount table */
-    refcount_table = g_malloc0(cluster_size);
-    ret = bdrv_pwrite(bs, cluster_size, refcount_table, cluster_size);
-    g_free(refcount_table);
-
-    if (ret < 0) {
-        goto out;
-    }
-
-    bdrv_close(bs);
-
-    /*
-     * And now open the image and make it consistent first (i.e. increase the
-     * refcount of the cluster that is occupied by the header and the refcount
-     * table)
-     */
-    BlockDriver* drv = bdrv_find_format("qcow2");
-    assert(drv != NULL);
-    ret = bdrv_open(bs, filename, NULL,
-        BDRV_O_RDWR | BDRV_O_CACHE_WB | BDRV_O_NO_FLUSH, drv);
-    if (ret < 0) {
-        goto out;
-    }
-
-    ret = qcow2_alloc_clusters(bs, 2 * cluster_size);
-    if (ret < 0) {
-        goto out;
-
-    } else if (ret != 0) {
-        error_report("Huh, first cluster in empty image is already in use?");
-        abort();
-    }
-
-    /* Okay, now that we have a valid image, let's give it the right size */
-    ret = bdrv_truncate(bs, total_size * BDRV_SECTOR_SIZE);
-    if (ret < 0) {
-        goto out;
-    }
-
-    /* Want a backing file? There you go.*/
-    if (backing_file) {
-        ret = bdrv_change_backing_file(bs, backing_file, backing_format);
-        if (ret < 0) {
-            goto out;
-        }
-    }
-
-    /* And if we're supposed to preallocate metadata, do that now */
-    if (prealloc) {
-        BDRVQcowState *s = bs->opaque;
-        qemu_co_mutex_lock(&s->lock);
-        ret = preallocate(bs);
-        qemu_co_mutex_unlock(&s->lock);
-        if (ret < 0) {
-            goto out;
-        }
-    }
-
-    ret = 0;
-out:
-    bdrv_delete(bs);
-    return ret;
-}
-
-static int qcow2_create(const char *filename, QEMUOptionParameter *options)
-{
-    const char *backing_file = NULL;
-    const char *backing_fmt = NULL;
-    uint64_t sectors = 0;
-    int flags = 0;
-    size_t cluster_size = DEFAULT_CLUSTER_SIZE;
-    int prealloc = 0;
-    int version = 2;
-
-    /* Read out options */
-    while (options && options->name) {
-        if (!strcmp(options->name, BLOCK_OPT_SIZE)) {
-            sectors = options->value.n / 512;
-        } else if (!strcmp(options->name, BLOCK_OPT_BACKING_FILE)) {
-            backing_file = options->value.s;
-        } else if (!strcmp(options->name, BLOCK_OPT_BACKING_FMT)) {
-            backing_fmt = options->value.s;
-        } else if (!strcmp(options->name, BLOCK_OPT_ENCRYPT)) {
-            flags |= options->value.n ? BLOCK_FLAG_ENCRYPT : 0;
-        } else if (!strcmp(options->name, BLOCK_OPT_CLUSTER_SIZE)) {
-            if (options->value.n) {
-                cluster_size = options->value.n;
-            }
-        } else if (!strcmp(options->name, BLOCK_OPT_PREALLOC)) {
-            if (!options->value.s || !strcmp(options->value.s, "off")) {
-                prealloc = 0;
-            } else if (!strcmp(options->value.s, "metadata")) {
-                prealloc = 1;
-            } else {
-                fprintf(stderr, "Invalid preallocation mode: '%s'\n",
-                    options->value.s);
-                return -EINVAL;
-            }
-        } else if (!strcmp(options->name, BLOCK_OPT_COMPAT_LEVEL)) {
-            if (!options->value.s || !strcmp(options->value.s, "0.10")) {
-                version = 2;
-            } else if (!strcmp(options->value.s, "1.1")) {
-                version = 3;
-            } else {
-                fprintf(stderr, "Invalid compatibility level: '%s'\n",
-                    options->value.s);
-                return -EINVAL;
-            }
-        } else if (!strcmp(options->name, BLOCK_OPT_LAZY_REFCOUNTS)) {
-            flags |= options->value.n ? BLOCK_FLAG_LAZY_REFCOUNTS : 0;
-        }
-        options++;
-    }
-
-    if (backing_file && prealloc) {
-        fprintf(stderr, "Backing file and preallocation cannot be used at "
-            "the same time\n");
-        return -EINVAL;
-    }
-
-    if (version < 3 && (flags & BLOCK_FLAG_LAZY_REFCOUNTS)) {
-        fprintf(stderr, "Lazy refcounts only supported with compatibility "
-                "level 1.1 and above (use compat=1.1 or greater)\n");
-        return -EINVAL;
-    }
-
-    return qcow2_create2(filename, sectors, backing_file, backing_fmt, flags,
-                         cluster_size, prealloc, options, version);
-}
-
-static int qcow2_make_empty(BlockDriverState *bs)
-{
-#if 0
-    /* XXX: not correct */
-    BDRVQcowState *s = bs->opaque;
-    uint32_t l1_length = s->l1_size * sizeof(uint64_t);
-    int ret;
-
-    memset(s->l1_table, 0, l1_length);
-    if (bdrv_pwrite(bs->file, s->l1_table_offset, s->l1_table, l1_length) < 0)
-        return -1;
-    ret = bdrv_truncate(bs->file, s->l1_table_offset + l1_length);
-    if (ret < 0)
-        return ret;
-
-    l2_cache_reset(bs);
-#endif
-    return 0;
-}
-
-static coroutine_fn int qcow2_co_write_zeroes(BlockDriverState *bs,
-    int64_t sector_num, int nb_sectors)
-{
-    int ret;
-    BDRVQcowState *s = bs->opaque;
-
-    /* Emulate misaligned zero writes */
-    if (sector_num % s->cluster_sectors || nb_sectors % s->cluster_sectors) {
-        return -ENOTSUP;
-    }
-
-    /* Whatever is left can use real zero clusters */
-    qemu_co_mutex_lock(&s->lock);
-    ret = qcow2_zero_clusters(bs, sector_num << BDRV_SECTOR_BITS,
-        nb_sectors);
-    qemu_co_mutex_unlock(&s->lock);
-
-    return ret;
-}
-
-static coroutine_fn int qcow2_co_discard(BlockDriverState *bs,
-    int64_t sector_num, int nb_sectors)
-{
-    int ret;
-    BDRVQcowState *s = bs->opaque;
-
-    qemu_co_mutex_lock(&s->lock);
-    ret = qcow2_discard_clusters(bs, sector_num << BDRV_SECTOR_BITS,
-        nb_sectors);
-    qemu_co_mutex_unlock(&s->lock);
-    return ret;
-}
-
-static int qcow2_truncate(BlockDriverState *bs, int64_t offset)
-{
-    BDRVQcowState *s = bs->opaque;
-    int64_t new_l1_size;
-    int ret;
-
-    if (offset & 511) {
-        error_report("The new size must be a multiple of 512");
-        return -EINVAL;
-    }
-
-    /* cannot proceed if image has snapshots */
-    if (s->nb_snapshots) {
-        error_report("Can't resize an image which has snapshots");
-        return -ENOTSUP;
-    }
-
-    /* shrinking is currently not supported */
-    if (offset < bs->total_sectors * 512) {
-        error_report("qcow2 doesn't support shrinking images yet");
-        return -ENOTSUP;
-    }
-
-    new_l1_size = size_to_l1(s, offset);
-    ret = qcow2_grow_l1_table(bs, new_l1_size, true);
-    if (ret < 0) {
-        return ret;
-    }
-
-    /* write updated header.size */
-    offset = cpu_to_be64(offset);
-    ret = bdrv_pwrite_sync(bs->file, offsetof(QCowHeader, size),
-                           &offset, sizeof(uint64_t));
-    if (ret < 0) {
-        return ret;
-    }
-
-    s->l1_vm_state_index = new_l1_size;
-    return 0;
-}
-
-/* XXX: put compressed sectors first, then all the cluster aligned
-   tables to avoid losing bytes in alignment */
-static int qcow2_write_compressed(BlockDriverState *bs, int64_t sector_num,
-                                  const uint8_t *buf, int nb_sectors)
-{
-    BDRVQcowState *s = bs->opaque;
-    z_stream strm;
-    int ret, out_len;
-    uint8_t *out_buf;
-    uint64_t cluster_offset;
-
-    if (nb_sectors == 0) {
-        /* align end of file to a sector boundary to ease reading with
-           sector based I/Os */
-        cluster_offset = bdrv_getlength(bs->file);
-        cluster_offset = (cluster_offset + 511) & ~511;
-        bdrv_truncate(bs->file, cluster_offset);
-        return 0;
-    }
-
-    if (nb_sectors != s->cluster_sectors) {
-        ret = -EINVAL;
-
-        /* Zero-pad last write if image size is not cluster aligned */
-        if (sector_num + nb_sectors == bs->total_sectors &&
-            nb_sectors < s->cluster_sectors) {
-            uint8_t *pad_buf = qemu_blockalign(bs, s->cluster_size);
-            memset(pad_buf, 0, s->cluster_size);
-            memcpy(pad_buf, buf, nb_sectors * BDRV_SECTOR_SIZE);
-            ret = qcow2_write_compressed(bs, sector_num,
-                                         pad_buf, s->cluster_sectors);
-            qemu_vfree(pad_buf);
-        }
-        return ret;
-    }
-
-    out_buf = g_malloc(s->cluster_size + (s->cluster_size / 1000) + 128);
-
-    /* best compression, small window, no zlib header */
-    memset(&strm, 0, sizeof(strm));
-    ret = deflateInit2(&strm, Z_DEFAULT_COMPRESSION,
-                       Z_DEFLATED, -12,
-                       9, Z_DEFAULT_STRATEGY);
-    if (ret != 0) {
-        ret = -EINVAL;
-        goto fail;
-    }
-
-    strm.avail_in = s->cluster_size;
-    strm.next_in = (uint8_t *)buf;
-    strm.avail_out = s->cluster_size;
-    strm.next_out = out_buf;
-
-    ret = deflate(&strm, Z_FINISH);
-    if (ret != Z_STREAM_END && ret != Z_OK) {
-        deflateEnd(&strm);
-        ret = -EINVAL;
-        goto fail;
-    }
-    out_len = strm.next_out - out_buf;
-
-    deflateEnd(&strm);
-
-    if (ret != Z_STREAM_END || out_len >= s->cluster_size) {
-        /* could not compress: write normal cluster */
-        ret = bdrv_write(bs, sector_num, buf, s->cluster_sectors);
-        if (ret < 0) {
-            goto fail;
-        }
-    } else {
-        cluster_offset = qcow2_alloc_compressed_cluster_offset(bs,
-            sector_num << 9, out_len);
-        if (!cluster_offset) {
-            ret = -EIO;
-            goto fail;
-        }
-        cluster_offset &= s->cluster_offset_mask;
-        BLKDBG_EVENT(bs->file, BLKDBG_WRITE_COMPRESSED);
-        ret = bdrv_pwrite(bs->file, cluster_offset, out_buf, out_len);
-        if (ret < 0) {
-            goto fail;
-        }
-    }
-
-    ret = 0;
-fail:
-    g_free(out_buf);
-    return ret;
-}
-
-static coroutine_fn int qcow2_co_flush_to_os(BlockDriverState *bs)
-{
-    BDRVQcowState *s = bs->opaque;
-    int ret;
-
-    qemu_co_mutex_lock(&s->lock);
-    ret = qcow2_cache_flush(bs, s->l2_table_cache);
-    if (ret < 0) {
-        qemu_co_mutex_unlock(&s->lock);
-        return ret;
-    }
-
-    if (qcow2_need_accurate_refcounts(s)) {
-        ret = qcow2_cache_flush(bs, s->refcount_block_cache);
-        if (ret < 0) {
-            qemu_co_mutex_unlock(&s->lock);
-            return ret;
-        }
-    }
-    qemu_co_mutex_unlock(&s->lock);
-
-    return 0;
-}
-
-static int64_t qcow2_vm_state_offset(BDRVQcowState *s)
-{
-	return (int64_t)s->l1_vm_state_index << (s->cluster_bits + s->l2_bits);
-}
-
-static int qcow2_get_info(BlockDriverState *bs, BlockDriverInfo *bdi)
-{
-    BDRVQcowState *s = bs->opaque;
-    bdi->cluster_size = s->cluster_size;
-    bdi->vm_state_offset = qcow2_vm_state_offset(s);
-    return 0;
-}
-
-#if 0
-static void dump_refcounts(BlockDriverState *bs)
-{
-    BDRVQcowState *s = bs->opaque;
-    int64_t nb_clusters, k, k1, size;
-    int refcount;
-
-    size = bdrv_getlength(bs->file);
-    nb_clusters = size_to_clusters(s, size);
-    for(k = 0; k < nb_clusters;) {
-        k1 = k;
-        refcount = get_refcount(bs, k);
-        k++;
-        while (k < nb_clusters && get_refcount(bs, k) == refcount)
-            k++;
-        printf("%" PRId64 ": refcount=%d nb=%" PRId64 "\n", k, refcount,
-               k - k1);
-    }
-}
-#endif
-
-static int qcow2_save_vmstate(BlockDriverState *bs, QEMUIOVector *qiov,
-                              int64_t pos)
-{
-    BDRVQcowState *s = bs->opaque;
-    int growable = bs->growable;
-    int ret;
-
-    BLKDBG_EVENT(bs->file, BLKDBG_VMSTATE_SAVE);
-    bs->growable = 1;
-    ret = bdrv_pwritev(bs, qcow2_vm_state_offset(s) + pos, qiov);
-    bs->growable = growable;
-
-    return ret;
-}
-
-static int qcow2_load_vmstate(BlockDriverState *bs, uint8_t *buf,
-                              int64_t pos, int size)
-{
-    BDRVQcowState *s = bs->opaque;
-    int growable = bs->growable;
-    int ret;
-
-    BLKDBG_EVENT(bs->file, BLKDBG_VMSTATE_LOAD);
-    bs->growable = 1;
-    ret = bdrv_pread(bs, qcow2_vm_state_offset(s) + pos, buf, size);
-    bs->growable = growable;
-
-    return ret;
-}
-
-static QEMUOptionParameter qcow2_create_options[] = {
-    {
-        .name = BLOCK_OPT_SIZE,
-        .type = OPT_SIZE,
-        .help = "Virtual disk size"
-    },
-    {
-        .name = BLOCK_OPT_COMPAT_LEVEL,
-        .type = OPT_STRING,
-        .help = "Compatibility level (0.10 or 1.1)"
-    },
-    {
-        .name = BLOCK_OPT_BACKING_FILE,
-        .type = OPT_STRING,
-        .help = "File name of a base image"
-    },
-    {
-        .name = BLOCK_OPT_BACKING_FMT,
-        .type = OPT_STRING,
-        .help = "Image format of the base image"
-    },
-    {
-        .name = BLOCK_OPT_ENCRYPT,
-        .type = OPT_FLAG,
-        .help = "Encrypt the image"
-    },
-    {
-        .name = BLOCK_OPT_CLUSTER_SIZE,
-        .type = OPT_SIZE,
-        .help = "qcow2 cluster size",
-        .value = { .n = DEFAULT_CLUSTER_SIZE },
-    },
-    {
-        .name = BLOCK_OPT_PREALLOC,
-        .type = OPT_STRING,
-        .help = "Preallocation mode (allowed values: off, metadata)"
-    },
-    {
-        .name = BLOCK_OPT_LAZY_REFCOUNTS,
-        .type = OPT_FLAG,
-        .help = "Postpone refcount updates",
-    },
-    { NULL }
-};
-
-static BlockDriver bdrv_qcow2 = {
-    .format_name        = "qcow2",
-    .instance_size      = sizeof(BDRVQcowState),
-    .bdrv_probe         = qcow2_probe,
-    .bdrv_open          = qcow2_open,
-    .bdrv_close         = qcow2_close,
-    .bdrv_reopen_prepare  = qcow2_reopen_prepare,
-    .bdrv_create        = qcow2_create,
-    .bdrv_has_zero_init = bdrv_has_zero_init_1,
-    .bdrv_co_is_allocated = qcow2_co_is_allocated,
-    .bdrv_set_key       = qcow2_set_key,
-    .bdrv_make_empty    = qcow2_make_empty,
-
-    .bdrv_co_readv          = qcow2_co_readv,
-    .bdrv_co_writev         = qcow2_co_writev,
-    .bdrv_co_flush_to_os    = qcow2_co_flush_to_os,
-
-    .bdrv_co_write_zeroes   = qcow2_co_write_zeroes,
-    .bdrv_co_discard        = qcow2_co_discard,
-    .bdrv_truncate          = qcow2_truncate,
-    .bdrv_write_compressed  = qcow2_write_compressed,
-
-    .bdrv_snapshot_create   = qcow2_snapshot_create,
-    .bdrv_snapshot_goto     = qcow2_snapshot_goto,
-    .bdrv_snapshot_delete   = qcow2_snapshot_delete,
-    .bdrv_snapshot_list     = qcow2_snapshot_list,
-    .bdrv_snapshot_load_tmp     = qcow2_snapshot_load_tmp,
-    .bdrv_get_info      = qcow2_get_info,
-
-    .bdrv_save_vmstate    = qcow2_save_vmstate,
-    .bdrv_load_vmstate    = qcow2_load_vmstate,
-
-    .bdrv_change_backing_file   = qcow2_change_backing_file,
-
-    .bdrv_invalidate_cache      = qcow2_invalidate_cache,
-
-    .create_options = qcow2_create_options,
-    .bdrv_check = qcow2_check,
-};
-
-static void bdrv_qcow2_init(void)
-{
-    bdrv_register(&bdrv_qcow2);
-}
-
-block_init(bdrv_qcow2_init);
diff --git a/contrib/qemu/block/qcow2.h b/contrib/qemu/block/qcow2.h
deleted file mode 100644
index 3b2d5cda71f..00000000000
--- a/contrib/qemu/block/qcow2.h
+++ /dev/null
@@ -1,437 +0,0 @@
-/*
- * Block driver for the QCOW version 2 format
- *
- * Copyright (c) 2004-2006 Fabrice Bellard
- *
- * Permission is hereby granted, free of charge, to any person obtaining a copy
- * of this software and associated documentation files (the "Software"), to deal
- * in the Software without restriction, including without limitation the rights
- * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
- * copies of the Software, and to permit persons to whom the Software is
- * furnished to do so, subject to the following conditions:
- *
- * The above copyright notice and this permission notice shall be included in
- * all copies or substantial portions of the Software.
- *
- * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
- * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
- * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
- * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
- * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
- * THE SOFTWARE.
- */
-
-#ifndef BLOCK_QCOW2_H
-#define BLOCK_QCOW2_H
-
-#include "qemu/aes.h"
-#include "block/coroutine.h"
-
-//#define DEBUG_ALLOC
-//#define DEBUG_ALLOC2
-//#define DEBUG_EXT
-
-#define QCOW_MAGIC (('Q' << 24) | ('F' << 16) | ('I' << 8) | 0xfb)
-
-#define QCOW_CRYPT_NONE 0
-#define QCOW_CRYPT_AES  1
-
-#define QCOW_MAX_CRYPT_CLUSTERS 32
-
-/* indicate that the refcount of the referenced cluster is exactly one. */
-#define QCOW_OFLAG_COPIED     (1LL << 63)
-/* indicate that the cluster is compressed (they never have the copied flag) */
-#define QCOW_OFLAG_COMPRESSED (1LL << 62)
-/* The cluster reads as all zeros */
-#define QCOW_OFLAG_ZERO (1LL << 0)
-
-#define REFCOUNT_SHIFT 1 /* refcount size is 2 bytes */
-
-#define MIN_CLUSTER_BITS 9
-#define MAX_CLUSTER_BITS 21
-
-#define L2_CACHE_SIZE 16
-
-/* Must be at least 4 to cover all cases of refcount table growth */
-#define REFCOUNT_CACHE_SIZE 4
-
-#define DEFAULT_CLUSTER_SIZE 65536
-
-
-#define QCOW2_OPT_LAZY_REFCOUNTS "lazy_refcounts"
-#define QCOW2_OPT_DISCARD_REQUEST "pass_discard_request"
-#define QCOW2_OPT_DISCARD_SNAPSHOT "pass_discard_snapshot"
-#define QCOW2_OPT_DISCARD_OTHER "pass_discard_other"
-
-typedef struct QCowHeader {
-    uint32_t magic;
-    uint32_t version;
-    uint64_t backing_file_offset;
-    uint32_t backing_file_size;
-    uint32_t cluster_bits;
-    uint64_t size; /* in bytes */
-    uint32_t crypt_method;
-    uint32_t l1_size; /* XXX: save number of clusters instead ? */
-    uint64_t l1_table_offset;
-    uint64_t refcount_table_offset;
-    uint32_t refcount_table_clusters;
-    uint32_t nb_snapshots;
-    uint64_t snapshots_offset;
-
-    /* The following fields are only valid for version >= 3 */
-    uint64_t incompatible_features;
-    uint64_t compatible_features;
-    uint64_t autoclear_features;
-
-    uint32_t refcount_order;
-    uint32_t header_length;
-} QCowHeader;
-
-typedef struct QCowSnapshot {
-    uint64_t l1_table_offset;
-    uint32_t l1_size;
-    char *id_str;
-    char *name;
-    uint64_t disk_size;
-    uint64_t vm_state_size;
-    uint32_t date_sec;
-    uint32_t date_nsec;
-    uint64_t vm_clock_nsec;
-} QCowSnapshot;
-
-struct Qcow2Cache;
-typedef struct Qcow2Cache Qcow2Cache;
-
-typedef struct Qcow2UnknownHeaderExtension {
-    uint32_t magic;
-    uint32_t len;
-    QLIST_ENTRY(Qcow2UnknownHeaderExtension) next;
-    uint8_t data[];
-} Qcow2UnknownHeaderExtension;
-
-enum {
-    QCOW2_FEAT_TYPE_INCOMPATIBLE    = 0,
-    QCOW2_FEAT_TYPE_COMPATIBLE      = 1,
-    QCOW2_FEAT_TYPE_AUTOCLEAR       = 2,
-};
-
-/* Incompatible feature bits */
-enum {
-    QCOW2_INCOMPAT_DIRTY_BITNR   = 0,
-    QCOW2_INCOMPAT_DIRTY         = 1 << QCOW2_INCOMPAT_DIRTY_BITNR,
-
-    QCOW2_INCOMPAT_MASK          = QCOW2_INCOMPAT_DIRTY,
-};
-
-/* Compatible feature bits */
-enum {
-    QCOW2_COMPAT_LAZY_REFCOUNTS_BITNR = 0,
-    QCOW2_COMPAT_LAZY_REFCOUNTS       = 1 << QCOW2_COMPAT_LAZY_REFCOUNTS_BITNR,
-
-    QCOW2_COMPAT_FEAT_MASK            = QCOW2_COMPAT_LAZY_REFCOUNTS,
-};
-
-enum qcow2_discard_type {
-    QCOW2_DISCARD_NEVER = 0,
-    QCOW2_DISCARD_ALWAYS,
-    QCOW2_DISCARD_REQUEST,
-    QCOW2_DISCARD_SNAPSHOT,
-    QCOW2_DISCARD_OTHER,
-    QCOW2_DISCARD_MAX
-};
-
-typedef struct Qcow2Feature {
-    uint8_t type;
-    uint8_t bit;
-    char    name[46];
-} QEMU_PACKED Qcow2Feature;
-
-typedef struct Qcow2DiscardRegion {
-    BlockDriverState *bs;
-    uint64_t offset;
-    uint64_t bytes;
-    QTAILQ_ENTRY(Qcow2DiscardRegion) next;
-} Qcow2DiscardRegion;
-
-typedef struct BDRVQcowState {
-    int cluster_bits;
-    int cluster_size;
-    int cluster_sectors;
-    int l2_bits;
-    int l2_size;
-    int l1_size;
-    int l1_vm_state_index;
-    int csize_shift;
-    int csize_mask;
-    uint64_t cluster_offset_mask;
-    uint64_t l1_table_offset;
-    uint64_t *l1_table;
-
-    Qcow2Cache* l2_table_cache;
-    Qcow2Cache* refcount_block_cache;
-
-    uint8_t *cluster_cache;
-    uint8_t *cluster_data;
-    uint64_t cluster_cache_offset;
-    QLIST_HEAD(QCowClusterAlloc, QCowL2Meta) cluster_allocs;
-
-    uint64_t *refcount_table;
-    uint64_t refcount_table_offset;
-    uint32_t refcount_table_size;
-    int64_t free_cluster_index;
-    int64_t free_byte_offset;
-
-    CoMutex lock;
-
-    uint32_t crypt_method; /* current crypt method, 0 if no key yet */
-    uint32_t crypt_method_header;
-    AES_KEY aes_encrypt_key;
-    AES_KEY aes_decrypt_key;
-    uint64_t snapshots_offset;
-    int snapshots_size;
-    int nb_snapshots;
-    QCowSnapshot *snapshots;
-
-    int flags;
-    int qcow_version;
-    bool use_lazy_refcounts;
-
-    bool discard_passthrough[QCOW2_DISCARD_MAX];
-
-    uint64_t incompatible_features;
-    uint64_t compatible_features;
-    uint64_t autoclear_features;
-
-    size_t unknown_header_fields_size;
-    void* unknown_header_fields;
-    QLIST_HEAD(, Qcow2UnknownHeaderExtension) unknown_header_ext;
-    QTAILQ_HEAD (, Qcow2DiscardRegion) discards;
-    bool cache_discards;
-} BDRVQcowState;
-
-/* XXX: use std qcow open function ? */
-typedef struct QCowCreateState {
-    int cluster_size;
-    int cluster_bits;
-    uint16_t *refcount_block;
-    uint64_t *refcount_table;
-    int64_t l1_table_offset;
-    int64_t refcount_table_offset;
-    int64_t refcount_block_offset;
-} QCowCreateState;
-
-struct QCowAIOCB;
-
-typedef struct Qcow2COWRegion {
-    /**
-     * Offset of the COW region in bytes from the start of the first cluster
-     * touched by the request.
-     */
-    uint64_t    offset;
-
-    /** Number of sectors to copy */
-    int         nb_sectors;
-} Qcow2COWRegion;
-
-/**
- * Describes an in-flight (part of a) write request that writes to clusters
- * that are not referenced in their L2 table yet.
- */
-typedef struct QCowL2Meta
-{
-    /** Guest offset of the first newly allocated cluster */
-    uint64_t offset;
-
-    /** Host offset of the first newly allocated cluster */
-    uint64_t alloc_offset;
-
-    /**
-     * Number of sectors from the start of the first allocated cluster to
-     * the end of the (possibly shortened) request
-     */
-    int nb_available;
-
-    /** Number of newly allocated clusters */
-    int nb_clusters;
-
-    /**
-     * Requests that overlap with this allocation and wait to be restarted
-     * when the allocating request has completed.
-     */
-    CoQueue dependent_requests;
-
-    /**
-     * The COW Region between the start of the first allocated cluster and the
-     * area the guest actually writes to.
-     */
-    Qcow2COWRegion cow_start;
-
-    /**
-     * The COW Region between the area the guest actually writes to and the
-     * end of the last allocated cluster.
-     */
-    Qcow2COWRegion cow_end;
-
-    /** Pointer to next L2Meta of the same write request */
-    struct QCowL2Meta *next;
-
-    QLIST_ENTRY(QCowL2Meta) next_in_flight;
-} QCowL2Meta;
-
-enum {
-    QCOW2_CLUSTER_UNALLOCATED,
-    QCOW2_CLUSTER_NORMAL,
-    QCOW2_CLUSTER_COMPRESSED,
-    QCOW2_CLUSTER_ZERO
-};
-
-#define L1E_OFFSET_MASK 0x00ffffffffffff00ULL
-#define L2E_OFFSET_MASK 0x00ffffffffffff00ULL
-#define L2E_COMPRESSED_OFFSET_SIZE_MASK 0x3fffffffffffffffULL
-
-#define REFT_OFFSET_MASK 0xffffffffffffff00ULL
-
-static inline int64_t start_of_cluster(BDRVQcowState *s, int64_t offset)
-{
-    return offset & ~(s->cluster_size - 1);
-}
-
-static inline int64_t offset_into_cluster(BDRVQcowState *s, int64_t offset)
-{
-    return offset & (s->cluster_size - 1);
-}
-
-static inline int size_to_clusters(BDRVQcowState *s, int64_t size)
-{
-    return (size + (s->cluster_size - 1)) >> s->cluster_bits;
-}
-
-static inline int64_t size_to_l1(BDRVQcowState *s, int64_t size)
-{
-    int shift = s->cluster_bits + s->l2_bits;
-    return (size + (1ULL << shift) - 1) >> shift;
-}
-
-static inline int offset_to_l2_index(BDRVQcowState *s, int64_t offset)
-{
-    return (offset >> s->cluster_bits) & (s->l2_size - 1);
-}
-
-static inline int64_t align_offset(int64_t offset, int n)
-{
-    offset = (offset + n - 1) & ~(n - 1);
-    return offset;
-}
-
-static inline int qcow2_get_cluster_type(uint64_t l2_entry)
-{
-    if (l2_entry & QCOW_OFLAG_COMPRESSED) {
-        return QCOW2_CLUSTER_COMPRESSED;
-    } else if (l2_entry & QCOW_OFLAG_ZERO) {
-        return QCOW2_CLUSTER_ZERO;
-    } else if (!(l2_entry & L2E_OFFSET_MASK)) {
-        return QCOW2_CLUSTER_UNALLOCATED;
-    } else {
-        return QCOW2_CLUSTER_NORMAL;
-    }
-}
-
-/* Check whether refcounts are eager or lazy */
-static inline bool qcow2_need_accurate_refcounts(BDRVQcowState *s)
-{
-    return !(s->incompatible_features & QCOW2_INCOMPAT_DIRTY);
-}
-
-static inline uint64_t l2meta_cow_start(QCowL2Meta *m)
-{
-    return m->offset + m->cow_start.offset;
-}
-
-static inline uint64_t l2meta_cow_end(QCowL2Meta *m)
-{
-    return m->offset + m->cow_end.offset
-        + (m->cow_end.nb_sectors << BDRV_SECTOR_BITS);
-}
-
-// FIXME Need qcow2_ prefix to global functions
-
-/* qcow2.c functions */
-int qcow2_backing_read1(BlockDriverState *bs, QEMUIOVector *qiov,
-                  int64_t sector_num, int nb_sectors);
-
-int qcow2_mark_dirty(BlockDriverState *bs);
-int qcow2_update_header(BlockDriverState *bs);
-
-/* qcow2-refcount.c functions */
-int qcow2_refcount_init(BlockDriverState *bs);
-void qcow2_refcount_close(BlockDriverState *bs);
-
-int64_t qcow2_alloc_clusters(BlockDriverState *bs, int64_t size);
-int qcow2_alloc_clusters_at(BlockDriverState *bs, uint64_t offset,
-    int nb_clusters);
-int64_t qcow2_alloc_bytes(BlockDriverState *bs, int size);
-void qcow2_free_clusters(BlockDriverState *bs,
-                          int64_t offset, int64_t size,
-                          enum qcow2_discard_type type);
-void qcow2_free_any_clusters(BlockDriverState *bs, uint64_t l2_entry,
-                             int nb_clusters, enum qcow2_discard_type type);
-
-int qcow2_update_snapshot_refcount(BlockDriverState *bs,
-    int64_t l1_table_offset, int l1_size, int addend);
-
-int qcow2_check_refcounts(BlockDriverState *bs, BdrvCheckResult *res,
-                          BdrvCheckMode fix);
-
-void qcow2_process_discards(BlockDriverState *bs, int ret);
-
-/* qcow2-cluster.c functions */
-int qcow2_grow_l1_table(BlockDriverState *bs, uint64_t min_size,
-                        bool exact_size);
-void qcow2_l2_cache_reset(BlockDriverState *bs);
-int qcow2_decompress_cluster(BlockDriverState *bs, uint64_t cluster_offset);
-void qcow2_encrypt_sectors(BDRVQcowState *s, int64_t sector_num,
-                     uint8_t *out_buf, const uint8_t *in_buf,
-                     int nb_sectors, int enc,
-                     const AES_KEY *key);
-
-int qcow2_get_cluster_offset(BlockDriverState *bs, uint64_t offset,
-    int *num, uint64_t *cluster_offset);
-int qcow2_alloc_cluster_offset(BlockDriverState *bs, uint64_t offset,
-    int n_start, int n_end, int *num, uint64_t *host_offset, QCowL2Meta **m);
-uint64_t qcow2_alloc_compressed_cluster_offset(BlockDriverState *bs,
-                                         uint64_t offset,
-                                         int compressed_size);
-
-int qcow2_alloc_cluster_link_l2(BlockDriverState *bs, QCowL2Meta *m);
-int qcow2_discard_clusters(BlockDriverState *bs, uint64_t offset,
-    int nb_sectors);
-int qcow2_zero_clusters(BlockDriverState *bs, uint64_t offset, int nb_sectors);
-
-/* qcow2-snapshot.c functions */
-int qcow2_snapshot_create(BlockDriverState *bs, QEMUSnapshotInfo *sn_info);
-int qcow2_snapshot_goto(BlockDriverState *bs, const char *snapshot_id);
-int qcow2_snapshot_delete(BlockDriverState *bs, const char *snapshot_id);
-int qcow2_snapshot_list(BlockDriverState *bs, QEMUSnapshotInfo **psn_tab);
-int qcow2_snapshot_load_tmp(BlockDriverState *bs, const char *snapshot_name);
-
-void qcow2_free_snapshots(BlockDriverState *bs);
-int qcow2_read_snapshots(BlockDriverState *bs);
-
-/* qcow2-cache.c functions */
-Qcow2Cache *qcow2_cache_create(BlockDriverState *bs, int num_tables);
-int qcow2_cache_destroy(BlockDriverState* bs, Qcow2Cache *c);
-
-void qcow2_cache_entry_mark_dirty(Qcow2Cache *c, void *table);
-int qcow2_cache_flush(BlockDriverState *bs, Qcow2Cache *c);
-int qcow2_cache_set_dependency(BlockDriverState *bs, Qcow2Cache *c,
-    Qcow2Cache *dependency);
-void qcow2_cache_depends_on_flush(Qcow2Cache *c);
-
-int qcow2_cache_get(BlockDriverState *bs, Qcow2Cache *c, uint64_t offset,
-    void **table);
-int qcow2_cache_get_empty(BlockDriverState *bs, Qcow2Cache *c, uint64_t offset,
-    void **table);
-int qcow2_cache_put(BlockDriverState *bs, Qcow2Cache *c, void **table);
-
-#endif
diff --git a/contrib/qemu/block/qed-check.c b/contrib/qemu/block/qed-check.c
deleted file mode 100644
index b473dcd61f6..00000000000
--- a/contrib/qemu/block/qed-check.c
+++ /dev/null
@@ -1,248 +0,0 @@
-/*
- * QEMU Enhanced Disk Format Consistency Check
- *
- * Copyright IBM, Corp. 2010
- *
- * Authors:
- *  Stefan Hajnoczi   <stefanha@linux.vnet.ibm.com>
- *
- * This work is licensed under the terms of the GNU LGPL, version 2 or later.
- * See the COPYING.LIB file in the top-level directory.
- *
- */
-
-#include "qed.h"
-
-typedef struct {
-    BDRVQEDState *s;
-    BdrvCheckResult *result;
-    bool fix;                           /* whether to fix invalid offsets */
-
-    uint64_t nclusters;
-    uint32_t *used_clusters;            /* referenced cluster bitmap */
-
-    QEDRequest request;
-} QEDCheck;
-
-static bool qed_test_bit(uint32_t *bitmap, uint64_t n) {
-    return !!(bitmap[n / 32] & (1 << (n % 32)));
-}
-
-static void qed_set_bit(uint32_t *bitmap, uint64_t n) {
-    bitmap[n / 32] |= 1 << (n % 32);
-}
-
-/**
- * Set bitmap bits for clusters
- *
- * @check:          Check structure
- * @offset:         Starting offset in bytes
- * @n:              Number of clusters
- */
-static bool qed_set_used_clusters(QEDCheck *check, uint64_t offset,
-                                  unsigned int n)
-{
-    uint64_t cluster = qed_bytes_to_clusters(check->s, offset);
-    unsigned int corruptions = 0;
-
-    while (n-- != 0) {
-        /* Clusters should only be referenced once */
-        if (qed_test_bit(check->used_clusters, cluster)) {
-            corruptions++;
-        }
-
-        qed_set_bit(check->used_clusters, cluster);
-        cluster++;
-    }
-
-    check->result->corruptions += corruptions;
-    return corruptions == 0;
-}
-
-/**
- * Check an L2 table
- *
- * @ret:            Number of invalid cluster offsets
- */
-static unsigned int qed_check_l2_table(QEDCheck *check, QEDTable *table)
-{
-    BDRVQEDState *s = check->s;
-    unsigned int i, num_invalid = 0;
-    uint64_t last_offset = 0;
-
-    for (i = 0; i < s->table_nelems; i++) {
-        uint64_t offset = table->offsets[i];
-
-        if (qed_offset_is_unalloc_cluster(offset) ||
-            qed_offset_is_zero_cluster(offset)) {
-            continue;
-        }
-        check->result->bfi.allocated_clusters++;
-        if (last_offset && (last_offset + s->header.cluster_size != offset)) {
-            check->result->bfi.fragmented_clusters++;
-        }
-        last_offset = offset;
-
-        /* Detect invalid cluster offset */
-        if (!qed_check_cluster_offset(s, offset)) {
-            if (check->fix) {
-                table->offsets[i] = 0;
-                check->result->corruptions_fixed++;
-            } else {
-                check->result->corruptions++;
-            }
-
-            num_invalid++;
-            continue;
-        }
-
-        qed_set_used_clusters(check, offset, 1);
-    }
-
-    return num_invalid;
-}
-
-/**
- * Descend tables and check each cluster is referenced once only
- */
-static int qed_check_l1_table(QEDCheck *check, QEDTable *table)
-{
-    BDRVQEDState *s = check->s;
-    unsigned int i, num_invalid_l1 = 0;
-    int ret, last_error = 0;
-
-    /* Mark L1 table clusters used */
-    qed_set_used_clusters(check, s->header.l1_table_offset,
-                          s->header.table_size);
-
-    for (i = 0; i < s->table_nelems; i++) {
-        unsigned int num_invalid_l2;
-        uint64_t offset = table->offsets[i];
-
-        if (qed_offset_is_unalloc_cluster(offset)) {
-            continue;
-        }
-
-        /* Detect invalid L2 offset */
-        if (!qed_check_table_offset(s, offset)) {
-            /* Clear invalid offset */
-            if (check->fix) {
-                table->offsets[i] = 0;
-                check->result->corruptions_fixed++;
-            } else {
-                check->result->corruptions++;
-            }
-
-            num_invalid_l1++;
-            continue;
-        }
-
-        if (!qed_set_used_clusters(check, offset, s->header.table_size)) {
-            continue; /* skip an invalid table */
-        }
-
-        ret = qed_read_l2_table_sync(s, &check->request, offset);
-        if (ret) {
-            check->result->check_errors++;
-            last_error = ret;
-            continue;
-        }
-
-        num_invalid_l2 = qed_check_l2_table(check,
-                                            check->request.l2_table->table);
-
-        /* Write out fixed L2 table */
-        if (num_invalid_l2 > 0 && check->fix) {
-            ret = qed_write_l2_table_sync(s, &check->request, 0,
-                                          s->table_nelems, false);
-            if (ret) {
-                check->result->check_errors++;
-                last_error = ret;
-                continue;
-            }
-        }
-    }
-
-    /* Drop reference to final table */
-    qed_unref_l2_cache_entry(check->request.l2_table);
-    check->request.l2_table = NULL;
-
-    /* Write out fixed L1 table */
-    if (num_invalid_l1 > 0 && check->fix) {
-        ret = qed_write_l1_table_sync(s, 0, s->table_nelems);
-        if (ret) {
-            check->result->check_errors++;
-            last_error = ret;
-        }
-    }
-
-    return last_error;
-}
-
-/**
- * Check for unreferenced (leaked) clusters
- */
-static void qed_check_for_leaks(QEDCheck *check)
-{
-    BDRVQEDState *s = check->s;
-    uint64_t i;
-
-    for (i = s->header.header_size; i < check->nclusters; i++) {
-        if (!qed_test_bit(check->used_clusters, i)) {
-            check->result->leaks++;
-        }
-    }
-}
-
-/**
- * Mark an image clean once it passes check or has been repaired
- */
-static void qed_check_mark_clean(BDRVQEDState *s, BdrvCheckResult *result)
-{
-    /* Skip if there were unfixable corruptions or I/O errors */
-    if (result->corruptions > 0 || result->check_errors > 0) {
-        return;
-    }
-
-    /* Skip if image is already marked clean */
-    if (!(s->header.features & QED_F_NEED_CHECK)) {
-        return;
-    }
-
-    /* Ensure fixes reach storage before clearing check bit */
-    bdrv_flush(s->bs);
-
-    s->header.features &= ~QED_F_NEED_CHECK;
-    qed_write_header_sync(s);
-}
-
-int qed_check(BDRVQEDState *s, BdrvCheckResult *result, bool fix)
-{
-    QEDCheck check = {
-        .s = s,
-        .result = result,
-        .nclusters = qed_bytes_to_clusters(s, s->file_size),
-        .request = { .l2_table = NULL },
-        .fix = fix,
-    };
-    int ret;
-
-    check.used_clusters = g_malloc0(((check.nclusters + 31) / 32) *
-                                       sizeof(check.used_clusters[0]));
-
-    check.result->bfi.total_clusters =
-        (s->header.image_size + s->header.cluster_size - 1) /
-            s->header.cluster_size;
-    ret = qed_check_l1_table(&check, s->l1_table);
-    if (ret == 0) {
-        /* Only check for leaks if entire image was scanned successfully */
-        qed_check_for_leaks(&check);
-
-        if (fix) {
-            qed_check_mark_clean(s, result);
-        }
-    }
-
-    g_free(check.used_clusters);
-    return ret;
-}
diff --git a/contrib/qemu/block/qed-cluster.c b/contrib/qemu/block/qed-cluster.c
deleted file mode 100644
index f64b2af8f7e..00000000000
--- a/contrib/qemu/block/qed-cluster.c
+++ /dev/null
@@ -1,165 +0,0 @@
-/*
- * QEMU Enhanced Disk Format Cluster functions
- *
- * Copyright IBM, Corp. 2010
- *
- * Authors:
- *  Stefan Hajnoczi   <stefanha@linux.vnet.ibm.com>
- *  Anthony Liguori   <aliguori@us.ibm.com>
- *
- * This work is licensed under the terms of the GNU LGPL, version 2 or later.
- * See the COPYING.LIB file in the top-level directory.
- *
- */
-
-#include "qed.h"
-
-/**
- * Count the number of contiguous data clusters
- *
- * @s:              QED state
- * @table:          L2 table
- * @index:          First cluster index
- * @n:              Maximum number of clusters
- * @offset:         Set to first cluster offset
- *
- * This function scans tables for contiguous clusters.  A contiguous run of
- * clusters may be allocated, unallocated, or zero.
- */
-static unsigned int qed_count_contiguous_clusters(BDRVQEDState *s,
-                                                  QEDTable *table,
-                                                  unsigned int index,
-                                                  unsigned int n,
-                                                  uint64_t *offset)
-{
-    unsigned int end = MIN(index + n, s->table_nelems);
-    uint64_t last = table->offsets[index];
-    unsigned int i;
-
-    *offset = last;
-
-    for (i = index + 1; i < end; i++) {
-        if (qed_offset_is_unalloc_cluster(last)) {
-            /* Counting unallocated clusters */
-            if (!qed_offset_is_unalloc_cluster(table->offsets[i])) {
-                break;
-            }
-        } else if (qed_offset_is_zero_cluster(last)) {
-            /* Counting zero clusters */
-            if (!qed_offset_is_zero_cluster(table->offsets[i])) {
-                break;
-            }
-        } else {
-            /* Counting allocated clusters */
-            if (table->offsets[i] != last + s->header.cluster_size) {
-                break;
-            }
-            last = table->offsets[i];
-        }
-    }
-    return i - index;
-}
-
-typedef struct {
-    BDRVQEDState *s;
-    uint64_t pos;
-    size_t len;
-
-    QEDRequest *request;
-
-    /* User callback */
-    QEDFindClusterFunc *cb;
-    void *opaque;
-} QEDFindClusterCB;
-
-static void qed_find_cluster_cb(void *opaque, int ret)
-{
-    QEDFindClusterCB *find_cluster_cb = opaque;
-    BDRVQEDState *s = find_cluster_cb->s;
-    QEDRequest *request = find_cluster_cb->request;
-    uint64_t offset = 0;
-    size_t len = 0;
-    unsigned int index;
-    unsigned int n;
-
-    if (ret) {
-        goto out;
-    }
-
-    index = qed_l2_index(s, find_cluster_cb->pos);
-    n = qed_bytes_to_clusters(s,
-                              qed_offset_into_cluster(s, find_cluster_cb->pos) +
-                              find_cluster_cb->len);
-    n = qed_count_contiguous_clusters(s, request->l2_table->table,
-                                      index, n, &offset);
-
-    if (qed_offset_is_unalloc_cluster(offset)) {
-        ret = QED_CLUSTER_L2;
-    } else if (qed_offset_is_zero_cluster(offset)) {
-        ret = QED_CLUSTER_ZERO;
-    } else if (qed_check_cluster_offset(s, offset)) {
-        ret = QED_CLUSTER_FOUND;
-    } else {
-        ret = -EINVAL;
-    }
-
-    len = MIN(find_cluster_cb->len, n * s->header.cluster_size -
-              qed_offset_into_cluster(s, find_cluster_cb->pos));
-
-out:
-    find_cluster_cb->cb(find_cluster_cb->opaque, ret, offset, len);
-    g_free(find_cluster_cb);
-}
-
-/**
- * Find the offset of a data cluster
- *
- * @s:          QED state
- * @request:    L2 cache entry
- * @pos:        Byte position in device
- * @len:        Number of bytes
- * @cb:         Completion function
- * @opaque:     User data for completion function
- *
- * This function translates a position in the block device to an offset in the
- * image file.  It invokes the cb completion callback to report back the
- * translated offset or unallocated range in the image file.
- *
- * If the L2 table exists, request->l2_table points to the L2 table cache entry
- * and the caller must free the reference when they are finished.  The cache
- * entry is exposed in this way to avoid callers having to read the L2 table
- * again later during request processing.  If request->l2_table is non-NULL it
- * will be unreferenced before taking on the new cache entry.
- */
-void qed_find_cluster(BDRVQEDState *s, QEDRequest *request, uint64_t pos,
-                      size_t len, QEDFindClusterFunc *cb, void *opaque)
-{
-    QEDFindClusterCB *find_cluster_cb;
-    uint64_t l2_offset;
-
-    /* Limit length to L2 boundary.  Requests are broken up at the L2 boundary
-     * so that a request acts on one L2 table at a time.
-     */
-    len = MIN(len, (((pos >> s->l1_shift) + 1) << s->l1_shift) - pos);
-
-    l2_offset = s->l1_table->offsets[qed_l1_index(s, pos)];
-    if (qed_offset_is_unalloc_cluster(l2_offset)) {
-        cb(opaque, QED_CLUSTER_L1, 0, len);
-        return;
-    }
-    if (!qed_check_table_offset(s, l2_offset)) {
-        cb(opaque, -EINVAL, 0, 0);
-        return;
-    }
-
-    find_cluster_cb = g_malloc(sizeof(*find_cluster_cb));
-    find_cluster_cb->s = s;
-    find_cluster_cb->pos = pos;
-    find_cluster_cb->len = len;
-    find_cluster_cb->cb = cb;
-    find_cluster_cb->opaque = opaque;
-    find_cluster_cb->request = request;
-
-    qed_read_l2_table(s, request, l2_offset,
-                      qed_find_cluster_cb, find_cluster_cb);
-}
diff --git a/contrib/qemu/block/qed-gencb.c b/contrib/qemu/block/qed-gencb.c
deleted file mode 100644
index 7d7ac1ffc8e..00000000000
--- a/contrib/qemu/block/qed-gencb.c
+++ /dev/null
@@ -1,32 +0,0 @@
-/*
- * QEMU Enhanced Disk Format
- *
- * Copyright IBM, Corp. 2010
- *
- * Authors:
- *  Stefan Hajnoczi   <stefanha@linux.vnet.ibm.com>
- *
- * This work is licensed under the terms of the GNU LGPL, version 2 or later.
- * See the COPYING.LIB file in the top-level directory.
- *
- */
-
-#include "qed.h"
-
-void *gencb_alloc(size_t len, BlockDriverCompletionFunc *cb, void *opaque)
-{
-    GenericCB *gencb = g_malloc(len);
-    gencb->cb = cb;
-    gencb->opaque = opaque;
-    return gencb;
-}
-
-void gencb_complete(void *opaque, int ret)
-{
-    GenericCB *gencb = opaque;
-    BlockDriverCompletionFunc *cb = gencb->cb;
-    void *user_opaque = gencb->opaque;
-
-    g_free(gencb);
-    cb(user_opaque, ret);
-}
diff --git a/contrib/qemu/block/qed-l2-cache.c b/contrib/qemu/block/qed-l2-cache.c
deleted file mode 100644
index e9b2aae44d9..00000000000
--- a/contrib/qemu/block/qed-l2-cache.c
+++ /dev/null
@@ -1,187 +0,0 @@
-/*
- * QEMU Enhanced Disk Format L2 Cache
- *
- * Copyright IBM, Corp. 2010
- *
- * Authors:
- *  Anthony Liguori   <aliguori@us.ibm.com>
- *
- * This work is licensed under the terms of the GNU LGPL, version 2 or later.
- * See the COPYING.LIB file in the top-level directory.
- *
- */
-
-/*
- * L2 table cache usage is as follows:
- *
- * An open image has one L2 table cache that is used to avoid accessing the
- * image file for recently referenced L2 tables.
- *
- * Cluster offset lookup translates the logical offset within the block device
- * to a cluster offset within the image file.  This is done by indexing into
- * the L1 and L2 tables which store cluster offsets.  It is here where the L2
- * table cache serves up recently referenced L2 tables.
- *
- * If there is a cache miss, that L2 table is read from the image file and
- * committed to the cache.  Subsequent accesses to that L2 table will be served
- * from the cache until the table is evicted from the cache.
- *
- * L2 tables are also committed to the cache when new L2 tables are allocated
- * in the image file.  Since the L2 table cache is write-through, the new L2
- * table is first written out to the image file and then committed to the
- * cache.
- *
- * Multiple I/O requests may be using an L2 table cache entry at any given
- * time.  That means an entry may be in use across several requests and
- * reference counting is needed to free the entry at the correct time.  In
- * particular, an entry evicted from the cache will only be freed once all
- * references are dropped.
- *
- * An in-flight I/O request will hold a reference to a L2 table cache entry for
- * the period during which it needs to access the L2 table.  This includes
- * cluster offset lookup, L2 table allocation, and L2 table update when a new
- * data cluster has been allocated.
- *
- * An interesting case occurs when two requests need to access an L2 table that
- * is not in the cache.  Since the operation to read the table from the image
- * file takes some time to complete, both requests may see a cache miss and
- * start reading the L2 table from the image file.  The first to finish will
- * commit its L2 table into the cache.  When the second tries to commit its
- * table will be deleted in favor of the existing cache entry.
- */
-
-#include "trace.h"
-#include "qed.h"
-
-/* Each L2 holds 2GB so this let's us fully cache a 100GB disk */
-#define MAX_L2_CACHE_SIZE 50
-
-/**
- * Initialize the L2 cache
- */
-void qed_init_l2_cache(L2TableCache *l2_cache)
-{
-    QTAILQ_INIT(&l2_cache->entries);
-    l2_cache->n_entries = 0;
-}
-
-/**
- * Free the L2 cache
- */
-void qed_free_l2_cache(L2TableCache *l2_cache)
-{
-    CachedL2Table *entry, *next_entry;
-
-    QTAILQ_FOREACH_SAFE(entry, &l2_cache->entries, node, next_entry) {
-        qemu_vfree(entry->table);
-        g_free(entry);
-    }
-}
-
-/**
- * Allocate an uninitialized entry from the cache
- *
- * The returned entry has a reference count of 1 and is owned by the caller.
- * The caller must allocate the actual table field for this entry and it must
- * be freeable using qemu_vfree().
- */
-CachedL2Table *qed_alloc_l2_cache_entry(L2TableCache *l2_cache)
-{
-    CachedL2Table *entry;
-
-    entry = g_malloc0(sizeof(*entry));
-    entry->ref++;
-
-    trace_qed_alloc_l2_cache_entry(l2_cache, entry);
-
-    return entry;
-}
-
-/**
- * Decrease an entry's reference count and free if necessary when the reference
- * count drops to zero.
- */
-void qed_unref_l2_cache_entry(CachedL2Table *entry)
-{
-    if (!entry) {
-        return;
-    }
-
-    entry->ref--;
-    trace_qed_unref_l2_cache_entry(entry, entry->ref);
-    if (entry->ref == 0) {
-        qemu_vfree(entry->table);
-        g_free(entry);
-    }
-}
-
-/**
- * Find an entry in the L2 cache.  This may return NULL and it's up to the
- * caller to satisfy the cache miss.
- *
- * For a cached entry, this function increases the reference count and returns
- * the entry.
- */
-CachedL2Table *qed_find_l2_cache_entry(L2TableCache *l2_cache, uint64_t offset)
-{
-    CachedL2Table *entry;
-
-    QTAILQ_FOREACH(entry, &l2_cache->entries, node) {
-        if (entry->offset == offset) {
-            trace_qed_find_l2_cache_entry(l2_cache, entry, offset, entry->ref);
-            entry->ref++;
-            return entry;
-        }
-    }
-    return NULL;
-}
-
-/**
- * Commit an L2 cache entry into the cache.  This is meant to be used as part of
- * the process to satisfy a cache miss.  A caller would allocate an entry which
- * is not actually in the L2 cache and then once the entry was valid and
- * present on disk, the entry can be committed into the cache.
- *
- * Since the cache is write-through, it's important that this function is not
- * called until the entry is present on disk and the L1 has been updated to
- * point to the entry.
- *
- * N.B. This function steals a reference to the l2_table from the caller so the
- * caller must obtain a new reference by issuing a call to
- * qed_find_l2_cache_entry().
- */
-void qed_commit_l2_cache_entry(L2TableCache *l2_cache, CachedL2Table *l2_table)
-{
-    CachedL2Table *entry;
-
-    entry = qed_find_l2_cache_entry(l2_cache, l2_table->offset);
-    if (entry) {
-        qed_unref_l2_cache_entry(entry);
-        qed_unref_l2_cache_entry(l2_table);
-        return;
-    }
-
-    /* Evict an unused cache entry so we have space.  If all entries are in use
-     * we can grow the cache temporarily and we try to shrink back down later.
-     */
-    if (l2_cache->n_entries >= MAX_L2_CACHE_SIZE) {
-        CachedL2Table *next;
-        QTAILQ_FOREACH_SAFE(entry, &l2_cache->entries, node, next) {
-            if (entry->ref > 1) {
-                continue;
-            }
-
-            QTAILQ_REMOVE(&l2_cache->entries, entry, node);
-            l2_cache->n_entries--;
-            qed_unref_l2_cache_entry(entry);
-
-            /* Stop evicting when we've shrunk back to max size */
-            if (l2_cache->n_entries < MAX_L2_CACHE_SIZE) {
-                break;
-            }
-        }
-    }
-
-    l2_cache->n_entries++;
-    QTAILQ_INSERT_TAIL(&l2_cache->entries, l2_table, node);
-}
diff --git a/contrib/qemu/block/qed-table.c b/contrib/qemu/block/qed-table.c
deleted file mode 100644
index 76d2dcccf81..00000000000
--- a/contrib/qemu/block/qed-table.c
+++ /dev/null
@@ -1,296 +0,0 @@
-/*
- * QEMU Enhanced Disk Format Table I/O
- *
- * Copyright IBM, Corp. 2010
- *
- * Authors:
- *  Stefan Hajnoczi   <stefanha@linux.vnet.ibm.com>
- *  Anthony Liguori   <aliguori@us.ibm.com>
- *
- * This work is licensed under the terms of the GNU LGPL, version 2 or later.
- * See the COPYING.LIB file in the top-level directory.
- *
- */
-
-#include "trace.h"
-#include "qemu/sockets.h" /* for EINPROGRESS on Windows */
-#include "qed.h"
-
-typedef struct {
-    GenericCB gencb;
-    BDRVQEDState *s;
-    QEDTable *table;
-
-    struct iovec iov;
-    QEMUIOVector qiov;
-} QEDReadTableCB;
-
-static void qed_read_table_cb(void *opaque, int ret)
-{
-    QEDReadTableCB *read_table_cb = opaque;
-    QEDTable *table = read_table_cb->table;
-    int noffsets = read_table_cb->qiov.size / sizeof(uint64_t);
-    int i;
-
-    /* Handle I/O error */
-    if (ret) {
-        goto out;
-    }
-
-    /* Byteswap offsets */
-    for (i = 0; i < noffsets; i++) {
-        table->offsets[i] = le64_to_cpu(table->offsets[i]);
-    }
-
-out:
-    /* Completion */
-    trace_qed_read_table_cb(read_table_cb->s, read_table_cb->table, ret);
-    gencb_complete(&read_table_cb->gencb, ret);
-}
-
-static void qed_read_table(BDRVQEDState *s, uint64_t offset, QEDTable *table,
-                           BlockDriverCompletionFunc *cb, void *opaque)
-{
-    QEDReadTableCB *read_table_cb = gencb_alloc(sizeof(*read_table_cb),
-                                                cb, opaque);
-    QEMUIOVector *qiov = &read_table_cb->qiov;
-
-    trace_qed_read_table(s, offset, table);
-
-    read_table_cb->s = s;
-    read_table_cb->table = table;
-    read_table_cb->iov.iov_base = table->offsets,
-    read_table_cb->iov.iov_len = s->header.cluster_size * s->header.table_size,
-
-    qemu_iovec_init_external(qiov, &read_table_cb->iov, 1);
-    bdrv_aio_readv(s->bs->file, offset / BDRV_SECTOR_SIZE, qiov,
-                   qiov->size / BDRV_SECTOR_SIZE,
-                   qed_read_table_cb, read_table_cb);
-}
-
-typedef struct {
-    GenericCB gencb;
-    BDRVQEDState *s;
-    QEDTable *orig_table;
-    QEDTable *table;
-    bool flush;             /* flush after write? */
-
-    struct iovec iov;
-    QEMUIOVector qiov;
-} QEDWriteTableCB;
-
-static void qed_write_table_cb(void *opaque, int ret)
-{
-    QEDWriteTableCB *write_table_cb = opaque;
-
-    trace_qed_write_table_cb(write_table_cb->s,
-                             write_table_cb->orig_table,
-                             write_table_cb->flush,
-                             ret);
-
-    if (ret) {
-        goto out;
-    }
-
-    if (write_table_cb->flush) {
-        /* We still need to flush first */
-        write_table_cb->flush = false;
-        bdrv_aio_flush(write_table_cb->s->bs, qed_write_table_cb,
-                       write_table_cb);
-        return;
-    }
-
-out:
-    qemu_vfree(write_table_cb->table);
-    gencb_complete(&write_table_cb->gencb, ret);
-}
-
-/**
- * Write out an updated part or all of a table
- *
- * @s:          QED state
- * @offset:     Offset of table in image file, in bytes
- * @table:      Table
- * @index:      Index of first element
- * @n:          Number of elements
- * @flush:      Whether or not to sync to disk
- * @cb:         Completion function
- * @opaque:     Argument for completion function
- */
-static void qed_write_table(BDRVQEDState *s, uint64_t offset, QEDTable *table,
-                            unsigned int index, unsigned int n, bool flush,
-                            BlockDriverCompletionFunc *cb, void *opaque)
-{
-    QEDWriteTableCB *write_table_cb;
-    unsigned int sector_mask = BDRV_SECTOR_SIZE / sizeof(uint64_t) - 1;
-    unsigned int start, end, i;
-    size_t len_bytes;
-
-    trace_qed_write_table(s, offset, table, index, n);
-
-    /* Calculate indices of the first and one after last elements */
-    start = index & ~sector_mask;
-    end = (index + n + sector_mask) & ~sector_mask;
-
-    len_bytes = (end - start) * sizeof(uint64_t);
-
-    write_table_cb = gencb_alloc(sizeof(*write_table_cb), cb, opaque);
-    write_table_cb->s = s;
-    write_table_cb->orig_table = table;
-    write_table_cb->flush = flush;
-    write_table_cb->table = qemu_blockalign(s->bs, len_bytes);
-    write_table_cb->iov.iov_base = write_table_cb->table->offsets;
-    write_table_cb->iov.iov_len = len_bytes;
-    qemu_iovec_init_external(&write_table_cb->qiov, &write_table_cb->iov, 1);
-
-    /* Byteswap table */
-    for (i = start; i < end; i++) {
-        uint64_t le_offset = cpu_to_le64(table->offsets[i]);
-        write_table_cb->table->offsets[i - start] = le_offset;
-    }
-
-    /* Adjust for offset into table */
-    offset += start * sizeof(uint64_t);
-
-    bdrv_aio_writev(s->bs->file, offset / BDRV_SECTOR_SIZE,
-                    &write_table_cb->qiov,
-                    write_table_cb->qiov.size / BDRV_SECTOR_SIZE,
-                    qed_write_table_cb, write_table_cb);
-}
-
-/**
- * Propagate return value from async callback
- */
-static void qed_sync_cb(void *opaque, int ret)
-{
-    *(int *)opaque = ret;
-}
-
-int qed_read_l1_table_sync(BDRVQEDState *s)
-{
-    int ret = -EINPROGRESS;
-
-    qed_read_table(s, s->header.l1_table_offset,
-                   s->l1_table, qed_sync_cb, &ret);
-    while (ret == -EINPROGRESS) {
-        qemu_aio_wait();
-    }
-
-    return ret;
-}
-
-void qed_write_l1_table(BDRVQEDState *s, unsigned int index, unsigned int n,
-                        BlockDriverCompletionFunc *cb, void *opaque)
-{
-    BLKDBG_EVENT(s->bs->file, BLKDBG_L1_UPDATE);
-    qed_write_table(s, s->header.l1_table_offset,
-                    s->l1_table, index, n, false, cb, opaque);
-}
-
-int qed_write_l1_table_sync(BDRVQEDState *s, unsigned int index,
-                            unsigned int n)
-{
-    int ret = -EINPROGRESS;
-
-    qed_write_l1_table(s, index, n, qed_sync_cb, &ret);
-    while (ret == -EINPROGRESS) {
-        qemu_aio_wait();
-    }
-
-    return ret;
-}
-
-typedef struct {
-    GenericCB gencb;
-    BDRVQEDState *s;
-    uint64_t l2_offset;
-    QEDRequest *request;
-} QEDReadL2TableCB;
-
-static void qed_read_l2_table_cb(void *opaque, int ret)
-{
-    QEDReadL2TableCB *read_l2_table_cb = opaque;
-    QEDRequest *request = read_l2_table_cb->request;
-    BDRVQEDState *s = read_l2_table_cb->s;
-    CachedL2Table *l2_table = request->l2_table;
-    uint64_t l2_offset = read_l2_table_cb->l2_offset;
-
-    if (ret) {
-        /* can't trust loaded L2 table anymore */
-        qed_unref_l2_cache_entry(l2_table);
-        request->l2_table = NULL;
-    } else {
-        l2_table->offset = l2_offset;
-
-        qed_commit_l2_cache_entry(&s->l2_cache, l2_table);
-
-        /* This is guaranteed to succeed because we just committed the entry
-         * to the cache.
-         */
-        request->l2_table = qed_find_l2_cache_entry(&s->l2_cache, l2_offset);
-        assert(request->l2_table != NULL);
-    }
-
-    gencb_complete(&read_l2_table_cb->gencb, ret);
-}
-
-void qed_read_l2_table(BDRVQEDState *s, QEDRequest *request, uint64_t offset,
-                       BlockDriverCompletionFunc *cb, void *opaque)
-{
-    QEDReadL2TableCB *read_l2_table_cb;
-
-    qed_unref_l2_cache_entry(request->l2_table);
-
-    /* Check for cached L2 entry */
-    request->l2_table = qed_find_l2_cache_entry(&s->l2_cache, offset);
-    if (request->l2_table) {
-        cb(opaque, 0);
-        return;
-    }
-
-    request->l2_table = qed_alloc_l2_cache_entry(&s->l2_cache);
-    request->l2_table->table = qed_alloc_table(s);
-
-    read_l2_table_cb = gencb_alloc(sizeof(*read_l2_table_cb), cb, opaque);
-    read_l2_table_cb->s = s;
-    read_l2_table_cb->l2_offset = offset;
-    read_l2_table_cb->request = request;
-
-    BLKDBG_EVENT(s->bs->file, BLKDBG_L2_LOAD);
-    qed_read_table(s, offset, request->l2_table->table,
-                   qed_read_l2_table_cb, read_l2_table_cb);
-}
-
-int qed_read_l2_table_sync(BDRVQEDState *s, QEDRequest *request, uint64_t offset)
-{
-    int ret = -EINPROGRESS;
-
-    qed_read_l2_table(s, request, offset, qed_sync_cb, &ret);
-    while (ret == -EINPROGRESS) {
-        qemu_aio_wait();
-    }
-
-    return ret;
-}
-
-void qed_write_l2_table(BDRVQEDState *s, QEDRequest *request,
-                        unsigned int index, unsigned int n, bool flush,
-                        BlockDriverCompletionFunc *cb, void *opaque)
-{
-    BLKDBG_EVENT(s->bs->file, BLKDBG_L2_UPDATE);
-    qed_write_table(s, request->l2_table->offset,
-                    request->l2_table->table, index, n, flush, cb, opaque);
-}
-
-int qed_write_l2_table_sync(BDRVQEDState *s, QEDRequest *request,
-                            unsigned int index, unsigned int n, bool flush)
-{
-    int ret = -EINPROGRESS;
-
-    qed_write_l2_table(s, request, index, n, flush, qed_sync_cb, &ret);
-    while (ret == -EINPROGRESS) {
-        qemu_aio_wait();
-    }
-
-    return ret;
-}
diff --git a/contrib/qemu/block/qed.c b/contrib/qemu/block/qed.c
deleted file mode 100644
index f767b0528ce..00000000000
--- a/contrib/qemu/block/qed.c
+++ /dev/null
@@ -1,1596 +0,0 @@
-/*
- * QEMU Enhanced Disk Format
- *
- * Copyright IBM, Corp. 2010
- *
- * Authors:
- *  Stefan Hajnoczi   <stefanha@linux.vnet.ibm.com>
- *  Anthony Liguori   <aliguori@us.ibm.com>
- *
- * This work is licensed under the terms of the GNU LGPL, version 2 or later.
- * See the COPYING.LIB file in the top-level directory.
- *
- */
-
-#include "qemu/timer.h"
-#include "trace.h"
-#include "qed.h"
-#include "qapi/qmp/qerror.h"
-#include "migration/migration.h"
-
-static void qed_aio_cancel(BlockDriverAIOCB *blockacb)
-{
-    QEDAIOCB *acb = (QEDAIOCB *)blockacb;
-    bool finished = false;
-
-    /* Wait for the request to finish */
-    acb->finished = &finished;
-    while (!finished) {
-        qemu_aio_wait();
-    }
-}
-
-static const AIOCBInfo qed_aiocb_info = {
-    .aiocb_size         = sizeof(QEDAIOCB),
-    .cancel             = qed_aio_cancel,
-};
-
-static int bdrv_qed_probe(const uint8_t *buf, int buf_size,
-                          const char *filename)
-{
-    const QEDHeader *header = (const QEDHeader *)buf;
-
-    if (buf_size < sizeof(*header)) {
-        return 0;
-    }
-    if (le32_to_cpu(header->magic) != QED_MAGIC) {
-        return 0;
-    }
-    return 100;
-}
-
-/**
- * Check whether an image format is raw
- *
- * @fmt:    Backing file format, may be NULL
- */
-static bool qed_fmt_is_raw(const char *fmt)
-{
-    return fmt && strcmp(fmt, "raw") == 0;
-}
-
-static void qed_header_le_to_cpu(const QEDHeader *le, QEDHeader *cpu)
-{
-    cpu->magic = le32_to_cpu(le->magic);
-    cpu->cluster_size = le32_to_cpu(le->cluster_size);
-    cpu->table_size = le32_to_cpu(le->table_size);
-    cpu->header_size = le32_to_cpu(le->header_size);
-    cpu->features = le64_to_cpu(le->features);
-    cpu->compat_features = le64_to_cpu(le->compat_features);
-    cpu->autoclear_features = le64_to_cpu(le->autoclear_features);
-    cpu->l1_table_offset = le64_to_cpu(le->l1_table_offset);
-    cpu->image_size = le64_to_cpu(le->image_size);
-    cpu->backing_filename_offset = le32_to_cpu(le->backing_filename_offset);
-    cpu->backing_filename_size = le32_to_cpu(le->backing_filename_size);
-}
-
-static void qed_header_cpu_to_le(const QEDHeader *cpu, QEDHeader *le)
-{
-    le->magic = cpu_to_le32(cpu->magic);
-    le->cluster_size = cpu_to_le32(cpu->cluster_size);
-    le->table_size = cpu_to_le32(cpu->table_size);
-    le->header_size = cpu_to_le32(cpu->header_size);
-    le->features = cpu_to_le64(cpu->features);
-    le->compat_features = cpu_to_le64(cpu->compat_features);
-    le->autoclear_features = cpu_to_le64(cpu->autoclear_features);
-    le->l1_table_offset = cpu_to_le64(cpu->l1_table_offset);
-    le->image_size = cpu_to_le64(cpu->image_size);
-    le->backing_filename_offset = cpu_to_le32(cpu->backing_filename_offset);
-    le->backing_filename_size = cpu_to_le32(cpu->backing_filename_size);
-}
-
-int qed_write_header_sync(BDRVQEDState *s)
-{
-    QEDHeader le;
-    int ret;
-
-    qed_header_cpu_to_le(&s->header, &le);
-    ret = bdrv_pwrite(s->bs->file, 0, &le, sizeof(le));
-    if (ret != sizeof(le)) {
-        return ret;
-    }
-    return 0;
-}
-
-typedef struct {
-    GenericCB gencb;
-    BDRVQEDState *s;
-    struct iovec iov;
-    QEMUIOVector qiov;
-    int nsectors;
-    uint8_t *buf;
-} QEDWriteHeaderCB;
-
-static void qed_write_header_cb(void *opaque, int ret)
-{
-    QEDWriteHeaderCB *write_header_cb = opaque;
-
-    qemu_vfree(write_header_cb->buf);
-    gencb_complete(write_header_cb, ret);
-}
-
-static void qed_write_header_read_cb(void *opaque, int ret)
-{
-    QEDWriteHeaderCB *write_header_cb = opaque;
-    BDRVQEDState *s = write_header_cb->s;
-
-    if (ret) {
-        qed_write_header_cb(write_header_cb, ret);
-        return;
-    }
-
-    /* Update header */
-    qed_header_cpu_to_le(&s->header, (QEDHeader *)write_header_cb->buf);
-
-    bdrv_aio_writev(s->bs->file, 0, &write_header_cb->qiov,
-                    write_header_cb->nsectors, qed_write_header_cb,
-                    write_header_cb);
-}
-
-/**
- * Update header in-place (does not rewrite backing filename or other strings)
- *
- * This function only updates known header fields in-place and does not affect
- * extra data after the QED header.
- */
-static void qed_write_header(BDRVQEDState *s, BlockDriverCompletionFunc cb,
-                             void *opaque)
-{
-    /* We must write full sectors for O_DIRECT but cannot necessarily generate
-     * the data following the header if an unrecognized compat feature is
-     * active.  Therefore, first read the sectors containing the header, update
-     * them, and write back.
-     */
-
-    int nsectors = (sizeof(QEDHeader) + BDRV_SECTOR_SIZE - 1) /
-                   BDRV_SECTOR_SIZE;
-    size_t len = nsectors * BDRV_SECTOR_SIZE;
-    QEDWriteHeaderCB *write_header_cb = gencb_alloc(sizeof(*write_header_cb),
-                                                    cb, opaque);
-
-    write_header_cb->s = s;
-    write_header_cb->nsectors = nsectors;
-    write_header_cb->buf = qemu_blockalign(s->bs, len);
-    write_header_cb->iov.iov_base = write_header_cb->buf;
-    write_header_cb->iov.iov_len = len;
-    qemu_iovec_init_external(&write_header_cb->qiov, &write_header_cb->iov, 1);
-
-    bdrv_aio_readv(s->bs->file, 0, &write_header_cb->qiov, nsectors,
-                   qed_write_header_read_cb, write_header_cb);
-}
-
-static uint64_t qed_max_image_size(uint32_t cluster_size, uint32_t table_size)
-{
-    uint64_t table_entries;
-    uint64_t l2_size;
-
-    table_entries = (table_size * cluster_size) / sizeof(uint64_t);
-    l2_size = table_entries * cluster_size;
-
-    return l2_size * table_entries;
-}
-
-static bool qed_is_cluster_size_valid(uint32_t cluster_size)
-{
-    if (cluster_size < QED_MIN_CLUSTER_SIZE ||
-        cluster_size > QED_MAX_CLUSTER_SIZE) {
-        return false;
-    }
-    if (cluster_size & (cluster_size - 1)) {
-        return false; /* not power of 2 */
-    }
-    return true;
-}
-
-static bool qed_is_table_size_valid(uint32_t table_size)
-{
-    if (table_size < QED_MIN_TABLE_SIZE ||
-        table_size > QED_MAX_TABLE_SIZE) {
-        return false;
-    }
-    if (table_size & (table_size - 1)) {
-        return false; /* not power of 2 */
-    }
-    return true;
-}
-
-static bool qed_is_image_size_valid(uint64_t image_size, uint32_t cluster_size,
-                                    uint32_t table_size)
-{
-    if (image_size % BDRV_SECTOR_SIZE != 0) {
-        return false; /* not multiple of sector size */
-    }
-    if (image_size > qed_max_image_size(cluster_size, table_size)) {
-        return false; /* image is too large */
-    }
-    return true;
-}
-
-/**
- * Read a string of known length from the image file
- *
- * @file:       Image file
- * @offset:     File offset to start of string, in bytes
- * @n:          String length in bytes
- * @buf:        Destination buffer
- * @buflen:     Destination buffer length in bytes
- * @ret:        0 on success, -errno on failure
- *
- * The string is NUL-terminated.
- */
-static int qed_read_string(BlockDriverState *file, uint64_t offset, size_t n,
-                           char *buf, size_t buflen)
-{
-    int ret;
-    if (n >= buflen) {
-        return -EINVAL;
-    }
-    ret = bdrv_pread(file, offset, buf, n);
-    if (ret < 0) {
-        return ret;
-    }
-    buf[n] = '\0';
-    return 0;
-}
-
-/**
- * Allocate new clusters
- *
- * @s:          QED state
- * @n:          Number of contiguous clusters to allocate
- * @ret:        Offset of first allocated cluster
- *
- * This function only produces the offset where the new clusters should be
- * written.  It updates BDRVQEDState but does not make any changes to the image
- * file.
- */
-static uint64_t qed_alloc_clusters(BDRVQEDState *s, unsigned int n)
-{
-    uint64_t offset = s->file_size;
-    s->file_size += n * s->header.cluster_size;
-    return offset;
-}
-
-QEDTable *qed_alloc_table(BDRVQEDState *s)
-{
-    /* Honor O_DIRECT memory alignment requirements */
-    return qemu_blockalign(s->bs,
-                           s->header.cluster_size * s->header.table_size);
-}
-
-/**
- * Allocate a new zeroed L2 table
- */
-static CachedL2Table *qed_new_l2_table(BDRVQEDState *s)
-{
-    CachedL2Table *l2_table = qed_alloc_l2_cache_entry(&s->l2_cache);
-
-    l2_table->table = qed_alloc_table(s);
-    l2_table->offset = qed_alloc_clusters(s, s->header.table_size);
-
-    memset(l2_table->table->offsets, 0,
-           s->header.cluster_size * s->header.table_size);
-    return l2_table;
-}
-
-static void qed_aio_next_io(void *opaque, int ret);
-
-static void qed_plug_allocating_write_reqs(BDRVQEDState *s)
-{
-    assert(!s->allocating_write_reqs_plugged);
-
-    s->allocating_write_reqs_plugged = true;
-}
-
-static void qed_unplug_allocating_write_reqs(BDRVQEDState *s)
-{
-    QEDAIOCB *acb;
-
-    assert(s->allocating_write_reqs_plugged);
-
-    s->allocating_write_reqs_plugged = false;
-
-    acb = QSIMPLEQ_FIRST(&s->allocating_write_reqs);
-    if (acb) {
-        qed_aio_next_io(acb, 0);
-    }
-}
-
-static void qed_finish_clear_need_check(void *opaque, int ret)
-{
-    /* Do nothing */
-}
-
-static void qed_flush_after_clear_need_check(void *opaque, int ret)
-{
-    BDRVQEDState *s = opaque;
-
-    bdrv_aio_flush(s->bs, qed_finish_clear_need_check, s);
-
-    /* No need to wait until flush completes */
-    qed_unplug_allocating_write_reqs(s);
-}
-
-static void qed_clear_need_check(void *opaque, int ret)
-{
-    BDRVQEDState *s = opaque;
-
-    if (ret) {
-        qed_unplug_allocating_write_reqs(s);
-        return;
-    }
-
-    s->header.features &= ~QED_F_NEED_CHECK;
-    qed_write_header(s, qed_flush_after_clear_need_check, s);
-}
-
-static void qed_need_check_timer_cb(void *opaque)
-{
-    BDRVQEDState *s = opaque;
-
-    /* The timer should only fire when allocating writes have drained */
-    assert(!QSIMPLEQ_FIRST(&s->allocating_write_reqs));
-
-    trace_qed_need_check_timer_cb(s);
-
-    qed_plug_allocating_write_reqs(s);
-
-    /* Ensure writes are on disk before clearing flag */
-    bdrv_aio_flush(s->bs, qed_clear_need_check, s);
-}
-
-static void qed_start_need_check_timer(BDRVQEDState *s)
-{
-    trace_qed_start_need_check_timer(s);
-
-    /* Use vm_clock so we don't alter the image file while suspended for
-     * migration.
-     */
-    qemu_mod_timer(s->need_check_timer, qemu_get_clock_ns(vm_clock) +
-                   get_ticks_per_sec() * QED_NEED_CHECK_TIMEOUT);
-}
-
-/* It's okay to call this multiple times or when no timer is started */
-static void qed_cancel_need_check_timer(BDRVQEDState *s)
-{
-    trace_qed_cancel_need_check_timer(s);
-    qemu_del_timer(s->need_check_timer);
-}
-
-static void bdrv_qed_rebind(BlockDriverState *bs)
-{
-    BDRVQEDState *s = bs->opaque;
-    s->bs = bs;
-}
-
-static int bdrv_qed_open(BlockDriverState *bs, QDict *options, int flags)
-{
-    BDRVQEDState *s = bs->opaque;
-    QEDHeader le_header;
-    int64_t file_size;
-    int ret;
-
-    s->bs = bs;
-    QSIMPLEQ_INIT(&s->allocating_write_reqs);
-
-    ret = bdrv_pread(bs->file, 0, &le_header, sizeof(le_header));
-    if (ret < 0) {
-        return ret;
-    }
-    qed_header_le_to_cpu(&le_header, &s->header);
-
-    if (s->header.magic != QED_MAGIC) {
-        return -EMEDIUMTYPE;
-    }
-    if (s->header.features & ~QED_FEATURE_MASK) {
-        /* image uses unsupported feature bits */
-        char buf[64];
-        snprintf(buf, sizeof(buf), "%" PRIx64,
-            s->header.features & ~QED_FEATURE_MASK);
-        qerror_report(QERR_UNKNOWN_BLOCK_FORMAT_FEATURE,
-            bs->device_name, "QED", buf);
-        return -ENOTSUP;
-    }
-    if (!qed_is_cluster_size_valid(s->header.cluster_size)) {
-        return -EINVAL;
-    }
-
-    /* Round down file size to the last cluster */
-    file_size = bdrv_getlength(bs->file);
-    if (file_size < 0) {
-        return file_size;
-    }
-    s->file_size = qed_start_of_cluster(s, file_size);
-
-    if (!qed_is_table_size_valid(s->header.table_size)) {
-        return -EINVAL;
-    }
-    if (!qed_is_image_size_valid(s->header.image_size,
-                                 s->header.cluster_size,
-                                 s->header.table_size)) {
-        return -EINVAL;
-    }
-    if (!qed_check_table_offset(s, s->header.l1_table_offset)) {
-        return -EINVAL;
-    }
-
-    s->table_nelems = (s->header.cluster_size * s->header.table_size) /
-                      sizeof(uint64_t);
-    s->l2_shift = ffs(s->header.cluster_size) - 1;
-    s->l2_mask = s->table_nelems - 1;
-    s->l1_shift = s->l2_shift + ffs(s->table_nelems) - 1;
-
-    if ((s->header.features & QED_F_BACKING_FILE)) {
-        if ((uint64_t)s->header.backing_filename_offset +
-            s->header.backing_filename_size >
-            s->header.cluster_size * s->header.header_size) {
-            return -EINVAL;
-        }
-
-        ret = qed_read_string(bs->file, s->header.backing_filename_offset,
-                              s->header.backing_filename_size, bs->backing_file,
-                              sizeof(bs->backing_file));
-        if (ret < 0) {
-            return ret;
-        }
-
-        if (s->header.features & QED_F_BACKING_FORMAT_NO_PROBE) {
-            pstrcpy(bs->backing_format, sizeof(bs->backing_format), "raw");
-        }
-    }
-
-    /* Reset unknown autoclear feature bits.  This is a backwards
-     * compatibility mechanism that allows images to be opened by older
-     * programs, which "knock out" unknown feature bits.  When an image is
-     * opened by a newer program again it can detect that the autoclear
-     * feature is no longer valid.
-     */
-    if ((s->header.autoclear_features & ~QED_AUTOCLEAR_FEATURE_MASK) != 0 &&
-        !bdrv_is_read_only(bs->file) && !(flags & BDRV_O_INCOMING)) {
-        s->header.autoclear_features &= QED_AUTOCLEAR_FEATURE_MASK;
-
-        ret = qed_write_header_sync(s);
-        if (ret) {
-            return ret;
-        }
-
-        /* From here on only known autoclear feature bits are valid */
-        bdrv_flush(bs->file);
-    }
-
-    s->l1_table = qed_alloc_table(s);
-    qed_init_l2_cache(&s->l2_cache);
-
-    ret = qed_read_l1_table_sync(s);
-    if (ret) {
-        goto out;
-    }
-
-    /* If image was not closed cleanly, check consistency */
-    if (!(flags & BDRV_O_CHECK) && (s->header.features & QED_F_NEED_CHECK)) {
-        /* Read-only images cannot be fixed.  There is no risk of corruption
-         * since write operations are not possible.  Therefore, allow
-         * potentially inconsistent images to be opened read-only.  This can
-         * aid data recovery from an otherwise inconsistent image.
-         */
-        if (!bdrv_is_read_only(bs->file) &&
-            !(flags & BDRV_O_INCOMING)) {
-            BdrvCheckResult result = {0};
-
-            ret = qed_check(s, &result, true);
-            if (ret) {
-                goto out;
-            }
-        }
-    }
-
-    s->need_check_timer = qemu_new_timer_ns(vm_clock,
-                                            qed_need_check_timer_cb, s);
-
-out:
-    if (ret) {
-        qed_free_l2_cache(&s->l2_cache);
-        qemu_vfree(s->l1_table);
-    }
-    return ret;
-}
-
-/* We have nothing to do for QED reopen, stubs just return
- * success */
-static int bdrv_qed_reopen_prepare(BDRVReopenState *state,
-                                   BlockReopenQueue *queue, Error **errp)
-{
-    return 0;
-}
-
-static void bdrv_qed_close(BlockDriverState *bs)
-{
-    BDRVQEDState *s = bs->opaque;
-
-    qed_cancel_need_check_timer(s);
-    qemu_free_timer(s->need_check_timer);
-
-    /* Ensure writes reach stable storage */
-    bdrv_flush(bs->file);
-
-    /* Clean shutdown, no check required on next open */
-    if (s->header.features & QED_F_NEED_CHECK) {
-        s->header.features &= ~QED_F_NEED_CHECK;
-        qed_write_header_sync(s);
-    }
-
-    qed_free_l2_cache(&s->l2_cache);
-    qemu_vfree(s->l1_table);
-}
-
-static int qed_create(const char *filename, uint32_t cluster_size,
-                      uint64_t image_size, uint32_t table_size,
-                      const char *backing_file, const char *backing_fmt)
-{
-    QEDHeader header = {
-        .magic = QED_MAGIC,
-        .cluster_size = cluster_size,
-        .table_size = table_size,
-        .header_size = 1,
-        .features = 0,
-        .compat_features = 0,
-        .l1_table_offset = cluster_size,
-        .image_size = image_size,
-    };
-    QEDHeader le_header;
-    uint8_t *l1_table = NULL;
-    size_t l1_size = header.cluster_size * header.table_size;
-    int ret = 0;
-    BlockDriverState *bs = NULL;
-
-    ret = bdrv_create_file(filename, NULL);
-    if (ret < 0) {
-        return ret;
-    }
-
-    ret = bdrv_file_open(&bs, filename, NULL, BDRV_O_RDWR | BDRV_O_CACHE_WB);
-    if (ret < 0) {
-        return ret;
-    }
-
-    /* File must start empty and grow, check truncate is supported */
-    ret = bdrv_truncate(bs, 0);
-    if (ret < 0) {
-        goto out;
-    }
-
-    if (backing_file) {
-        header.features |= QED_F_BACKING_FILE;
-        header.backing_filename_offset = sizeof(le_header);
-        header.backing_filename_size = strlen(backing_file);
-
-        if (qed_fmt_is_raw(backing_fmt)) {
-            header.features |= QED_F_BACKING_FORMAT_NO_PROBE;
-        }
-    }
-
-    qed_header_cpu_to_le(&header, &le_header);
-    ret = bdrv_pwrite(bs, 0, &le_header, sizeof(le_header));
-    if (ret < 0) {
-        goto out;
-    }
-    ret = bdrv_pwrite(bs, sizeof(le_header), backing_file,
-                      header.backing_filename_size);
-    if (ret < 0) {
-        goto out;
-    }
-
-    l1_table = g_malloc0(l1_size);
-    ret = bdrv_pwrite(bs, header.l1_table_offset, l1_table, l1_size);
-    if (ret < 0) {
-        goto out;
-    }
-
-    ret = 0; /* success */
-out:
-    g_free(l1_table);
-    bdrv_delete(bs);
-    return ret;
-}
-
-static int bdrv_qed_create(const char *filename, QEMUOptionParameter *options)
-{
-    uint64_t image_size = 0;
-    uint32_t cluster_size = QED_DEFAULT_CLUSTER_SIZE;
-    uint32_t table_size = QED_DEFAULT_TABLE_SIZE;
-    const char *backing_file = NULL;
-    const char *backing_fmt = NULL;
-
-    while (options && options->name) {
-        if (!strcmp(options->name, BLOCK_OPT_SIZE)) {
-            image_size = options->value.n;
-        } else if (!strcmp(options->name, BLOCK_OPT_BACKING_FILE)) {
-            backing_file = options->value.s;
-        } else if (!strcmp(options->name, BLOCK_OPT_BACKING_FMT)) {
-            backing_fmt = options->value.s;
-        } else if (!strcmp(options->name, BLOCK_OPT_CLUSTER_SIZE)) {
-            if (options->value.n) {
-                cluster_size = options->value.n;
-            }
-        } else if (!strcmp(options->name, BLOCK_OPT_TABLE_SIZE)) {
-            if (options->value.n) {
-                table_size = options->value.n;
-            }
-        }
-        options++;
-    }
-
-    if (!qed_is_cluster_size_valid(cluster_size)) {
-        fprintf(stderr, "QED cluster size must be within range [%u, %u] and power of 2\n",
-                QED_MIN_CLUSTER_SIZE, QED_MAX_CLUSTER_SIZE);
-        return -EINVAL;
-    }
-    if (!qed_is_table_size_valid(table_size)) {
-        fprintf(stderr, "QED table size must be within range [%u, %u] and power of 2\n",
-                QED_MIN_TABLE_SIZE, QED_MAX_TABLE_SIZE);
-        return -EINVAL;
-    }
-    if (!qed_is_image_size_valid(image_size, cluster_size, table_size)) {
-        fprintf(stderr, "QED image size must be a non-zero multiple of "
-                        "cluster size and less than %" PRIu64 " bytes\n",
-                qed_max_image_size(cluster_size, table_size));
-        return -EINVAL;
-    }
-
-    return qed_create(filename, cluster_size, image_size, table_size,
-                      backing_file, backing_fmt);
-}
-
-typedef struct {
-    Coroutine *co;
-    int is_allocated;
-    int *pnum;
-} QEDIsAllocatedCB;
-
-static void qed_is_allocated_cb(void *opaque, int ret, uint64_t offset, size_t len)
-{
-    QEDIsAllocatedCB *cb = opaque;
-    *cb->pnum = len / BDRV_SECTOR_SIZE;
-    cb->is_allocated = (ret == QED_CLUSTER_FOUND || ret == QED_CLUSTER_ZERO);
-    if (cb->co) {
-        qemu_coroutine_enter(cb->co, NULL);
-    }
-}
-
-static int coroutine_fn bdrv_qed_co_is_allocated(BlockDriverState *bs,
-                                                 int64_t sector_num,
-                                                 int nb_sectors, int *pnum)
-{
-    BDRVQEDState *s = bs->opaque;
-    uint64_t pos = (uint64_t)sector_num * BDRV_SECTOR_SIZE;
-    size_t len = (size_t)nb_sectors * BDRV_SECTOR_SIZE;
-    QEDIsAllocatedCB cb = {
-        .is_allocated = -1,
-        .pnum = pnum,
-    };
-    QEDRequest request = { .l2_table = NULL };
-
-    qed_find_cluster(s, &request, pos, len, qed_is_allocated_cb, &cb);
-
-    /* Now sleep if the callback wasn't invoked immediately */
-    while (cb.is_allocated == -1) {
-        cb.co = qemu_coroutine_self();
-        qemu_coroutine_yield();
-    }
-
-    qed_unref_l2_cache_entry(request.l2_table);
-
-    return cb.is_allocated;
-}
-
-static int bdrv_qed_make_empty(BlockDriverState *bs)
-{
-    return -ENOTSUP;
-}
-
-static BDRVQEDState *acb_to_s(QEDAIOCB *acb)
-{
-    return acb->common.bs->opaque;
-}
-
-/**
- * Read from the backing file or zero-fill if no backing file
- *
- * @s:          QED state
- * @pos:        Byte position in device
- * @qiov:       Destination I/O vector
- * @cb:         Completion function
- * @opaque:     User data for completion function
- *
- * This function reads qiov->size bytes starting at pos from the backing file.
- * If there is no backing file then zeroes are read.
- */
-static void qed_read_backing_file(BDRVQEDState *s, uint64_t pos,
-                                  QEMUIOVector *qiov,
-                                  BlockDriverCompletionFunc *cb, void *opaque)
-{
-    uint64_t backing_length = 0;
-    size_t size;
-
-    /* If there is a backing file, get its length.  Treat the absence of a
-     * backing file like a zero length backing file.
-     */
-    if (s->bs->backing_hd) {
-        int64_t l = bdrv_getlength(s->bs->backing_hd);
-        if (l < 0) {
-            cb(opaque, l);
-            return;
-        }
-        backing_length = l;
-    }
-
-    /* Zero all sectors if reading beyond the end of the backing file */
-    if (pos >= backing_length ||
-        pos + qiov->size > backing_length) {
-        qemu_iovec_memset(qiov, 0, 0, qiov->size);
-    }
-
-    /* Complete now if there are no backing file sectors to read */
-    if (pos >= backing_length) {
-        cb(opaque, 0);
-        return;
-    }
-
-    /* If the read straddles the end of the backing file, shorten it */
-    size = MIN((uint64_t)backing_length - pos, qiov->size);
-
-    BLKDBG_EVENT(s->bs->file, BLKDBG_READ_BACKING_AIO);
-    bdrv_aio_readv(s->bs->backing_hd, pos / BDRV_SECTOR_SIZE,
-                   qiov, size / BDRV_SECTOR_SIZE, cb, opaque);
-}
-
-typedef struct {
-    GenericCB gencb;
-    BDRVQEDState *s;
-    QEMUIOVector qiov;
-    struct iovec iov;
-    uint64_t offset;
-} CopyFromBackingFileCB;
-
-static void qed_copy_from_backing_file_cb(void *opaque, int ret)
-{
-    CopyFromBackingFileCB *copy_cb = opaque;
-    qemu_vfree(copy_cb->iov.iov_base);
-    gencb_complete(&copy_cb->gencb, ret);
-}
-
-static void qed_copy_from_backing_file_write(void *opaque, int ret)
-{
-    CopyFromBackingFileCB *copy_cb = opaque;
-    BDRVQEDState *s = copy_cb->s;
-
-    if (ret) {
-        qed_copy_from_backing_file_cb(copy_cb, ret);
-        return;
-    }
-
-    BLKDBG_EVENT(s->bs->file, BLKDBG_COW_WRITE);
-    bdrv_aio_writev(s->bs->file, copy_cb->offset / BDRV_SECTOR_SIZE,
-                    &copy_cb->qiov, copy_cb->qiov.size / BDRV_SECTOR_SIZE,
-                    qed_copy_from_backing_file_cb, copy_cb);
-}
-
-/**
- * Copy data from backing file into the image
- *
- * @s:          QED state
- * @pos:        Byte position in device
- * @len:        Number of bytes
- * @offset:     Byte offset in image file
- * @cb:         Completion function
- * @opaque:     User data for completion function
- */
-static void qed_copy_from_backing_file(BDRVQEDState *s, uint64_t pos,
-                                       uint64_t len, uint64_t offset,
-                                       BlockDriverCompletionFunc *cb,
-                                       void *opaque)
-{
-    CopyFromBackingFileCB *copy_cb;
-
-    /* Skip copy entirely if there is no work to do */
-    if (len == 0) {
-        cb(opaque, 0);
-        return;
-    }
-
-    copy_cb = gencb_alloc(sizeof(*copy_cb), cb, opaque);
-    copy_cb->s = s;
-    copy_cb->offset = offset;
-    copy_cb->iov.iov_base = qemu_blockalign(s->bs, len);
-    copy_cb->iov.iov_len = len;
-    qemu_iovec_init_external(&copy_cb->qiov, &copy_cb->iov, 1);
-
-    qed_read_backing_file(s, pos, &copy_cb->qiov,
-                          qed_copy_from_backing_file_write, copy_cb);
-}
-
-/**
- * Link one or more contiguous clusters into a table
- *
- * @s:              QED state
- * @table:          L2 table
- * @index:          First cluster index
- * @n:              Number of contiguous clusters
- * @cluster:        First cluster offset
- *
- * The cluster offset may be an allocated byte offset in the image file, the
- * zero cluster marker, or the unallocated cluster marker.
- */
-static void qed_update_l2_table(BDRVQEDState *s, QEDTable *table, int index,
-                                unsigned int n, uint64_t cluster)
-{
-    int i;
-    for (i = index; i < index + n; i++) {
-        table->offsets[i] = cluster;
-        if (!qed_offset_is_unalloc_cluster(cluster) &&
-            !qed_offset_is_zero_cluster(cluster)) {
-            cluster += s->header.cluster_size;
-        }
-    }
-}
-
-static void qed_aio_complete_bh(void *opaque)
-{
-    QEDAIOCB *acb = opaque;
-    BlockDriverCompletionFunc *cb = acb->common.cb;
-    void *user_opaque = acb->common.opaque;
-    int ret = acb->bh_ret;
-    bool *finished = acb->finished;
-
-    qemu_bh_delete(acb->bh);
-    qemu_aio_release(acb);
-
-    /* Invoke callback */
-    cb(user_opaque, ret);
-
-    /* Signal cancel completion */
-    if (finished) {
-        *finished = true;
-    }
-}
-
-static void qed_aio_complete(QEDAIOCB *acb, int ret)
-{
-    BDRVQEDState *s = acb_to_s(acb);
-
-    trace_qed_aio_complete(s, acb, ret);
-
-    /* Free resources */
-    qemu_iovec_destroy(&acb->cur_qiov);
-    qed_unref_l2_cache_entry(acb->request.l2_table);
-
-    /* Free the buffer we may have allocated for zero writes */
-    if (acb->flags & QED_AIOCB_ZERO) {
-        qemu_vfree(acb->qiov->iov[0].iov_base);
-        acb->qiov->iov[0].iov_base = NULL;
-    }
-
-    /* Arrange for a bh to invoke the completion function */
-    acb->bh_ret = ret;
-    acb->bh = qemu_bh_new(qed_aio_complete_bh, acb);
-    qemu_bh_schedule(acb->bh);
-
-    /* Start next allocating write request waiting behind this one.  Note that
-     * requests enqueue themselves when they first hit an unallocated cluster
-     * but they wait until the entire request is finished before waking up the
-     * next request in the queue.  This ensures that we don't cycle through
-     * requests multiple times but rather finish one at a time completely.
-     */
-    if (acb == QSIMPLEQ_FIRST(&s->allocating_write_reqs)) {
-        QSIMPLEQ_REMOVE_HEAD(&s->allocating_write_reqs, next);
-        acb = QSIMPLEQ_FIRST(&s->allocating_write_reqs);
-        if (acb) {
-            qed_aio_next_io(acb, 0);
-        } else if (s->header.features & QED_F_NEED_CHECK) {
-            qed_start_need_check_timer(s);
-        }
-    }
-}
-
-/**
- * Commit the current L2 table to the cache
- */
-static void qed_commit_l2_update(void *opaque, int ret)
-{
-    QEDAIOCB *acb = opaque;
-    BDRVQEDState *s = acb_to_s(acb);
-    CachedL2Table *l2_table = acb->request.l2_table;
-    uint64_t l2_offset = l2_table->offset;
-
-    qed_commit_l2_cache_entry(&s->l2_cache, l2_table);
-
-    /* This is guaranteed to succeed because we just committed the entry to the
-     * cache.
-     */
-    acb->request.l2_table = qed_find_l2_cache_entry(&s->l2_cache, l2_offset);
-    assert(acb->request.l2_table != NULL);
-
-    qed_aio_next_io(opaque, ret);
-}
-
-/**
- * Update L1 table with new L2 table offset and write it out
- */
-static void qed_aio_write_l1_update(void *opaque, int ret)
-{
-    QEDAIOCB *acb = opaque;
-    BDRVQEDState *s = acb_to_s(acb);
-    int index;
-
-    if (ret) {
-        qed_aio_complete(acb, ret);
-        return;
-    }
-
-    index = qed_l1_index(s, acb->cur_pos);
-    s->l1_table->offsets[index] = acb->request.l2_table->offset;
-
-    qed_write_l1_table(s, index, 1, qed_commit_l2_update, acb);
-}
-
-/**
- * Update L2 table with new cluster offsets and write them out
- */
-static void qed_aio_write_l2_update(QEDAIOCB *acb, int ret, uint64_t offset)
-{
-    BDRVQEDState *s = acb_to_s(acb);
-    bool need_alloc = acb->find_cluster_ret == QED_CLUSTER_L1;
-    int index;
-
-    if (ret) {
-        goto err;
-    }
-
-    if (need_alloc) {
-        qed_unref_l2_cache_entry(acb->request.l2_table);
-        acb->request.l2_table = qed_new_l2_table(s);
-    }
-
-    index = qed_l2_index(s, acb->cur_pos);
-    qed_update_l2_table(s, acb->request.l2_table->table, index, acb->cur_nclusters,
-                         offset);
-
-    if (need_alloc) {
-        /* Write out the whole new L2 table */
-        qed_write_l2_table(s, &acb->request, 0, s->table_nelems, true,
-                            qed_aio_write_l1_update, acb);
-    } else {
-        /* Write out only the updated part of the L2 table */
-        qed_write_l2_table(s, &acb->request, index, acb->cur_nclusters, false,
-                            qed_aio_next_io, acb);
-    }
-    return;
-
-err:
-    qed_aio_complete(acb, ret);
-}
-
-static void qed_aio_write_l2_update_cb(void *opaque, int ret)
-{
-    QEDAIOCB *acb = opaque;
-    qed_aio_write_l2_update(acb, ret, acb->cur_cluster);
-}
-
-/**
- * Flush new data clusters before updating the L2 table
- *
- * This flush is necessary when a backing file is in use.  A crash during an
- * allocating write could result in empty clusters in the image.  If the write
- * only touched a subregion of the cluster, then backing image sectors have
- * been lost in the untouched region.  The solution is to flush after writing a
- * new data cluster and before updating the L2 table.
- */
-static void qed_aio_write_flush_before_l2_update(void *opaque, int ret)
-{
-    QEDAIOCB *acb = opaque;
-    BDRVQEDState *s = acb_to_s(acb);
-
-    if (!bdrv_aio_flush(s->bs->file, qed_aio_write_l2_update_cb, opaque)) {
-        qed_aio_complete(acb, -EIO);
-    }
-}
-
-/**
- * Write data to the image file
- */
-static void qed_aio_write_main(void *opaque, int ret)
-{
-    QEDAIOCB *acb = opaque;
-    BDRVQEDState *s = acb_to_s(acb);
-    uint64_t offset = acb->cur_cluster +
-                      qed_offset_into_cluster(s, acb->cur_pos);
-    BlockDriverCompletionFunc *next_fn;
-
-    trace_qed_aio_write_main(s, acb, ret, offset, acb->cur_qiov.size);
-
-    if (ret) {
-        qed_aio_complete(acb, ret);
-        return;
-    }
-
-    if (acb->find_cluster_ret == QED_CLUSTER_FOUND) {
-        next_fn = qed_aio_next_io;
-    } else {
-        if (s->bs->backing_hd) {
-            next_fn = qed_aio_write_flush_before_l2_update;
-        } else {
-            next_fn = qed_aio_write_l2_update_cb;
-        }
-    }
-
-    BLKDBG_EVENT(s->bs->file, BLKDBG_WRITE_AIO);
-    bdrv_aio_writev(s->bs->file, offset / BDRV_SECTOR_SIZE,
-                    &acb->cur_qiov, acb->cur_qiov.size / BDRV_SECTOR_SIZE,
-                    next_fn, acb);
-}
-
-/**
- * Populate back untouched region of new data cluster
- */
-static void qed_aio_write_postfill(void *opaque, int ret)
-{
-    QEDAIOCB *acb = opaque;
-    BDRVQEDState *s = acb_to_s(acb);
-    uint64_t start = acb->cur_pos + acb->cur_qiov.size;
-    uint64_t len =
-        qed_start_of_cluster(s, start + s->header.cluster_size - 1) - start;
-    uint64_t offset = acb->cur_cluster +
-                      qed_offset_into_cluster(s, acb->cur_pos) +
-                      acb->cur_qiov.size;
-
-    if (ret) {
-        qed_aio_complete(acb, ret);
-        return;
-    }
-
-    trace_qed_aio_write_postfill(s, acb, start, len, offset);
-    qed_copy_from_backing_file(s, start, len, offset,
-                                qed_aio_write_main, acb);
-}
-
-/**
- * Populate front untouched region of new data cluster
- */
-static void qed_aio_write_prefill(void *opaque, int ret)
-{
-    QEDAIOCB *acb = opaque;
-    BDRVQEDState *s = acb_to_s(acb);
-    uint64_t start = qed_start_of_cluster(s, acb->cur_pos);
-    uint64_t len = qed_offset_into_cluster(s, acb->cur_pos);
-
-    trace_qed_aio_write_prefill(s, acb, start, len, acb->cur_cluster);
-    qed_copy_from_backing_file(s, start, len, acb->cur_cluster,
-                                qed_aio_write_postfill, acb);
-}
-
-/**
- * Check if the QED_F_NEED_CHECK bit should be set during allocating write
- */
-static bool qed_should_set_need_check(BDRVQEDState *s)
-{
-    /* The flush before L2 update path ensures consistency */
-    if (s->bs->backing_hd) {
-        return false;
-    }
-
-    return !(s->header.features & QED_F_NEED_CHECK);
-}
-
-static void qed_aio_write_zero_cluster(void *opaque, int ret)
-{
-    QEDAIOCB *acb = opaque;
-
-    if (ret) {
-        qed_aio_complete(acb, ret);
-        return;
-    }
-
-    qed_aio_write_l2_update(acb, 0, 1);
-}
-
-/**
- * Write new data cluster
- *
- * @acb:        Write request
- * @len:        Length in bytes
- *
- * This path is taken when writing to previously unallocated clusters.
- */
-static void qed_aio_write_alloc(QEDAIOCB *acb, size_t len)
-{
-    BDRVQEDState *s = acb_to_s(acb);
-    BlockDriverCompletionFunc *cb;
-
-    /* Cancel timer when the first allocating request comes in */
-    if (QSIMPLEQ_EMPTY(&s->allocating_write_reqs)) {
-        qed_cancel_need_check_timer(s);
-    }
-
-    /* Freeze this request if another allocating write is in progress */
-    if (acb != QSIMPLEQ_FIRST(&s->allocating_write_reqs)) {
-        QSIMPLEQ_INSERT_TAIL(&s->allocating_write_reqs, acb, next);
-    }
-    if (acb != QSIMPLEQ_FIRST(&s->allocating_write_reqs) ||
-        s->allocating_write_reqs_plugged) {
-        return; /* wait for existing request to finish */
-    }
-
-    acb->cur_nclusters = qed_bytes_to_clusters(s,
-            qed_offset_into_cluster(s, acb->cur_pos) + len);
-    qemu_iovec_concat(&acb->cur_qiov, acb->qiov, acb->qiov_offset, len);
-
-    if (acb->flags & QED_AIOCB_ZERO) {
-        /* Skip ahead if the clusters are already zero */
-        if (acb->find_cluster_ret == QED_CLUSTER_ZERO) {
-            qed_aio_next_io(acb, 0);
-            return;
-        }
-
-        cb = qed_aio_write_zero_cluster;
-    } else {
-        cb = qed_aio_write_prefill;
-        acb->cur_cluster = qed_alloc_clusters(s, acb->cur_nclusters);
-    }
-
-    if (qed_should_set_need_check(s)) {
-        s->header.features |= QED_F_NEED_CHECK;
-        qed_write_header(s, cb, acb);
-    } else {
-        cb(acb, 0);
-    }
-}
-
-/**
- * Write data cluster in place
- *
- * @acb:        Write request
- * @offset:     Cluster offset in bytes
- * @len:        Length in bytes
- *
- * This path is taken when writing to already allocated clusters.
- */
-static void qed_aio_write_inplace(QEDAIOCB *acb, uint64_t offset, size_t len)
-{
-    /* Allocate buffer for zero writes */
-    if (acb->flags & QED_AIOCB_ZERO) {
-        struct iovec *iov = acb->qiov->iov;
-
-        if (!iov->iov_base) {
-            iov->iov_base = qemu_blockalign(acb->common.bs, iov->iov_len);
-            memset(iov->iov_base, 0, iov->iov_len);
-        }
-    }
-
-    /* Calculate the I/O vector */
-    acb->cur_cluster = offset;
-    qemu_iovec_concat(&acb->cur_qiov, acb->qiov, acb->qiov_offset, len);
-
-    /* Do the actual write */
-    qed_aio_write_main(acb, 0);
-}
-
-/**
- * Write data cluster
- *
- * @opaque:     Write request
- * @ret:        QED_CLUSTER_FOUND, QED_CLUSTER_L2, QED_CLUSTER_L1,
- *              or -errno
- * @offset:     Cluster offset in bytes
- * @len:        Length in bytes
- *
- * Callback from qed_find_cluster().
- */
-static void qed_aio_write_data(void *opaque, int ret,
-                               uint64_t offset, size_t len)
-{
-    QEDAIOCB *acb = opaque;
-
-    trace_qed_aio_write_data(acb_to_s(acb), acb, ret, offset, len);
-
-    acb->find_cluster_ret = ret;
-
-    switch (ret) {
-    case QED_CLUSTER_FOUND:
-        qed_aio_write_inplace(acb, offset, len);
-        break;
-
-    case QED_CLUSTER_L2:
-    case QED_CLUSTER_L1:
-    case QED_CLUSTER_ZERO:
-        qed_aio_write_alloc(acb, len);
-        break;
-
-    default:
-        qed_aio_complete(acb, ret);
-        break;
-    }
-}
-
-/**
- * Read data cluster
- *
- * @opaque:     Read request
- * @ret:        QED_CLUSTER_FOUND, QED_CLUSTER_L2, QED_CLUSTER_L1,
- *              or -errno
- * @offset:     Cluster offset in bytes
- * @len:        Length in bytes
- *
- * Callback from qed_find_cluster().
- */
-static void qed_aio_read_data(void *opaque, int ret,
-                              uint64_t offset, size_t len)
-{
-    QEDAIOCB *acb = opaque;
-    BDRVQEDState *s = acb_to_s(acb);
-    BlockDriverState *bs = acb->common.bs;
-
-    /* Adjust offset into cluster */
-    offset += qed_offset_into_cluster(s, acb->cur_pos);
-
-    trace_qed_aio_read_data(s, acb, ret, offset, len);
-
-    if (ret < 0) {
-        goto err;
-    }
-
-    qemu_iovec_concat(&acb->cur_qiov, acb->qiov, acb->qiov_offset, len);
-
-    /* Handle zero cluster and backing file reads */
-    if (ret == QED_CLUSTER_ZERO) {
-        qemu_iovec_memset(&acb->cur_qiov, 0, 0, acb->cur_qiov.size);
-        qed_aio_next_io(acb, 0);
-        return;
-    } else if (ret != QED_CLUSTER_FOUND) {
-        qed_read_backing_file(s, acb->cur_pos, &acb->cur_qiov,
-                              qed_aio_next_io, acb);
-        return;
-    }
-
-    BLKDBG_EVENT(bs->file, BLKDBG_READ_AIO);
-    bdrv_aio_readv(bs->file, offset / BDRV_SECTOR_SIZE,
-                   &acb->cur_qiov, acb->cur_qiov.size / BDRV_SECTOR_SIZE,
-                   qed_aio_next_io, acb);
-    return;
-
-err:
-    qed_aio_complete(acb, ret);
-}
-
-/**
- * Begin next I/O or complete the request
- */
-static void qed_aio_next_io(void *opaque, int ret)
-{
-    QEDAIOCB *acb = opaque;
-    BDRVQEDState *s = acb_to_s(acb);
-    QEDFindClusterFunc *io_fn = (acb->flags & QED_AIOCB_WRITE) ?
-                                qed_aio_write_data : qed_aio_read_data;
-
-    trace_qed_aio_next_io(s, acb, ret, acb->cur_pos + acb->cur_qiov.size);
-
-    /* Handle I/O error */
-    if (ret) {
-        qed_aio_complete(acb, ret);
-        return;
-    }
-
-    acb->qiov_offset += acb->cur_qiov.size;
-    acb->cur_pos += acb->cur_qiov.size;
-    qemu_iovec_reset(&acb->cur_qiov);
-
-    /* Complete request */
-    if (acb->cur_pos >= acb->end_pos) {
-        qed_aio_complete(acb, 0);
-        return;
-    }
-
-    /* Find next cluster and start I/O */
-    qed_find_cluster(s, &acb->request,
-                      acb->cur_pos, acb->end_pos - acb->cur_pos,
-                      io_fn, acb);
-}
-
-static BlockDriverAIOCB *qed_aio_setup(BlockDriverState *bs,
-                                       int64_t sector_num,
-                                       QEMUIOVector *qiov, int nb_sectors,
-                                       BlockDriverCompletionFunc *cb,
-                                       void *opaque, int flags)
-{
-    QEDAIOCB *acb = qemu_aio_get(&qed_aiocb_info, bs, cb, opaque);
-
-    trace_qed_aio_setup(bs->opaque, acb, sector_num, nb_sectors,
-                        opaque, flags);
-
-    acb->flags = flags;
-    acb->finished = NULL;
-    acb->qiov = qiov;
-    acb->qiov_offset = 0;
-    acb->cur_pos = (uint64_t)sector_num * BDRV_SECTOR_SIZE;
-    acb->end_pos = acb->cur_pos + nb_sectors * BDRV_SECTOR_SIZE;
-    acb->request.l2_table = NULL;
-    qemu_iovec_init(&acb->cur_qiov, qiov->niov);
-
-    /* Start request */
-    qed_aio_next_io(acb, 0);
-    return &acb->common;
-}
-
-static BlockDriverAIOCB *bdrv_qed_aio_readv(BlockDriverState *bs,
-                                            int64_t sector_num,
-                                            QEMUIOVector *qiov, int nb_sectors,
-                                            BlockDriverCompletionFunc *cb,
-                                            void *opaque)
-{
-    return qed_aio_setup(bs, sector_num, qiov, nb_sectors, cb, opaque, 0);
-}
-
-static BlockDriverAIOCB *bdrv_qed_aio_writev(BlockDriverState *bs,
-                                             int64_t sector_num,
-                                             QEMUIOVector *qiov, int nb_sectors,
-                                             BlockDriverCompletionFunc *cb,
-                                             void *opaque)
-{
-    return qed_aio_setup(bs, sector_num, qiov, nb_sectors, cb,
-                         opaque, QED_AIOCB_WRITE);
-}
-
-typedef struct {
-    Coroutine *co;
-    int ret;
-    bool done;
-} QEDWriteZeroesCB;
-
-static void coroutine_fn qed_co_write_zeroes_cb(void *opaque, int ret)
-{
-    QEDWriteZeroesCB *cb = opaque;
-
-    cb->done = true;
-    cb->ret = ret;
-    if (cb->co) {
-        qemu_coroutine_enter(cb->co, NULL);
-    }
-}
-
-static int coroutine_fn bdrv_qed_co_write_zeroes(BlockDriverState *bs,
-                                                 int64_t sector_num,
-                                                 int nb_sectors)
-{
-    BlockDriverAIOCB *blockacb;
-    BDRVQEDState *s = bs->opaque;
-    QEDWriteZeroesCB cb = { .done = false };
-    QEMUIOVector qiov;
-    struct iovec iov;
-
-    /* Refuse if there are untouched backing file sectors */
-    if (bs->backing_hd) {
-        if (qed_offset_into_cluster(s, sector_num * BDRV_SECTOR_SIZE) != 0) {
-            return -ENOTSUP;
-        }
-        if (qed_offset_into_cluster(s, nb_sectors * BDRV_SECTOR_SIZE) != 0) {
-            return -ENOTSUP;
-        }
-    }
-
-    /* Zero writes start without an I/O buffer.  If a buffer becomes necessary
-     * then it will be allocated during request processing.
-     */
-    iov.iov_base = NULL,
-    iov.iov_len  = nb_sectors * BDRV_SECTOR_SIZE,
-
-    qemu_iovec_init_external(&qiov, &iov, 1);
-    blockacb = qed_aio_setup(bs, sector_num, &qiov, nb_sectors,
-                             qed_co_write_zeroes_cb, &cb,
-                             QED_AIOCB_WRITE | QED_AIOCB_ZERO);
-    if (!blockacb) {
-        return -EIO;
-    }
-    if (!cb.done) {
-        cb.co = qemu_coroutine_self();
-        qemu_coroutine_yield();
-    }
-    assert(cb.done);
-    return cb.ret;
-}
-
-static int bdrv_qed_truncate(BlockDriverState *bs, int64_t offset)
-{
-    BDRVQEDState *s = bs->opaque;
-    uint64_t old_image_size;
-    int ret;
-
-    if (!qed_is_image_size_valid(offset, s->header.cluster_size,
-                                 s->header.table_size)) {
-        return -EINVAL;
-    }
-
-    /* Shrinking is currently not supported */
-    if ((uint64_t)offset < s->header.image_size) {
-        return -ENOTSUP;
-    }
-
-    old_image_size = s->header.image_size;
-    s->header.image_size = offset;
-    ret = qed_write_header_sync(s);
-    if (ret < 0) {
-        s->header.image_size = old_image_size;
-    }
-    return ret;
-}
-
-static int64_t bdrv_qed_getlength(BlockDriverState *bs)
-{
-    BDRVQEDState *s = bs->opaque;
-    return s->header.image_size;
-}
-
-static int bdrv_qed_get_info(BlockDriverState *bs, BlockDriverInfo *bdi)
-{
-    BDRVQEDState *s = bs->opaque;
-
-    memset(bdi, 0, sizeof(*bdi));
-    bdi->cluster_size = s->header.cluster_size;
-    bdi->is_dirty = s->header.features & QED_F_NEED_CHECK;
-    return 0;
-}
-
-static int bdrv_qed_change_backing_file(BlockDriverState *bs,
-                                        const char *backing_file,
-                                        const char *backing_fmt)
-{
-    BDRVQEDState *s = bs->opaque;
-    QEDHeader new_header, le_header;
-    void *buffer;
-    size_t buffer_len, backing_file_len;
-    int ret;
-
-    /* Refuse to set backing filename if unknown compat feature bits are
-     * active.  If the image uses an unknown compat feature then we may not
-     * know the layout of data following the header structure and cannot safely
-     * add a new string.
-     */
-    if (backing_file && (s->header.compat_features &
-                         ~QED_COMPAT_FEATURE_MASK)) {
-        return -ENOTSUP;
-    }
-
-    memcpy(&new_header, &s->header, sizeof(new_header));
-
-    new_header.features &= ~(QED_F_BACKING_FILE |
-                             QED_F_BACKING_FORMAT_NO_PROBE);
-
-    /* Adjust feature flags */
-    if (backing_file) {
-        new_header.features |= QED_F_BACKING_FILE;
-
-        if (qed_fmt_is_raw(backing_fmt)) {
-            new_header.features |= QED_F_BACKING_FORMAT_NO_PROBE;
-        }
-    }
-
-    /* Calculate new header size */
-    backing_file_len = 0;
-
-    if (backing_file) {
-        backing_file_len = strlen(backing_file);
-    }
-
-    buffer_len = sizeof(new_header);
-    new_header.backing_filename_offset = buffer_len;
-    new_header.backing_filename_size = backing_file_len;
-    buffer_len += backing_file_len;
-
-    /* Make sure we can rewrite header without failing */
-    if (buffer_len > new_header.header_size * new_header.cluster_size) {
-        return -ENOSPC;
-    }
-
-    /* Prepare new header */
-    buffer = g_malloc(buffer_len);
-
-    qed_header_cpu_to_le(&new_header, &le_header);
-    memcpy(buffer, &le_header, sizeof(le_header));
-    buffer_len = sizeof(le_header);
-
-    if (backing_file) {
-        memcpy(buffer + buffer_len, backing_file, backing_file_len);
-        buffer_len += backing_file_len;
-    }
-
-    /* Write new header */
-    ret = bdrv_pwrite_sync(bs->file, 0, buffer, buffer_len);
-    g_free(buffer);
-    if (ret == 0) {
-        memcpy(&s->header, &new_header, sizeof(new_header));
-    }
-    return ret;
-}
-
-static void bdrv_qed_invalidate_cache(BlockDriverState *bs)
-{
-    BDRVQEDState *s = bs->opaque;
-
-    bdrv_qed_close(bs);
-    memset(s, 0, sizeof(BDRVQEDState));
-    bdrv_qed_open(bs, NULL, bs->open_flags);
-}
-
-static int bdrv_qed_check(BlockDriverState *bs, BdrvCheckResult *result,
-                          BdrvCheckMode fix)
-{
-    BDRVQEDState *s = bs->opaque;
-
-    return qed_check(s, result, !!fix);
-}
-
-static QEMUOptionParameter qed_create_options[] = {
-    {
-        .name = BLOCK_OPT_SIZE,
-        .type = OPT_SIZE,
-        .help = "Virtual disk size (in bytes)"
-    }, {
-        .name = BLOCK_OPT_BACKING_FILE,
-        .type = OPT_STRING,
-        .help = "File name of a base image"
-    }, {
-        .name = BLOCK_OPT_BACKING_FMT,
-        .type = OPT_STRING,
-        .help = "Image format of the base image"
-    }, {
-        .name = BLOCK_OPT_CLUSTER_SIZE,
-        .type = OPT_SIZE,
-        .help = "Cluster size (in bytes)",
-        .value = { .n = QED_DEFAULT_CLUSTER_SIZE },
-    }, {
-        .name = BLOCK_OPT_TABLE_SIZE,
-        .type = OPT_SIZE,
-        .help = "L1/L2 table size (in clusters)"
-    },
-    { /* end of list */ }
-};
-
-static BlockDriver bdrv_qed = {
-    .format_name              = "qed",
-    .instance_size            = sizeof(BDRVQEDState),
-    .create_options           = qed_create_options,
-
-    .bdrv_probe               = bdrv_qed_probe,
-    .bdrv_rebind              = bdrv_qed_rebind,
-    .bdrv_open                = bdrv_qed_open,
-    .bdrv_close               = bdrv_qed_close,
-    .bdrv_reopen_prepare      = bdrv_qed_reopen_prepare,
-    .bdrv_create              = bdrv_qed_create,
-    .bdrv_has_zero_init       = bdrv_has_zero_init_1,
-    .bdrv_co_is_allocated     = bdrv_qed_co_is_allocated,
-    .bdrv_make_empty          = bdrv_qed_make_empty,
-    .bdrv_aio_readv           = bdrv_qed_aio_readv,
-    .bdrv_aio_writev          = bdrv_qed_aio_writev,
-    .bdrv_co_write_zeroes     = bdrv_qed_co_write_zeroes,
-    .bdrv_truncate            = bdrv_qed_truncate,
-    .bdrv_getlength           = bdrv_qed_getlength,
-    .bdrv_get_info            = bdrv_qed_get_info,
-    .bdrv_change_backing_file = bdrv_qed_change_backing_file,
-    .bdrv_invalidate_cache    = bdrv_qed_invalidate_cache,
-    .bdrv_check               = bdrv_qed_check,
-};
-
-static void bdrv_qed_init(void)
-{
-    bdrv_register(&bdrv_qed);
-}
-
-block_init(bdrv_qed_init);
diff --git a/contrib/qemu/block/qed.h b/contrib/qemu/block/qed.h
deleted file mode 100644
index 2b4ddedf313..00000000000
--- a/contrib/qemu/block/qed.h
+++ /dev/null
@@ -1,344 +0,0 @@
-/*
- * QEMU Enhanced Disk Format
- *
- * Copyright IBM, Corp. 2010
- *
- * Authors:
- *  Stefan Hajnoczi   <stefanha@linux.vnet.ibm.com>
- *  Anthony Liguori   <aliguori@us.ibm.com>
- *
- * This work is licensed under the terms of the GNU LGPL, version 2 or later.
- * See the COPYING.LIB file in the top-level directory.
- *
- */
-
-#ifndef BLOCK_QED_H
-#define BLOCK_QED_H
-
-#include "block/block_int.h"
-
-/* The layout of a QED file is as follows:
- *
- * +--------+----------+----------+----------+-----+
- * | header | L1 table | cluster0 | cluster1 | ... |
- * +--------+----------+----------+----------+-----+
- *
- * There is a 2-level pagetable for cluster allocation:
- *
- *                     +----------+
- *                     | L1 table |
- *                     +----------+
- *                ,------'  |  '------.
- *           +----------+   |    +----------+
- *           | L2 table |  ...   | L2 table |
- *           +----------+        +----------+
- *       ,------'  |  '------.
- *  +----------+   |    +----------+
- *  |   Data   |  ...   |   Data   |
- *  +----------+        +----------+
- *
- * The L1 table is fixed size and always present.  L2 tables are allocated on
- * demand.  The L1 table size determines the maximum possible image size; it
- * can be influenced using the cluster_size and table_size values.
- *
- * All fields are little-endian on disk.
- */
-
-enum {
-    QED_MAGIC = 'Q' | 'E' << 8 | 'D' << 16 | '\0' << 24,
-
-    /* The image supports a backing file */
-    QED_F_BACKING_FILE = 0x01,
-
-    /* The image needs a consistency check before use */
-    QED_F_NEED_CHECK = 0x02,
-
-    /* The backing file format must not be probed, treat as raw image */
-    QED_F_BACKING_FORMAT_NO_PROBE = 0x04,
-
-    /* Feature bits must be used when the on-disk format changes */
-    QED_FEATURE_MASK = QED_F_BACKING_FILE | /* supported feature bits */
-                       QED_F_NEED_CHECK |
-                       QED_F_BACKING_FORMAT_NO_PROBE,
-    QED_COMPAT_FEATURE_MASK = 0,            /* supported compat feature bits */
-    QED_AUTOCLEAR_FEATURE_MASK = 0,         /* supported autoclear feature bits */
-
-    /* Data is stored in groups of sectors called clusters.  Cluster size must
-     * be large to avoid keeping too much metadata.  I/O requests that have
-     * sub-cluster size will require read-modify-write.
-     */
-    QED_MIN_CLUSTER_SIZE = 4 * 1024, /* in bytes */
-    QED_MAX_CLUSTER_SIZE = 64 * 1024 * 1024,
-    QED_DEFAULT_CLUSTER_SIZE = 64 * 1024,
-
-    /* Allocated clusters are tracked using a 2-level pagetable.  Table size is
-     * a multiple of clusters so large maximum image sizes can be supported
-     * without jacking up the cluster size too much.
-     */
-    QED_MIN_TABLE_SIZE = 1,        /* in clusters */
-    QED_MAX_TABLE_SIZE = 16,
-    QED_DEFAULT_TABLE_SIZE = 4,
-
-    /* Delay to flush and clean image after last allocating write completes */
-    QED_NEED_CHECK_TIMEOUT = 5,    /* in seconds */
-};
-
-typedef struct {
-    uint32_t magic;                 /* QED\0 */
-
-    uint32_t cluster_size;          /* in bytes */
-    uint32_t table_size;            /* for L1 and L2 tables, in clusters */
-    uint32_t header_size;           /* in clusters */
-
-    uint64_t features;              /* format feature bits */
-    uint64_t compat_features;       /* compatible feature bits */
-    uint64_t autoclear_features;    /* self-resetting feature bits */
-
-    uint64_t l1_table_offset;       /* in bytes */
-    uint64_t image_size;            /* total logical image size, in bytes */
-
-    /* if (features & QED_F_BACKING_FILE) */
-    uint32_t backing_filename_offset; /* in bytes from start of header */
-    uint32_t backing_filename_size;   /* in bytes */
-} QEDHeader;
-
-typedef struct {
-    uint64_t offsets[0];            /* in bytes */
-} QEDTable;
-
-/* The L2 cache is a simple write-through cache for L2 structures */
-typedef struct CachedL2Table {
-    QEDTable *table;
-    uint64_t offset;    /* offset=0 indicates an invalidate entry */
-    QTAILQ_ENTRY(CachedL2Table) node;
-    int ref;
-} CachedL2Table;
-
-typedef struct {
-    QTAILQ_HEAD(, CachedL2Table) entries;
-    unsigned int n_entries;
-} L2TableCache;
-
-typedef struct QEDRequest {
-    CachedL2Table *l2_table;
-} QEDRequest;
-
-enum {
-    QED_AIOCB_WRITE = 0x0001,       /* read or write? */
-    QED_AIOCB_ZERO  = 0x0002,       /* zero write, used with QED_AIOCB_WRITE */
-};
-
-typedef struct QEDAIOCB {
-    BlockDriverAIOCB common;
-    QEMUBH *bh;
-    int bh_ret;                     /* final return status for completion bh */
-    QSIMPLEQ_ENTRY(QEDAIOCB) next;  /* next request */
-    int flags;                      /* QED_AIOCB_* bits ORed together */
-    bool *finished;                 /* signal for cancel completion */
-    uint64_t end_pos;               /* request end on block device, in bytes */
-
-    /* User scatter-gather list */
-    QEMUIOVector *qiov;
-    size_t qiov_offset;             /* byte count already processed */
-
-    /* Current cluster scatter-gather list */
-    QEMUIOVector cur_qiov;
-    uint64_t cur_pos;               /* position on block device, in bytes */
-    uint64_t cur_cluster;           /* cluster offset in image file */
-    unsigned int cur_nclusters;     /* number of clusters being accessed */
-    int find_cluster_ret;           /* used for L1/L2 update */
-
-    QEDRequest request;
-} QEDAIOCB;
-
-typedef struct {
-    BlockDriverState *bs;           /* device */
-    uint64_t file_size;             /* length of image file, in bytes */
-
-    QEDHeader header;               /* always cpu-endian */
-    QEDTable *l1_table;
-    L2TableCache l2_cache;          /* l2 table cache */
-    uint32_t table_nelems;
-    uint32_t l1_shift;
-    uint32_t l2_shift;
-    uint32_t l2_mask;
-
-    /* Allocating write request queue */
-    QSIMPLEQ_HEAD(, QEDAIOCB) allocating_write_reqs;
-    bool allocating_write_reqs_plugged;
-
-    /* Periodic flush and clear need check flag */
-    QEMUTimer *need_check_timer;
-} BDRVQEDState;
-
-enum {
-    QED_CLUSTER_FOUND,         /* cluster found */
-    QED_CLUSTER_ZERO,          /* zero cluster found */
-    QED_CLUSTER_L2,            /* cluster missing in L2 */
-    QED_CLUSTER_L1,            /* cluster missing in L1 */
-};
-
-/**
- * qed_find_cluster() completion callback
- *
- * @opaque:     User data for completion callback
- * @ret:        QED_CLUSTER_FOUND   Success
- *              QED_CLUSTER_L2      Data cluster unallocated in L2
- *              QED_CLUSTER_L1      L2 unallocated in L1
- *              -errno              POSIX error occurred
- * @offset:     Data cluster offset
- * @len:        Contiguous bytes starting from cluster offset
- *
- * This function is invoked when qed_find_cluster() completes.
- *
- * On success ret is QED_CLUSTER_FOUND and offset/len are a contiguous range
- * in the image file.
- *
- * On failure ret is QED_CLUSTER_L2 or QED_CLUSTER_L1 for missing L2 or L1
- * table offset, respectively.  len is number of contiguous unallocated bytes.
- */
-typedef void QEDFindClusterFunc(void *opaque, int ret, uint64_t offset, size_t len);
-
-/**
- * Generic callback for chaining async callbacks
- */
-typedef struct {
-    BlockDriverCompletionFunc *cb;
-    void *opaque;
-} GenericCB;
-
-void *gencb_alloc(size_t len, BlockDriverCompletionFunc *cb, void *opaque);
-void gencb_complete(void *opaque, int ret);
-
-/**
- * Header functions
- */
-int qed_write_header_sync(BDRVQEDState *s);
-
-/**
- * L2 cache functions
- */
-void qed_init_l2_cache(L2TableCache *l2_cache);
-void qed_free_l2_cache(L2TableCache *l2_cache);
-CachedL2Table *qed_alloc_l2_cache_entry(L2TableCache *l2_cache);
-void qed_unref_l2_cache_entry(CachedL2Table *entry);
-CachedL2Table *qed_find_l2_cache_entry(L2TableCache *l2_cache, uint64_t offset);
-void qed_commit_l2_cache_entry(L2TableCache *l2_cache, CachedL2Table *l2_table);
-
-/**
- * Table I/O functions
- */
-int qed_read_l1_table_sync(BDRVQEDState *s);
-void qed_write_l1_table(BDRVQEDState *s, unsigned int index, unsigned int n,
-                        BlockDriverCompletionFunc *cb, void *opaque);
-int qed_write_l1_table_sync(BDRVQEDState *s, unsigned int index,
-                            unsigned int n);
-int qed_read_l2_table_sync(BDRVQEDState *s, QEDRequest *request,
-                           uint64_t offset);
-void qed_read_l2_table(BDRVQEDState *s, QEDRequest *request, uint64_t offset,
-                       BlockDriverCompletionFunc *cb, void *opaque);
-void qed_write_l2_table(BDRVQEDState *s, QEDRequest *request,
-                        unsigned int index, unsigned int n, bool flush,
-                        BlockDriverCompletionFunc *cb, void *opaque);
-int qed_write_l2_table_sync(BDRVQEDState *s, QEDRequest *request,
-                            unsigned int index, unsigned int n, bool flush);
-
-/**
- * Cluster functions
- */
-void qed_find_cluster(BDRVQEDState *s, QEDRequest *request, uint64_t pos,
-                      size_t len, QEDFindClusterFunc *cb, void *opaque);
-
-/**
- * Consistency check
- */
-int qed_check(BDRVQEDState *s, BdrvCheckResult *result, bool fix);
-
-QEDTable *qed_alloc_table(BDRVQEDState *s);
-
-/**
- * Round down to the start of a cluster
- */
-static inline uint64_t qed_start_of_cluster(BDRVQEDState *s, uint64_t offset)
-{
-    return offset & ~(uint64_t)(s->header.cluster_size - 1);
-}
-
-static inline uint64_t qed_offset_into_cluster(BDRVQEDState *s, uint64_t offset)
-{
-    return offset & (s->header.cluster_size - 1);
-}
-
-static inline uint64_t qed_bytes_to_clusters(BDRVQEDState *s, uint64_t bytes)
-{
-    return qed_start_of_cluster(s, bytes + (s->header.cluster_size - 1)) /
-           (s->header.cluster_size - 1);
-}
-
-static inline unsigned int qed_l1_index(BDRVQEDState *s, uint64_t pos)
-{
-    return pos >> s->l1_shift;
-}
-
-static inline unsigned int qed_l2_index(BDRVQEDState *s, uint64_t pos)
-{
-    return (pos >> s->l2_shift) & s->l2_mask;
-}
-
-/**
- * Test if a cluster offset is valid
- */
-static inline bool qed_check_cluster_offset(BDRVQEDState *s, uint64_t offset)
-{
-    uint64_t header_size = (uint64_t)s->header.header_size *
-                           s->header.cluster_size;
-
-    if (offset & (s->header.cluster_size - 1)) {
-        return false;
-    }
-    return offset >= header_size && offset < s->file_size;
-}
-
-/**
- * Test if a table offset is valid
- */
-static inline bool qed_check_table_offset(BDRVQEDState *s, uint64_t offset)
-{
-    uint64_t end_offset = offset + (s->header.table_size - 1) *
-                          s->header.cluster_size;
-
-    /* Overflow check */
-    if (end_offset <= offset) {
-        return false;
-    }
-
-    return qed_check_cluster_offset(s, offset) &&
-           qed_check_cluster_offset(s, end_offset);
-}
-
-static inline bool qed_offset_is_cluster_aligned(BDRVQEDState *s,
-                                                 uint64_t offset)
-{
-    if (qed_offset_into_cluster(s, offset)) {
-        return false;
-    }
-    return true;
-}
-
-static inline bool qed_offset_is_unalloc_cluster(uint64_t offset)
-{
-    if (offset == 0) {
-        return true;
-    }
-    return false;
-}
-
-static inline bool qed_offset_is_zero_cluster(uint64_t offset)
-{
-    if (offset == 1) {
-        return true;
-    }
-    return false;
-}
-
-#endif /* BLOCK_QED_H */
diff --git a/contrib/qemu/block/snapshot.c b/contrib/qemu/block/snapshot.c
deleted file mode 100644
index 6c6d9deea1f..00000000000
--- a/contrib/qemu/block/snapshot.c
+++ /dev/null
@@ -1,157 +0,0 @@
-/*
- * Block layer snapshot related functions
- *
- * Copyright (c) 2003-2008 Fabrice Bellard
- *
- * Permission is hereby granted, free of charge, to any person obtaining a copy
- * of this software and associated documentation files (the "Software"), to deal
- * in the Software without restriction, including without limitation the rights
- * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
- * copies of the Software, and to permit persons to whom the Software is
- * furnished to do so, subject to the following conditions:
- *
- * The above copyright notice and this permission notice shall be included in
- * all copies or substantial portions of the Software.
- *
- * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
- * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
- * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
- * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
- * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
- * THE SOFTWARE.
- */
-
-#include "block/snapshot.h"
-#include "block/block_int.h"
-
-int bdrv_snapshot_find(BlockDriverState *bs, QEMUSnapshotInfo *sn_info,
-                       const char *name)
-{
-    QEMUSnapshotInfo *sn_tab, *sn;
-    int nb_sns, i, ret;
-
-    ret = -ENOENT;
-    nb_sns = bdrv_snapshot_list(bs, &sn_tab);
-    if (nb_sns < 0) {
-        return ret;
-    }
-    for (i = 0; i < nb_sns; i++) {
-        sn = &sn_tab[i];
-        if (!strcmp(sn->id_str, name) || !strcmp(sn->name, name)) {
-            *sn_info = *sn;
-            ret = 0;
-            break;
-        }
-    }
-    g_free(sn_tab);
-    return ret;
-}
-
-int bdrv_can_snapshot(BlockDriverState *bs)
-{
-    BlockDriver *drv = bs->drv;
-    if (!drv || !bdrv_is_inserted(bs) || bdrv_is_read_only(bs)) {
-        return 0;
-    }
-
-    if (!drv->bdrv_snapshot_create) {
-        if (bs->file != NULL) {
-            return bdrv_can_snapshot(bs->file);
-        }
-        return 0;
-    }
-
-    return 1;
-}
-
-int bdrv_snapshot_create(BlockDriverState *bs,
-                         QEMUSnapshotInfo *sn_info)
-{
-    BlockDriver *drv = bs->drv;
-    if (!drv) {
-        return -ENOMEDIUM;
-    }
-    if (drv->bdrv_snapshot_create) {
-        return drv->bdrv_snapshot_create(bs, sn_info);
-    }
-    if (bs->file) {
-        return bdrv_snapshot_create(bs->file, sn_info);
-    }
-    return -ENOTSUP;
-}
-
-int bdrv_snapshot_goto(BlockDriverState *bs,
-                       const char *snapshot_id)
-{
-    BlockDriver *drv = bs->drv;
-    int ret, open_ret;
-
-    if (!drv) {
-        return -ENOMEDIUM;
-    }
-    if (drv->bdrv_snapshot_goto) {
-        return drv->bdrv_snapshot_goto(bs, snapshot_id);
-    }
-
-    if (bs->file) {
-        drv->bdrv_close(bs);
-        ret = bdrv_snapshot_goto(bs->file, snapshot_id);
-        open_ret = drv->bdrv_open(bs, NULL, bs->open_flags);
-        if (open_ret < 0) {
-            bdrv_delete(bs->file);
-            bs->drv = NULL;
-            return open_ret;
-        }
-        return ret;
-    }
-
-    return -ENOTSUP;
-}
-
-int bdrv_snapshot_delete(BlockDriverState *bs, const char *snapshot_id)
-{
-    BlockDriver *drv = bs->drv;
-    if (!drv) {
-        return -ENOMEDIUM;
-    }
-    if (drv->bdrv_snapshot_delete) {
-        return drv->bdrv_snapshot_delete(bs, snapshot_id);
-    }
-    if (bs->file) {
-        return bdrv_snapshot_delete(bs->file, snapshot_id);
-    }
-    return -ENOTSUP;
-}
-
-int bdrv_snapshot_list(BlockDriverState *bs,
-                       QEMUSnapshotInfo **psn_info)
-{
-    BlockDriver *drv = bs->drv;
-    if (!drv) {
-        return -ENOMEDIUM;
-    }
-    if (drv->bdrv_snapshot_list) {
-        return drv->bdrv_snapshot_list(bs, psn_info);
-    }
-    if (bs->file) {
-        return bdrv_snapshot_list(bs->file, psn_info);
-    }
-    return -ENOTSUP;
-}
-
-int bdrv_snapshot_load_tmp(BlockDriverState *bs,
-        const char *snapshot_name)
-{
-    BlockDriver *drv = bs->drv;
-    if (!drv) {
-        return -ENOMEDIUM;
-    }
-    if (!bs->read_only) {
-        return -EINVAL;
-    }
-    if (drv->bdrv_snapshot_load_tmp) {
-        return drv->bdrv_snapshot_load_tmp(bs, snapshot_name);
-    }
-    return -ENOTSUP;
-}
diff --git a/contrib/qemu/config-host.h b/contrib/qemu/config-host.h
deleted file mode 100644
index 6b5c8da1243..00000000000
--- a/contrib/qemu/config-host.h
+++ /dev/null
@@ -1,72 +0,0 @@
-/* Automatically generated by create_config - do not modify */
-#define CONFIG_QEMU_CONFDIR "/usr/local/etc/qemu"
-#define CONFIG_QEMU_DATADIR "/usr/local/share/qemu"
-#define CONFIG_QEMU_DOCDIR "/usr/local/share/doc/qemu"
-#define CONFIG_QEMU_LOCALSTATEDIR "/usr/local/var"
-#define CONFIG_QEMU_HELPERDIR "/usr/local/libexec"
-#define CONFIG_QEMU_LOCALEDIR "/usr/local/share/locale"
-#define HOST_X86_64 1
-#define CONFIG_QEMU_LDST_OPTIMIZATION 1
-#define CONFIG_POSIX 1
-#define CONFIG_LINUX 1
-#define CONFIG_SLIRP 1
-#define CONFIG_SMBD_COMMAND "/usr/sbin/smbd"
-#define CONFIG_AUDIO_DRIVERS \
-    &oss_audio_driver,\
-
-#define CONFIG_OSS 1
-#define CONFIG_BDRV_RW_WHITELIST\
-    NULL
-#define CONFIG_BDRV_RO_WHITELIST\
-    NULL
-#define CONFIG_VNC 1
-#define CONFIG_VNC_TLS 1
-#define CONFIG_VNC_SASL 1
-#define CONFIG_VNC_WS 1
-#define CONFIG_FNMATCH 1
-#define CONFIG_UUID 1
-#define CONFIG_XFS 1
-#define QEMU_VERSION "1.5.50"
-#define QEMU_PKGVERSION ""
-#define CONFIG_CURSES 1
-#define CONFIG_UTIMENSAT 1
-#define CONFIG_PIPE2 1
-#define CONFIG_ACCEPT4 1
-#define CONFIG_SPLICE 1
-#define CONFIG_EVENTFD 1
-#define CONFIG_FALLOCATE 1
-#define CONFIG_FALLOCATE_PUNCH_HOLE 1
-#define CONFIG_SYNC_FILE_RANGE 1
-#define CONFIG_FIEMAP 1
-#define CONFIG_DUP3 1
-#define CONFIG_EPOLL 1
-#define CONFIG_EPOLL_CREATE1 1
-#define CONFIG_EPOLL_PWAIT 1
-#define CONFIG_SENDFILE 1
-#define CONFIG_INOTIFY 1
-#define CONFIG_INOTIFY1 1
-#define CONFIG_BYTESWAP_H 1
-#define CONFIG_CURL 1
-#define CONFIG_LINUX_AIO 1
-#define CONFIG_ATTR 1
-#define CONFIG_VHOST_SCSI 1
-#define CONFIG_IOVEC 1
-#define CONFIG_PREADV 1
-#define CONFIG_FDT 1
-#define CONFIG_SIGNALFD 1
-#define CONFIG_FDATASYNC 1
-#define CONFIG_MADVISE 1
-#define CONFIG_POSIX_MADVISE 1
-#define CONFIG_SIGEV_THREAD_ID 1
-#define CONFIG_UNAME_RELEASE ""
-#define CONFIG_QOM_CAST_DEBUG 1
-#define CONFIG_COROUTINE_BACKEND ucontext
-#define CONFIG_OPEN_BY_HANDLE 1
-#define CONFIG_LINUX_MAGIC_H 1
-#define CONFIG_PRAGMA_DIAGNOSTIC_AVAILABLE 1
-#define CONFIG_HAS_ENVIRON 1
-#define CONFIG_CPUID_H 1
-#define CONFIG_VIRTIO_BLK_DATA_PLANE $(CONFIG_VIRTIO)
-#define CONFIG_TRACE_NOP 1
-#define CONFIG_TRACE_FILE trace
-#define CONFIG_TRACE_DEFAULT 1
diff --git a/contrib/qemu/coroutine-ucontext.c b/contrib/qemu/coroutine-ucontext.c
deleted file mode 100644
index 4bf2cde279b..00000000000
--- a/contrib/qemu/coroutine-ucontext.c
+++ /dev/null
@@ -1,225 +0,0 @@
-/*
- * ucontext coroutine initialization code
- *
- * Copyright (C) 2006  Anthony Liguori <anthony@codemonkey.ws>
- * Copyright (C) 2011  Kevin Wolf <kwolf@redhat.com>
- *
- * This library is free software; you can redistribute it and/or
- * modify it under the terms of the GNU Lesser General Public
- * License as published by the Free Software Foundation; either
- * version 2.0 of the License, or (at your option) any later version.
- *
- * This library is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
- * Lesser General Public License for more details.
- *
- * You should have received a copy of the GNU Lesser General Public
- * License along with this library; if not, see <http://www.gnu.org/licenses/>.
- */
-
-/* XXX Is there a nicer way to disable glibc's stack check for longjmp? */
-#ifdef _FORTIFY_SOURCE
-#undef _FORTIFY_SOURCE
-#endif
-#include <stdlib.h>
-#include <setjmp.h>
-#include <stdint.h>
-#include <pthread.h>
-#include <ucontext.h>
-#include "qemu-common.h"
-#include "block/coroutine_int.h"
-
-#ifdef CONFIG_VALGRIND_H
-#include <valgrind/valgrind.h>
-#endif
-
-typedef struct {
-    Coroutine base;
-    void *stack;
-    sigjmp_buf env;
-
-#ifdef CONFIG_VALGRIND_H
-    unsigned int valgrind_stack_id;
-#endif
-
-} CoroutineUContext;
-
-/**
- * Per-thread coroutine bookkeeping
- */
-typedef struct {
-    /** Currently executing coroutine */
-    Coroutine *current;
-
-    /** The default coroutine */
-    CoroutineUContext leader;
-} CoroutineThreadState;
-
-static pthread_key_t thread_state_key;
-
-/*
- * va_args to makecontext() must be type 'int', so passing
- * the pointer we need may require several int args. This
- * union is a quick hack to let us do that
- */
-union cc_arg {
-    void *p;
-    int i[2];
-};
-
-static CoroutineThreadState *coroutine_get_thread_state(void)
-{
-    CoroutineThreadState *s = pthread_getspecific(thread_state_key);
-
-    if (!s) {
-        s = g_malloc0(sizeof(*s));
-        s->current = &s->leader.base;
-        pthread_setspecific(thread_state_key, s);
-    }
-    return s;
-}
-
-static void qemu_coroutine_thread_cleanup(void *opaque)
-{
-    CoroutineThreadState *s = opaque;
-
-    g_free(s);
-}
-
-static void __attribute__((constructor)) coroutine_init(void)
-{
-    int ret;
-
-    ret = pthread_key_create(&thread_state_key, qemu_coroutine_thread_cleanup);
-    if (ret != 0) {
-        fprintf(stderr, "unable to create leader key: %s\n", strerror(errno));
-        abort();
-    }
-}
-
-static void coroutine_trampoline(int i0, int i1)
-{
-    union cc_arg arg;
-    CoroutineUContext *self;
-    Coroutine *co;
-
-    arg.i[0] = i0;
-    arg.i[1] = i1;
-    self = arg.p;
-    co = &self->base;
-
-    /* Initialize longjmp environment and switch back the caller */
-    if (!sigsetjmp(self->env, 0)) {
-        siglongjmp(*(sigjmp_buf *)co->entry_arg, 1);
-    }
-
-    while (true) {
-        co->entry(co->entry_arg);
-        qemu_coroutine_switch(co, co->caller, COROUTINE_TERMINATE);
-    }
-}
-
-Coroutine *qemu_coroutine_new(void)
-{
-    const size_t stack_size = 1 << 20;
-    CoroutineUContext *co;
-    ucontext_t old_uc, uc;
-    sigjmp_buf old_env;
-    union cc_arg arg = {0};
-
-    /* The ucontext functions preserve signal masks which incurs a
-     * system call overhead.  sigsetjmp(buf, 0)/siglongjmp() does not
-     * preserve signal masks but only works on the current stack.
-     * Since we need a way to create and switch to a new stack, use
-     * the ucontext functions for that but sigsetjmp()/siglongjmp() for
-     * everything else.
-     */
-
-    if (getcontext(&uc) == -1) {
-        abort();
-    }
-
-    co = g_malloc0(sizeof(*co));
-    co->stack = g_malloc(stack_size);
-    co->base.entry_arg = &old_env; /* stash away our jmp_buf */
-
-    uc.uc_link = &old_uc;
-    uc.uc_stack.ss_sp = co->stack;
-    uc.uc_stack.ss_size = stack_size;
-    uc.uc_stack.ss_flags = 0;
-
-#ifdef CONFIG_VALGRIND_H
-    co->valgrind_stack_id =
-        VALGRIND_STACK_REGISTER(co->stack, co->stack + stack_size);
-#endif
-
-    arg.p = co;
-
-    makecontext(&uc, (void (*)(void))coroutine_trampoline,
-                2, arg.i[0], arg.i[1]);
-
-    /* swapcontext() in, siglongjmp() back out */
-    if (!sigsetjmp(old_env, 0)) {
-        swapcontext(&old_uc, &uc);
-    }
-    return &co->base;
-}
-
-#ifdef CONFIG_VALGRIND_H
-#ifdef CONFIG_PRAGMA_DIAGNOSTIC_AVAILABLE
-/* Work around an unused variable in the valgrind.h macro... */
-#pragma GCC diagnostic push
-#pragma GCC diagnostic ignored "-Wunused-but-set-variable"
-#endif
-static inline void valgrind_stack_deregister(CoroutineUContext *co)
-{
-    VALGRIND_STACK_DEREGISTER(co->valgrind_stack_id);
-}
-#ifdef CONFIG_PRAGMA_DIAGNOSTIC_AVAILABLE
-#pragma GCC diagnostic pop
-#endif
-#endif
-
-void qemu_coroutine_delete(Coroutine *co_)
-{
-    CoroutineUContext *co = DO_UPCAST(CoroutineUContext, base, co_);
-
-#ifdef CONFIG_VALGRIND_H
-    valgrind_stack_deregister(co);
-#endif
-
-    g_free(co->stack);
-    g_free(co);
-}
-
-CoroutineAction qemu_coroutine_switch(Coroutine *from_, Coroutine *to_,
-                                      CoroutineAction action)
-{
-    CoroutineUContext *from = DO_UPCAST(CoroutineUContext, base, from_);
-    CoroutineUContext *to = DO_UPCAST(CoroutineUContext, base, to_);
-    CoroutineThreadState *s = coroutine_get_thread_state();
-    int ret;
-
-    s->current = to_;
-
-    ret = sigsetjmp(from->env, 0);
-    if (ret == 0) {
-        siglongjmp(to->env, action);
-    }
-    return ret;
-}
-
-Coroutine *qemu_coroutine_self(void)
-{
-    CoroutineThreadState *s = coroutine_get_thread_state();
-
-    return s->current;
-}
-
-bool qemu_in_coroutine(void)
-{
-    CoroutineThreadState *s = pthread_getspecific(thread_state_key);
-
-    return s && s->current->caller;
-}
diff --git a/contrib/qemu/include/block/aio.h b/contrib/qemu/include/block/aio.h
deleted file mode 100644
index 183679374fa..00000000000
--- a/contrib/qemu/include/block/aio.h
+++ /dev/null
@@ -1,247 +0,0 @@
-/*
- * QEMU aio implementation
- *
- * Copyright IBM, Corp. 2008
- *
- * Authors:
- *  Anthony Liguori   <aliguori@us.ibm.com>
- *
- * This work is licensed under the terms of the GNU GPL, version 2.  See
- * the COPYING file in the top-level directory.
- *
- */
-
-#ifndef QEMU_AIO_H
-#define QEMU_AIO_H
-
-#include "qemu-common.h"
-#include "qemu/queue.h"
-#include "qemu/event_notifier.h"
-
-typedef struct BlockDriverAIOCB BlockDriverAIOCB;
-typedef void BlockDriverCompletionFunc(void *opaque, int ret);
-
-typedef struct AIOCBInfo {
-    void (*cancel)(BlockDriverAIOCB *acb);
-    size_t aiocb_size;
-} AIOCBInfo;
-
-struct BlockDriverAIOCB {
-    const AIOCBInfo *aiocb_info;
-    BlockDriverState *bs;
-    BlockDriverCompletionFunc *cb;
-    void *opaque;
-};
-
-void *qemu_aio_get(const AIOCBInfo *aiocb_info, BlockDriverState *bs,
-                   BlockDriverCompletionFunc *cb, void *opaque);
-void qemu_aio_release(void *p);
-
-typedef struct AioHandler AioHandler;
-typedef void QEMUBHFunc(void *opaque);
-typedef void IOHandler(void *opaque);
-
-typedef struct AioContext {
-    GSource source;
-
-    /* The list of registered AIO handlers */
-    QLIST_HEAD(, AioHandler) aio_handlers;
-
-    /* This is a simple lock used to protect the aio_handlers list.
-     * Specifically, it's used to ensure that no callbacks are removed while
-     * we're walking and dispatching callbacks.
-     */
-    int walking_handlers;
-
-    /* Anchor of the list of Bottom Halves belonging to the context */
-    struct QEMUBH *first_bh;
-
-    /* A simple lock used to protect the first_bh list, and ensure that
-     * no callbacks are removed while we're walking and dispatching callbacks.
-     */
-    int walking_bh;
-
-    /* Used for aio_notify.  */
-    EventNotifier notifier;
-
-    /* GPollFDs for aio_poll() */
-    GArray *pollfds;
-
-    /* Thread pool for performing work and receiving completion callbacks */
-    struct ThreadPool *thread_pool;
-} AioContext;
-
-/* Returns 1 if there are still outstanding AIO requests; 0 otherwise */
-typedef int (AioFlushEventNotifierHandler)(EventNotifier *e);
-
-/**
- * aio_context_new: Allocate a new AioContext.
- *
- * AioContext provide a mini event-loop that can be waited on synchronously.
- * They also provide bottom halves, a service to execute a piece of code
- * as soon as possible.
- */
-AioContext *aio_context_new(void);
-
-/**
- * aio_context_ref:
- * @ctx: The AioContext to operate on.
- *
- * Add a reference to an AioContext.
- */
-void aio_context_ref(AioContext *ctx);
-
-/**
- * aio_context_unref:
- * @ctx: The AioContext to operate on.
- *
- * Drop a reference to an AioContext.
- */
-void aio_context_unref(AioContext *ctx);
-
-/**
- * aio_bh_new: Allocate a new bottom half structure.
- *
- * Bottom halves are lightweight callbacks whose invocation is guaranteed
- * to be wait-free, thread-safe and signal-safe.  The #QEMUBH structure
- * is opaque and must be allocated prior to its use.
- */
-QEMUBH *aio_bh_new(AioContext *ctx, QEMUBHFunc *cb, void *opaque);
-
-/**
- * aio_notify: Force processing of pending events.
- *
- * Similar to signaling a condition variable, aio_notify forces
- * aio_wait to exit, so that the next call will re-examine pending events.
- * The caller of aio_notify will usually call aio_wait again very soon,
- * or go through another iteration of the GLib main loop.  Hence, aio_notify
- * also has the side effect of recalculating the sets of file descriptors
- * that the main loop waits for.
- *
- * Calling aio_notify is rarely necessary, because for example scheduling
- * a bottom half calls it already.
- */
-void aio_notify(AioContext *ctx);
-
-/**
- * aio_bh_poll: Poll bottom halves for an AioContext.
- *
- * These are internal functions used by the QEMU main loop.
- */
-int aio_bh_poll(AioContext *ctx);
-
-/**
- * qemu_bh_schedule: Schedule a bottom half.
- *
- * Scheduling a bottom half interrupts the main loop and causes the
- * execution of the callback that was passed to qemu_bh_new.
- *
- * Bottom halves that are scheduled from a bottom half handler are instantly
- * invoked.  This can create an infinite loop if a bottom half handler
- * schedules itself.
- *
- * @bh: The bottom half to be scheduled.
- */
-void qemu_bh_schedule(QEMUBH *bh);
-
-/**
- * qemu_bh_cancel: Cancel execution of a bottom half.
- *
- * Canceling execution of a bottom half undoes the effect of calls to
- * qemu_bh_schedule without freeing its resources yet.  While cancellation
- * itself is also wait-free and thread-safe, it can of course race with the
- * loop that executes bottom halves unless you are holding the iothread
- * mutex.  This makes it mostly useless if you are not holding the mutex.
- *
- * @bh: The bottom half to be canceled.
- */
-void qemu_bh_cancel(QEMUBH *bh);
-
-/**
- *qemu_bh_delete: Cancel execution of a bottom half and free its resources.
- *
- * Deleting a bottom half frees the memory that was allocated for it by
- * qemu_bh_new.  It also implies canceling the bottom half if it was
- * scheduled.
- *
- * @bh: The bottom half to be deleted.
- */
-void qemu_bh_delete(QEMUBH *bh);
-
-/* Return whether there are any pending callbacks from the GSource
- * attached to the AioContext.
- *
- * This is used internally in the implementation of the GSource.
- */
-bool aio_pending(AioContext *ctx);
-
-/* Progress in completing AIO work to occur.  This can issue new pending
- * aio as a result of executing I/O completion or bh callbacks.
- *
- * If there is no pending AIO operation or completion (bottom half),
- * return false.  If there are pending AIO operations of bottom halves,
- * return true.
- *
- * If there are no pending bottom halves, but there are pending AIO
- * operations, it may not be possible to make any progress without
- * blocking.  If @blocking is true, this function will wait until one
- * or more AIO events have completed, to ensure something has moved
- * before returning.
- */
-bool aio_poll(AioContext *ctx, bool blocking);
-
-#ifdef CONFIG_POSIX
-/* Returns 1 if there are still outstanding AIO requests; 0 otherwise */
-typedef int (AioFlushHandler)(void *opaque);
-
-/* Register a file descriptor and associated callbacks.  Behaves very similarly
- * to qemu_set_fd_handler2.  Unlike qemu_set_fd_handler2, these callbacks will
- * be invoked when using qemu_aio_wait().
- *
- * Code that invokes AIO completion functions should rely on this function
- * instead of qemu_set_fd_handler[2].
- */
-void aio_set_fd_handler(AioContext *ctx,
-                        int fd,
-                        IOHandler *io_read,
-                        IOHandler *io_write,
-                        AioFlushHandler *io_flush,
-                        void *opaque);
-#endif
-
-/* Register an event notifier and associated callbacks.  Behaves very similarly
- * to event_notifier_set_handler.  Unlike event_notifier_set_handler, these callbacks
- * will be invoked when using qemu_aio_wait().
- *
- * Code that invokes AIO completion functions should rely on this function
- * instead of event_notifier_set_handler.
- */
-void aio_set_event_notifier(AioContext *ctx,
-                            EventNotifier *notifier,
-                            EventNotifierHandler *io_read,
-                            AioFlushEventNotifierHandler *io_flush);
-
-/* Return a GSource that lets the main loop poll the file descriptors attached
- * to this AioContext.
- */
-GSource *aio_get_g_source(AioContext *ctx);
-
-/* Return the ThreadPool bound to this AioContext */
-struct ThreadPool *aio_get_thread_pool(AioContext *ctx);
-
-/* Functions to operate on the main QEMU AioContext.  */
-
-bool qemu_aio_wait(void);
-void qemu_aio_set_event_notifier(EventNotifier *notifier,
-                                 EventNotifierHandler *io_read,
-                                 AioFlushEventNotifierHandler *io_flush);
-
-#ifdef CONFIG_POSIX
-void qemu_aio_set_fd_handler(int fd,
-                             IOHandler *io_read,
-                             IOHandler *io_write,
-                             AioFlushHandler *io_flush,
-                             void *opaque);
-#endif
-
-#endif
diff --git a/contrib/qemu/include/block/block.h b/contrib/qemu/include/block/block.h
deleted file mode 100644
index b6b9014a9ce..00000000000
--- a/contrib/qemu/include/block/block.h
+++ /dev/null
@@ -1,443 +0,0 @@
-#ifndef BLOCK_H
-#define BLOCK_H
-
-#include "block/aio.h"
-#include "qemu-common.h"
-#include "qemu/option.h"
-#include "block/coroutine.h"
-#include "qapi/qmp/qobject.h"
-#include "qapi-types.h"
-
-/* block.c */
-typedef struct BlockDriver BlockDriver;
-typedef struct BlockJob BlockJob;
-
-typedef struct BlockDriverInfo {
-    /* in bytes, 0 if irrelevant */
-    int cluster_size;
-    /* offset at which the VM state can be saved (0 if not possible) */
-    int64_t vm_state_offset;
-    bool is_dirty;
-} BlockDriverInfo;
-
-typedef struct BlockFragInfo {
-    uint64_t allocated_clusters;
-    uint64_t total_clusters;
-    uint64_t fragmented_clusters;
-    uint64_t compressed_clusters;
-} BlockFragInfo;
-
-/* Callbacks for block device models */
-typedef struct BlockDevOps {
-    /*
-     * Runs when virtual media changed (monitor commands eject, change)
-     * Argument load is true on load and false on eject.
-     * Beware: doesn't run when a host device's physical media
-     * changes.  Sure would be useful if it did.
-     * Device models with removable media must implement this callback.
-     */
-    void (*change_media_cb)(void *opaque, bool load);
-    /*
-     * Runs when an eject request is issued from the monitor, the tray
-     * is closed, and the medium is locked.
-     * Device models that do not implement is_medium_locked will not need
-     * this callback.  Device models that can lock the medium or tray might
-     * want to implement the callback and unlock the tray when "force" is
-     * true, even if they do not support eject requests.
-     */
-    void (*eject_request_cb)(void *opaque, bool force);
-    /*
-     * Is the virtual tray open?
-     * Device models implement this only when the device has a tray.
-     */
-    bool (*is_tray_open)(void *opaque);
-    /*
-     * Is the virtual medium locked into the device?
-     * Device models implement this only when device has such a lock.
-     */
-    bool (*is_medium_locked)(void *opaque);
-    /*
-     * Runs when the size changed (e.g. monitor command block_resize)
-     */
-    void (*resize_cb)(void *opaque);
-} BlockDevOps;
-
-#define BDRV_O_RDWR        0x0002
-#define BDRV_O_SNAPSHOT    0x0008 /* open the file read only and save writes in a snapshot */
-#define BDRV_O_NOCACHE     0x0020 /* do not use the host page cache */
-#define BDRV_O_CACHE_WB    0x0040 /* use write-back caching */
-#define BDRV_O_NATIVE_AIO  0x0080 /* use native AIO instead of the thread pool */
-#define BDRV_O_NO_BACKING  0x0100 /* don't open the backing file */
-#define BDRV_O_NO_FLUSH    0x0200 /* disable flushing on this disk */
-#define BDRV_O_COPY_ON_READ 0x0400 /* copy read backing sectors into image */
-#define BDRV_O_INCOMING    0x0800  /* consistency hint for incoming migration */
-#define BDRV_O_CHECK       0x1000  /* open solely for consistency check */
-#define BDRV_O_ALLOW_RDWR  0x2000  /* allow reopen to change from r/o to r/w */
-#define BDRV_O_UNMAP       0x4000  /* execute guest UNMAP/TRIM operations */
-
-#define BDRV_O_CACHE_MASK  (BDRV_O_NOCACHE | BDRV_O_CACHE_WB | BDRV_O_NO_FLUSH)
-
-#define BDRV_SECTOR_BITS   9
-#define BDRV_SECTOR_SIZE   (1ULL << BDRV_SECTOR_BITS)
-#define BDRV_SECTOR_MASK   ~(BDRV_SECTOR_SIZE - 1)
-
-typedef enum {
-    BDRV_ACTION_REPORT, BDRV_ACTION_IGNORE, BDRV_ACTION_STOP
-} BlockErrorAction;
-
-typedef QSIMPLEQ_HEAD(BlockReopenQueue, BlockReopenQueueEntry) BlockReopenQueue;
-
-typedef struct BDRVReopenState {
-    BlockDriverState *bs;
-    int flags;
-    void *opaque;
-} BDRVReopenState;
-
-
-void bdrv_iostatus_enable(BlockDriverState *bs);
-void bdrv_iostatus_reset(BlockDriverState *bs);
-void bdrv_iostatus_disable(BlockDriverState *bs);
-bool bdrv_iostatus_is_enabled(const BlockDriverState *bs);
-void bdrv_iostatus_set_err(BlockDriverState *bs, int error);
-void bdrv_info_print(Monitor *mon, const QObject *data);
-void bdrv_info(Monitor *mon, QObject **ret_data);
-void bdrv_stats_print(Monitor *mon, const QObject *data);
-void bdrv_info_stats(Monitor *mon, QObject **ret_data);
-
-/* disk I/O throttling */
-void bdrv_io_limits_enable(BlockDriverState *bs);
-void bdrv_io_limits_disable(BlockDriverState *bs);
-bool bdrv_io_limits_enabled(BlockDriverState *bs);
-
-void bdrv_init(void);
-void bdrv_init_with_whitelist(void);
-BlockDriver *bdrv_find_protocol(const char *filename,
-                                bool allow_protocol_prefix);
-BlockDriver *bdrv_find_format(const char *format_name);
-BlockDriver *bdrv_find_whitelisted_format(const char *format_name,
-                                          bool readonly);
-int bdrv_create(BlockDriver *drv, const char* filename,
-    QEMUOptionParameter *options);
-int bdrv_create_file(const char* filename, QEMUOptionParameter *options);
-BlockDriverState *bdrv_new(const char *device_name);
-void bdrv_make_anon(BlockDriverState *bs);
-void bdrv_swap(BlockDriverState *bs_new, BlockDriverState *bs_old);
-void bdrv_append(BlockDriverState *bs_new, BlockDriverState *bs_top);
-void bdrv_delete(BlockDriverState *bs);
-int bdrv_parse_cache_flags(const char *mode, int *flags);
-int bdrv_parse_discard_flags(const char *mode, int *flags);
-int bdrv_file_open(BlockDriverState **pbs, const char *filename,
-                   QDict *options, int flags);
-int bdrv_open_backing_file(BlockDriverState *bs, QDict *options);
-int bdrv_open(BlockDriverState *bs, const char *filename, QDict *options,
-              int flags, BlockDriver *drv);
-BlockReopenQueue *bdrv_reopen_queue(BlockReopenQueue *bs_queue,
-                                    BlockDriverState *bs, int flags);
-int bdrv_reopen_multiple(BlockReopenQueue *bs_queue, Error **errp);
-int bdrv_reopen(BlockDriverState *bs, int bdrv_flags, Error **errp);
-int bdrv_reopen_prepare(BDRVReopenState *reopen_state,
-                        BlockReopenQueue *queue, Error **errp);
-void bdrv_reopen_commit(BDRVReopenState *reopen_state);
-void bdrv_reopen_abort(BDRVReopenState *reopen_state);
-void bdrv_close(BlockDriverState *bs);
-void bdrv_add_close_notifier(BlockDriverState *bs, Notifier *notify);
-int bdrv_attach_dev(BlockDriverState *bs, void *dev);
-void bdrv_attach_dev_nofail(BlockDriverState *bs, void *dev);
-void bdrv_detach_dev(BlockDriverState *bs, void *dev);
-void *bdrv_get_attached_dev(BlockDriverState *bs);
-void bdrv_set_dev_ops(BlockDriverState *bs, const BlockDevOps *ops,
-                      void *opaque);
-void bdrv_dev_eject_request(BlockDriverState *bs, bool force);
-bool bdrv_dev_has_removable_media(BlockDriverState *bs);
-bool bdrv_dev_is_tray_open(BlockDriverState *bs);
-bool bdrv_dev_is_medium_locked(BlockDriverState *bs);
-int bdrv_read(BlockDriverState *bs, int64_t sector_num,
-              uint8_t *buf, int nb_sectors);
-int bdrv_read_unthrottled(BlockDriverState *bs, int64_t sector_num,
-                          uint8_t *buf, int nb_sectors);
-int bdrv_write(BlockDriverState *bs, int64_t sector_num,
-               const uint8_t *buf, int nb_sectors);
-int bdrv_writev(BlockDriverState *bs, int64_t sector_num, QEMUIOVector *qiov);
-int bdrv_pread(BlockDriverState *bs, int64_t offset,
-               void *buf, int count);
-int bdrv_pwrite(BlockDriverState *bs, int64_t offset,
-                const void *buf, int count);
-int bdrv_pwritev(BlockDriverState *bs, int64_t offset, QEMUIOVector *qiov);
-int bdrv_pwrite_sync(BlockDriverState *bs, int64_t offset,
-    const void *buf, int count);
-int coroutine_fn bdrv_co_readv(BlockDriverState *bs, int64_t sector_num,
-    int nb_sectors, QEMUIOVector *qiov);
-int coroutine_fn bdrv_co_copy_on_readv(BlockDriverState *bs,
-    int64_t sector_num, int nb_sectors, QEMUIOVector *qiov);
-int coroutine_fn bdrv_co_writev(BlockDriverState *bs, int64_t sector_num,
-    int nb_sectors, QEMUIOVector *qiov);
-/*
- * Efficiently zero a region of the disk image.  Note that this is a regular
- * I/O request like read or write and should have a reasonable size.  This
- * function is not suitable for zeroing the entire image in a single request
- * because it may allocate memory for the entire region.
- */
-int coroutine_fn bdrv_co_write_zeroes(BlockDriverState *bs, int64_t sector_num,
-    int nb_sectors);
-int coroutine_fn bdrv_co_is_allocated(BlockDriverState *bs, int64_t sector_num,
-    int nb_sectors, int *pnum);
-int coroutine_fn bdrv_co_is_allocated_above(BlockDriverState *top,
-                                            BlockDriverState *base,
-                                            int64_t sector_num,
-                                            int nb_sectors, int *pnum);
-BlockDriverState *bdrv_find_backing_image(BlockDriverState *bs,
-    const char *backing_file);
-int bdrv_get_backing_file_depth(BlockDriverState *bs);
-int bdrv_truncate(BlockDriverState *bs, int64_t offset);
-int64_t bdrv_getlength(BlockDriverState *bs);
-int64_t bdrv_get_allocated_file_size(BlockDriverState *bs);
-void bdrv_get_geometry(BlockDriverState *bs, uint64_t *nb_sectors_ptr);
-int bdrv_commit(BlockDriverState *bs);
-int bdrv_commit_all(void);
-int bdrv_change_backing_file(BlockDriverState *bs,
-    const char *backing_file, const char *backing_fmt);
-void bdrv_register(BlockDriver *bdrv);
-int bdrv_drop_intermediate(BlockDriverState *active, BlockDriverState *top,
-                           BlockDriverState *base);
-BlockDriverState *bdrv_find_overlay(BlockDriverState *active,
-                                    BlockDriverState *bs);
-BlockDriverState *bdrv_find_base(BlockDriverState *bs);
-
-
-typedef struct BdrvCheckResult {
-    int corruptions;
-    int leaks;
-    int check_errors;
-    int corruptions_fixed;
-    int leaks_fixed;
-    int64_t image_end_offset;
-    BlockFragInfo bfi;
-} BdrvCheckResult;
-
-typedef enum {
-    BDRV_FIX_LEAKS    = 1,
-    BDRV_FIX_ERRORS   = 2,
-} BdrvCheckMode;
-
-int bdrv_check(BlockDriverState *bs, BdrvCheckResult *res, BdrvCheckMode fix);
-
-/* async block I/O */
-typedef void BlockDriverDirtyHandler(BlockDriverState *bs, int64_t sector,
-                                     int sector_num);
-BlockDriverAIOCB *bdrv_aio_readv(BlockDriverState *bs, int64_t sector_num,
-                                 QEMUIOVector *iov, int nb_sectors,
-                                 BlockDriverCompletionFunc *cb, void *opaque);
-BlockDriverAIOCB *bdrv_aio_writev(BlockDriverState *bs, int64_t sector_num,
-                                  QEMUIOVector *iov, int nb_sectors,
-                                  BlockDriverCompletionFunc *cb, void *opaque);
-BlockDriverAIOCB *bdrv_aio_flush(BlockDriverState *bs,
-                                 BlockDriverCompletionFunc *cb, void *opaque);
-BlockDriverAIOCB *bdrv_aio_discard(BlockDriverState *bs,
-                                   int64_t sector_num, int nb_sectors,
-                                   BlockDriverCompletionFunc *cb, void *opaque);
-void bdrv_aio_cancel(BlockDriverAIOCB *acb);
-
-typedef struct BlockRequest {
-    /* Fields to be filled by multiwrite caller */
-    int64_t sector;
-    int nb_sectors;
-    QEMUIOVector *qiov;
-    BlockDriverCompletionFunc *cb;
-    void *opaque;
-
-    /* Filled by multiwrite implementation */
-    int error;
-} BlockRequest;
-
-int bdrv_aio_multiwrite(BlockDriverState *bs, BlockRequest *reqs,
-    int num_reqs);
-
-/* sg packet commands */
-int bdrv_ioctl(BlockDriverState *bs, unsigned long int req, void *buf);
-BlockDriverAIOCB *bdrv_aio_ioctl(BlockDriverState *bs,
-        unsigned long int req, void *buf,
-        BlockDriverCompletionFunc *cb, void *opaque);
-
-/* Invalidate any cached metadata used by image formats */
-void bdrv_invalidate_cache(BlockDriverState *bs);
-void bdrv_invalidate_cache_all(void);
-
-void bdrv_clear_incoming_migration_all(void);
-
-/* Ensure contents are flushed to disk.  */
-int bdrv_flush(BlockDriverState *bs);
-int coroutine_fn bdrv_co_flush(BlockDriverState *bs);
-int bdrv_flush_all(void);
-void bdrv_close_all(void);
-void bdrv_drain_all(void);
-
-int bdrv_discard(BlockDriverState *bs, int64_t sector_num, int nb_sectors);
-int bdrv_co_discard(BlockDriverState *bs, int64_t sector_num, int nb_sectors);
-int bdrv_has_zero_init_1(BlockDriverState *bs);
-int bdrv_has_zero_init(BlockDriverState *bs);
-int bdrv_is_allocated(BlockDriverState *bs, int64_t sector_num, int nb_sectors,
-                      int *pnum);
-int bdrv_is_allocated_above(BlockDriverState *top, BlockDriverState *base,
-                            int64_t sector_num, int nb_sectors, int *pnum);
-
-void bdrv_set_on_error(BlockDriverState *bs, BlockdevOnError on_read_error,
-                       BlockdevOnError on_write_error);
-BlockdevOnError bdrv_get_on_error(BlockDriverState *bs, bool is_read);
-BlockErrorAction bdrv_get_error_action(BlockDriverState *bs, bool is_read, int error);
-void bdrv_error_action(BlockDriverState *bs, BlockErrorAction action,
-                       bool is_read, int error);
-int bdrv_is_read_only(BlockDriverState *bs);
-int bdrv_is_sg(BlockDriverState *bs);
-int bdrv_enable_write_cache(BlockDriverState *bs);
-void bdrv_set_enable_write_cache(BlockDriverState *bs, bool wce);
-int bdrv_is_inserted(BlockDriverState *bs);
-int bdrv_media_changed(BlockDriverState *bs);
-void bdrv_lock_medium(BlockDriverState *bs, bool locked);
-void bdrv_eject(BlockDriverState *bs, bool eject_flag);
-const char *bdrv_get_format_name(BlockDriverState *bs);
-BlockDriverState *bdrv_find(const char *name);
-BlockDriverState *bdrv_next(BlockDriverState *bs);
-void bdrv_iterate(void (*it)(void *opaque, BlockDriverState *bs),
-                  void *opaque);
-int bdrv_is_encrypted(BlockDriverState *bs);
-int bdrv_key_required(BlockDriverState *bs);
-int bdrv_set_key(BlockDriverState *bs, const char *key);
-int bdrv_query_missing_keys(void);
-void bdrv_iterate_format(void (*it)(void *opaque, const char *name),
-                         void *opaque);
-const char *bdrv_get_device_name(BlockDriverState *bs);
-int bdrv_get_flags(BlockDriverState *bs);
-int bdrv_write_compressed(BlockDriverState *bs, int64_t sector_num,
-                          const uint8_t *buf, int nb_sectors);
-int bdrv_get_info(BlockDriverState *bs, BlockDriverInfo *bdi);
-void bdrv_round_to_clusters(BlockDriverState *bs,
-                            int64_t sector_num, int nb_sectors,
-                            int64_t *cluster_sector_num,
-                            int *cluster_nb_sectors);
-
-const char *bdrv_get_encrypted_filename(BlockDriverState *bs);
-void bdrv_get_backing_filename(BlockDriverState *bs,
-                               char *filename, int filename_size);
-void bdrv_get_full_backing_filename(BlockDriverState *bs,
-                                    char *dest, size_t sz);
-int bdrv_is_snapshot(BlockDriverState *bs);
-
-int path_is_absolute(const char *path);
-void path_combine(char *dest, int dest_size,
-                  const char *base_path,
-                  const char *filename);
-
-int bdrv_writev_vmstate(BlockDriverState *bs, QEMUIOVector *qiov, int64_t pos);
-int bdrv_save_vmstate(BlockDriverState *bs, const uint8_t *buf,
-                      int64_t pos, int size);
-
-int bdrv_load_vmstate(BlockDriverState *bs, uint8_t *buf,
-                      int64_t pos, int size);
-
-void bdrv_img_create(const char *filename, const char *fmt,
-                     const char *base_filename, const char *base_fmt,
-                     char *options, uint64_t img_size, int flags,
-                     Error **errp, bool quiet);
-
-void bdrv_set_buffer_alignment(BlockDriverState *bs, int align);
-void *qemu_blockalign(BlockDriverState *bs, size_t size);
-bool bdrv_qiov_is_aligned(BlockDriverState *bs, QEMUIOVector *qiov);
-
-struct HBitmapIter;
-void bdrv_set_dirty_tracking(BlockDriverState *bs, int granularity);
-int bdrv_get_dirty(BlockDriverState *bs, int64_t sector);
-void bdrv_set_dirty(BlockDriverState *bs, int64_t cur_sector, int nr_sectors);
-void bdrv_reset_dirty(BlockDriverState *bs, int64_t cur_sector, int nr_sectors);
-void bdrv_dirty_iter_init(BlockDriverState *bs, struct HBitmapIter *hbi);
-int64_t bdrv_get_dirty_count(BlockDriverState *bs);
-
-void bdrv_enable_copy_on_read(BlockDriverState *bs);
-void bdrv_disable_copy_on_read(BlockDriverState *bs);
-
-void bdrv_set_in_use(BlockDriverState *bs, int in_use);
-int bdrv_in_use(BlockDriverState *bs);
-
-#ifdef CONFIG_LINUX_AIO
-int raw_get_aio_fd(BlockDriverState *bs);
-#else
-static inline int raw_get_aio_fd(BlockDriverState *bs)
-{
-    return -ENOTSUP;
-}
-#endif
-
-enum BlockAcctType {
-    BDRV_ACCT_READ,
-    BDRV_ACCT_WRITE,
-    BDRV_ACCT_FLUSH,
-    BDRV_MAX_IOTYPE,
-};
-
-typedef struct BlockAcctCookie {
-    int64_t bytes;
-    int64_t start_time_ns;
-    enum BlockAcctType type;
-} BlockAcctCookie;
-
-void bdrv_acct_start(BlockDriverState *bs, BlockAcctCookie *cookie,
-        int64_t bytes, enum BlockAcctType type);
-void bdrv_acct_done(BlockDriverState *bs, BlockAcctCookie *cookie);
-
-typedef enum {
-    BLKDBG_L1_UPDATE,
-
-    BLKDBG_L1_GROW_ALLOC_TABLE,
-    BLKDBG_L1_GROW_WRITE_TABLE,
-    BLKDBG_L1_GROW_ACTIVATE_TABLE,
-
-    BLKDBG_L2_LOAD,
-    BLKDBG_L2_UPDATE,
-    BLKDBG_L2_UPDATE_COMPRESSED,
-    BLKDBG_L2_ALLOC_COW_READ,
-    BLKDBG_L2_ALLOC_WRITE,
-
-    BLKDBG_READ_AIO,
-    BLKDBG_READ_BACKING_AIO,
-    BLKDBG_READ_COMPRESSED,
-
-    BLKDBG_WRITE_AIO,
-    BLKDBG_WRITE_COMPRESSED,
-
-    BLKDBG_VMSTATE_LOAD,
-    BLKDBG_VMSTATE_SAVE,
-
-    BLKDBG_COW_READ,
-    BLKDBG_COW_WRITE,
-
-    BLKDBG_REFTABLE_LOAD,
-    BLKDBG_REFTABLE_GROW,
-
-    BLKDBG_REFBLOCK_LOAD,
-    BLKDBG_REFBLOCK_UPDATE,
-    BLKDBG_REFBLOCK_UPDATE_PART,
-    BLKDBG_REFBLOCK_ALLOC,
-    BLKDBG_REFBLOCK_ALLOC_HOOKUP,
-    BLKDBG_REFBLOCK_ALLOC_WRITE,
-    BLKDBG_REFBLOCK_ALLOC_WRITE_BLOCKS,
-    BLKDBG_REFBLOCK_ALLOC_WRITE_TABLE,
-    BLKDBG_REFBLOCK_ALLOC_SWITCH_TABLE,
-
-    BLKDBG_CLUSTER_ALLOC,
-    BLKDBG_CLUSTER_ALLOC_BYTES,
-    BLKDBG_CLUSTER_FREE,
-
-    BLKDBG_FLUSH_TO_OS,
-    BLKDBG_FLUSH_TO_DISK,
-
-    BLKDBG_EVENT_MAX,
-} BlkDebugEvent;
-
-#define BLKDBG_EVENT(bs, evt) bdrv_debug_event(bs, evt)
-void bdrv_debug_event(BlockDriverState *bs, BlkDebugEvent event);
-
-int bdrv_debug_breakpoint(BlockDriverState *bs, const char *event,
-                           const char *tag);
-int bdrv_debug_resume(BlockDriverState *bs, const char *tag);
-bool bdrv_debug_is_suspended(BlockDriverState *bs, const char *tag);
-
-#endif
diff --git a/contrib/qemu/include/block/block_int.h b/contrib/qemu/include/block/block_int.h
deleted file mode 100644
index c6ac871e210..00000000000
--- a/contrib/qemu/include/block/block_int.h
+++ /dev/null
@@ -1,421 +0,0 @@
-/*
- * QEMU System Emulator block driver
- *
- * Copyright (c) 2003 Fabrice Bellard
- *
- * Permission is hereby granted, free of charge, to any person obtaining a copy
- * of this software and associated documentation files (the "Software"), to deal
- * in the Software without restriction, including without limitation the rights
- * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
- * copies of the Software, and to permit persons to whom the Software is
- * furnished to do so, subject to the following conditions:
- *
- * The above copyright notice and this permission notice shall be included in
- * all copies or substantial portions of the Software.
- *
- * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
- * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
- * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
- * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
- * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
- * THE SOFTWARE.
- */
-#ifndef BLOCK_INT_H
-#define BLOCK_INT_H
-
-#include "block/block.h"
-#include "qemu/option.h"
-#include "qemu/queue.h"
-#include "block/coroutine.h"
-#include "qemu/timer.h"
-#include "qapi-types.h"
-#include "qapi/qmp/qerror.h"
-#include "monitor/monitor.h"
-#include "qemu/hbitmap.h"
-#include "block/snapshot.h"
-
-#define BLOCK_FLAG_ENCRYPT          1
-#define BLOCK_FLAG_COMPAT6          4
-#define BLOCK_FLAG_LAZY_REFCOUNTS   8
-
-#define BLOCK_IO_LIMIT_READ     0
-#define BLOCK_IO_LIMIT_WRITE    1
-#define BLOCK_IO_LIMIT_TOTAL    2
-
-#define BLOCK_IO_SLICE_TIME     100000000
-#define NANOSECONDS_PER_SECOND  1000000000.0
-
-#define BLOCK_OPT_SIZE              "size"
-#define BLOCK_OPT_ENCRYPT           "encryption"
-#define BLOCK_OPT_COMPAT6           "compat6"
-#define BLOCK_OPT_BACKING_FILE      "backing_file"
-#define BLOCK_OPT_BACKING_FMT       "backing_fmt"
-#define BLOCK_OPT_CLUSTER_SIZE      "cluster_size"
-#define BLOCK_OPT_TABLE_SIZE        "table_size"
-#define BLOCK_OPT_PREALLOC          "preallocation"
-#define BLOCK_OPT_SUBFMT            "subformat"
-#define BLOCK_OPT_COMPAT_LEVEL      "compat"
-#define BLOCK_OPT_LAZY_REFCOUNTS    "lazy_refcounts"
-#define BLOCK_OPT_ADAPTER_TYPE      "adapter_type"
-
-typedef struct BdrvTrackedRequest {
-    BlockDriverState *bs;
-    int64_t sector_num;
-    int nb_sectors;
-    bool is_write;
-    QLIST_ENTRY(BdrvTrackedRequest) list;
-    Coroutine *co; /* owner, used for deadlock detection */
-    CoQueue wait_queue; /* coroutines blocked on this request */
-} BdrvTrackedRequest;
-
-
-typedef struct BlockIOLimit {
-    int64_t bps[3];
-    int64_t iops[3];
-} BlockIOLimit;
-
-typedef struct BlockIOBaseValue {
-    uint64_t bytes[2];
-    uint64_t ios[2];
-} BlockIOBaseValue;
-
-struct BlockDriver {
-    const char *format_name;
-    int instance_size;
-    int (*bdrv_probe)(const uint8_t *buf, int buf_size, const char *filename);
-    int (*bdrv_probe_device)(const char *filename);
-
-    /* Any driver implementing this callback is expected to be able to handle
-     * NULL file names in its .bdrv_open() implementation */
-    void (*bdrv_parse_filename)(const char *filename, QDict *options, Error **errp);
-
-    /* For handling image reopen for split or non-split files */
-    int (*bdrv_reopen_prepare)(BDRVReopenState *reopen_state,
-                               BlockReopenQueue *queue, Error **errp);
-    void (*bdrv_reopen_commit)(BDRVReopenState *reopen_state);
-    void (*bdrv_reopen_abort)(BDRVReopenState *reopen_state);
-
-    int (*bdrv_open)(BlockDriverState *bs, QDict *options, int flags);
-    int (*bdrv_file_open)(BlockDriverState *bs, QDict *options, int flags);
-    int (*bdrv_read)(BlockDriverState *bs, int64_t sector_num,
-                     uint8_t *buf, int nb_sectors);
-    int (*bdrv_write)(BlockDriverState *bs, int64_t sector_num,
-                      const uint8_t *buf, int nb_sectors);
-    void (*bdrv_close)(BlockDriverState *bs);
-    void (*bdrv_rebind)(BlockDriverState *bs);
-    int (*bdrv_create)(const char *filename, QEMUOptionParameter *options);
-    int (*bdrv_set_key)(BlockDriverState *bs, const char *key);
-    int (*bdrv_make_empty)(BlockDriverState *bs);
-    /* aio */
-    BlockDriverAIOCB *(*bdrv_aio_readv)(BlockDriverState *bs,
-        int64_t sector_num, QEMUIOVector *qiov, int nb_sectors,
-        BlockDriverCompletionFunc *cb, void *opaque);
-    BlockDriverAIOCB *(*bdrv_aio_writev)(BlockDriverState *bs,
-        int64_t sector_num, QEMUIOVector *qiov, int nb_sectors,
-        BlockDriverCompletionFunc *cb, void *opaque);
-    BlockDriverAIOCB *(*bdrv_aio_flush)(BlockDriverState *bs,
-        BlockDriverCompletionFunc *cb, void *opaque);
-    BlockDriverAIOCB *(*bdrv_aio_discard)(BlockDriverState *bs,
-        int64_t sector_num, int nb_sectors,
-        BlockDriverCompletionFunc *cb, void *opaque);
-
-    int coroutine_fn (*bdrv_co_readv)(BlockDriverState *bs,
-        int64_t sector_num, int nb_sectors, QEMUIOVector *qiov);
-    int coroutine_fn (*bdrv_co_writev)(BlockDriverState *bs,
-        int64_t sector_num, int nb_sectors, QEMUIOVector *qiov);
-    /*
-     * Efficiently zero a region of the disk image.  Typically an image format
-     * would use a compact metadata representation to implement this.  This
-     * function pointer may be NULL and .bdrv_co_writev() will be called
-     * instead.
-     */
-    int coroutine_fn (*bdrv_co_write_zeroes)(BlockDriverState *bs,
-        int64_t sector_num, int nb_sectors);
-    int coroutine_fn (*bdrv_co_discard)(BlockDriverState *bs,
-        int64_t sector_num, int nb_sectors);
-    int coroutine_fn (*bdrv_co_is_allocated)(BlockDriverState *bs,
-        int64_t sector_num, int nb_sectors, int *pnum);
-
-    /*
-     * Invalidate any cached meta-data.
-     */
-    void (*bdrv_invalidate_cache)(BlockDriverState *bs);
-
-    /*
-     * Flushes all data that was already written to the OS all the way down to
-     * the disk (for example raw-posix calls fsync()).
-     */
-    int coroutine_fn (*bdrv_co_flush_to_disk)(BlockDriverState *bs);
-
-    /*
-     * Flushes all internal caches to the OS. The data may still sit in a
-     * writeback cache of the host OS, but it will survive a crash of the qemu
-     * process.
-     */
-    int coroutine_fn (*bdrv_co_flush_to_os)(BlockDriverState *bs);
-
-    const char *protocol_name;
-    int (*bdrv_truncate)(BlockDriverState *bs, int64_t offset);
-    int64_t (*bdrv_getlength)(BlockDriverState *bs);
-    int64_t (*bdrv_get_allocated_file_size)(BlockDriverState *bs);
-    int (*bdrv_write_compressed)(BlockDriverState *bs, int64_t sector_num,
-                                 const uint8_t *buf, int nb_sectors);
-
-    int (*bdrv_snapshot_create)(BlockDriverState *bs,
-                                QEMUSnapshotInfo *sn_info);
-    int (*bdrv_snapshot_goto)(BlockDriverState *bs,
-                              const char *snapshot_id);
-    int (*bdrv_snapshot_delete)(BlockDriverState *bs, const char *snapshot_id);
-    int (*bdrv_snapshot_list)(BlockDriverState *bs,
-                              QEMUSnapshotInfo **psn_info);
-    int (*bdrv_snapshot_load_tmp)(BlockDriverState *bs,
-                                  const char *snapshot_name);
-    int (*bdrv_get_info)(BlockDriverState *bs, BlockDriverInfo *bdi);
-
-    int (*bdrv_save_vmstate)(BlockDriverState *bs, QEMUIOVector *qiov,
-                             int64_t pos);
-    int (*bdrv_load_vmstate)(BlockDriverState *bs, uint8_t *buf,
-                             int64_t pos, int size);
-
-    int (*bdrv_change_backing_file)(BlockDriverState *bs,
-        const char *backing_file, const char *backing_fmt);
-
-    /* removable device specific */
-    int (*bdrv_is_inserted)(BlockDriverState *bs);
-    int (*bdrv_media_changed)(BlockDriverState *bs);
-    void (*bdrv_eject)(BlockDriverState *bs, bool eject_flag);
-    void (*bdrv_lock_medium)(BlockDriverState *bs, bool locked);
-
-    /* to control generic scsi devices */
-    int (*bdrv_ioctl)(BlockDriverState *bs, unsigned long int req, void *buf);
-    BlockDriverAIOCB *(*bdrv_aio_ioctl)(BlockDriverState *bs,
-        unsigned long int req, void *buf,
-        BlockDriverCompletionFunc *cb, void *opaque);
-
-    /* List of options for creating images, terminated by name == NULL */
-    QEMUOptionParameter *create_options;
-
-
-    /*
-     * Returns 0 for completed check, -errno for internal errors.
-     * The check results are stored in result.
-     */
-    int (*bdrv_check)(BlockDriverState* bs, BdrvCheckResult *result,
-        BdrvCheckMode fix);
-
-    void (*bdrv_debug_event)(BlockDriverState *bs, BlkDebugEvent event);
-
-    /* TODO Better pass a option string/QDict/QemuOpts to add any rule? */
-    int (*bdrv_debug_breakpoint)(BlockDriverState *bs, const char *event,
-        const char *tag);
-    int (*bdrv_debug_resume)(BlockDriverState *bs, const char *tag);
-    bool (*bdrv_debug_is_suspended)(BlockDriverState *bs, const char *tag);
-
-    /*
-     * Returns 1 if newly created images are guaranteed to contain only
-     * zeros, 0 otherwise.
-     */
-    int (*bdrv_has_zero_init)(BlockDriverState *bs);
-
-    QLIST_ENTRY(BlockDriver) list;
-};
-
-/*
- * Note: the function bdrv_append() copies and swaps contents of
- * BlockDriverStates, so if you add new fields to this struct, please
- * inspect bdrv_append() to determine if the new fields need to be
- * copied as well.
- */
-struct BlockDriverState {
-    int64_t total_sectors; /* if we are reading a disk image, give its
-                              size in sectors */
-    int read_only; /* if true, the media is read only */
-    int open_flags; /* flags used to open the file, re-used for re-open */
-    int encrypted; /* if true, the media is encrypted */
-    int valid_key; /* if true, a valid encryption key has been set */
-    int sg;        /* if true, the device is a /dev/sg* */
-    int copy_on_read; /* if true, copy read backing sectors into image
-                         note this is a reference count */
-
-    BlockDriver *drv; /* NULL means no media */
-    void *opaque;
-
-    void *dev;                  /* attached device model, if any */
-    /* TODO change to DeviceState when all users are qdevified */
-    const BlockDevOps *dev_ops;
-    void *dev_opaque;
-
-    char filename[1024];
-    char backing_file[1024]; /* if non zero, the image is a diff of
-                                this file image */
-    char backing_format[16]; /* if non-zero and backing_file exists */
-    int is_temporary;
-
-    BlockDriverState *backing_hd;
-    BlockDriverState *file;
-
-    NotifierList close_notifiers;
-
-    /* Callback before write request is processed */
-    NotifierWithReturnList before_write_notifiers;
-
-    /* number of in-flight copy-on-read requests */
-    unsigned int copy_on_read_in_flight;
-
-    /* the time for latest disk I/O */
-    int64_t slice_start;
-    int64_t slice_end;
-    BlockIOLimit io_limits;
-    BlockIOBaseValue slice_submitted;
-    CoQueue      throttled_reqs;
-    QEMUTimer    *block_timer;
-    bool         io_limits_enabled;
-
-    /* I/O stats (display with "info blockstats"). */
-    uint64_t nr_bytes[BDRV_MAX_IOTYPE];
-    uint64_t nr_ops[BDRV_MAX_IOTYPE];
-    uint64_t total_time_ns[BDRV_MAX_IOTYPE];
-    uint64_t wr_highest_sector;
-
-    /* Whether the disk can expand beyond total_sectors */
-    int growable;
-
-    /* the memory alignment required for the buffers handled by this driver */
-    int buffer_alignment;
-
-    /* do we need to tell the quest if we have a volatile write cache? */
-    int enable_write_cache;
-
-    /* NOTE: the following infos are only hints for real hardware
-       drivers. They are not used by the block driver */
-    BlockdevOnError on_read_error, on_write_error;
-    bool iostatus_enabled;
-    BlockDeviceIoStatus iostatus;
-    char device_name[32];
-    HBitmap *dirty_bitmap;
-    int in_use; /* users other than guest access, eg. block migration */
-    QTAILQ_ENTRY(BlockDriverState) list;
-
-    QLIST_HEAD(, BdrvTrackedRequest) tracked_requests;
-
-    /* long-running background operation */
-    BlockJob *job;
-
-    QDict *options;
-};
-
-int get_tmp_filename(char *filename, int size);
-
-void bdrv_set_io_limits(BlockDriverState *bs,
-                        BlockIOLimit *io_limits);
-
-/**
- * bdrv_add_before_write_notifier:
- *
- * Register a callback that is invoked before write requests are processed but
- * after any throttling or waiting for overlapping requests.
- */
-void bdrv_add_before_write_notifier(BlockDriverState *bs,
-                                    NotifierWithReturn *notifier);
-
-/**
- * bdrv_get_aio_context:
- *
- * Returns: the currently bound #AioContext
- */
-AioContext *bdrv_get_aio_context(BlockDriverState *bs);
-
-#ifdef _WIN32
-int is_windows_drive(const char *filename);
-#endif
-void bdrv_emit_qmp_error_event(const BlockDriverState *bdrv,
-                               enum MonitorEvent ev,
-                               BlockErrorAction action, bool is_read);
-
-/**
- * stream_start:
- * @bs: Block device to operate on.
- * @base: Block device that will become the new base, or %NULL to
- * flatten the whole backing file chain onto @bs.
- * @base_id: The file name that will be written to @bs as the new
- * backing file if the job completes.  Ignored if @base is %NULL.
- * @speed: The maximum speed, in bytes per second, or 0 for unlimited.
- * @on_error: The action to take upon error.
- * @cb: Completion function for the job.
- * @opaque: Opaque pointer value passed to @cb.
- * @errp: Error object.
- *
- * Start a streaming operation on @bs.  Clusters that are unallocated
- * in @bs, but allocated in any image between @base and @bs (both
- * exclusive) will be written to @bs.  At the end of a successful
- * streaming job, the backing file of @bs will be changed to
- * @base_id in the written image and to @base in the live BlockDriverState.
- */
-void stream_start(BlockDriverState *bs, BlockDriverState *base,
-                  const char *base_id, int64_t speed, BlockdevOnError on_error,
-                  BlockDriverCompletionFunc *cb,
-                  void *opaque, Error **errp);
-
-/**
- * commit_start:
- * @bs: Top Block device
- * @base: Block device that will be written into, and become the new top
- * @speed: The maximum speed, in bytes per second, or 0 for unlimited.
- * @on_error: The action to take upon error.
- * @cb: Completion function for the job.
- * @opaque: Opaque pointer value passed to @cb.
- * @errp: Error object.
- *
- */
-void commit_start(BlockDriverState *bs, BlockDriverState *base,
-                 BlockDriverState *top, int64_t speed,
-                 BlockdevOnError on_error, BlockDriverCompletionFunc *cb,
-                 void *opaque, Error **errp);
-
-/*
- * mirror_start:
- * @bs: Block device to operate on.
- * @target: Block device to write to.
- * @speed: The maximum speed, in bytes per second, or 0 for unlimited.
- * @granularity: The chosen granularity for the dirty bitmap.
- * @buf_size: The amount of data that can be in flight at one time.
- * @mode: Whether to collapse all images in the chain to the target.
- * @on_source_error: The action to take upon error reading from the source.
- * @on_target_error: The action to take upon error writing to the target.
- * @cb: Completion function for the job.
- * @opaque: Opaque pointer value passed to @cb.
- * @errp: Error object.
- *
- * Start a mirroring operation on @bs.  Clusters that are allocated
- * in @bs will be written to @bs until the job is cancelled or
- * manually completed.  At the end of a successful mirroring job,
- * @bs will be switched to read from @target.
- */
-void mirror_start(BlockDriverState *bs, BlockDriverState *target,
-                  int64_t speed, int64_t granularity, int64_t buf_size,
-                  MirrorSyncMode mode, BlockdevOnError on_source_error,
-                  BlockdevOnError on_target_error,
-                  BlockDriverCompletionFunc *cb,
-                  void *opaque, Error **errp);
-
-/*
- * backup_start:
- * @bs: Block device to operate on.
- * @target: Block device to write to.
- * @speed: The maximum speed, in bytes per second, or 0 for unlimited.
- * @on_source_error: The action to take upon error reading from the source.
- * @on_target_error: The action to take upon error writing to the target.
- * @cb: Completion function for the job.
- * @opaque: Opaque pointer value passed to @cb.
- *
- * Start a backup operation on @bs.  Clusters in @bs are written to @target
- * until the job is cancelled or manually completed.
- */
-void backup_start(BlockDriverState *bs, BlockDriverState *target,
-                  int64_t speed, BlockdevOnError on_source_error,
-                  BlockdevOnError on_target_error,
-                  BlockDriverCompletionFunc *cb, void *opaque,
-                  Error **errp);
-
-#endif /* BLOCK_INT_H */
diff --git a/contrib/qemu/include/block/blockjob.h b/contrib/qemu/include/block/blockjob.h
deleted file mode 100644
index c290d07bba0..00000000000
--- a/contrib/qemu/include/block/blockjob.h
+++ /dev/null
@@ -1,278 +0,0 @@
-/*
- * Declarations for long-running block device operations
- *
- * Copyright (c) 2011 IBM Corp.
- * Copyright (c) 2012 Red Hat, Inc.
- *
- * Permission is hereby granted, free of charge, to any person obtaining a copy
- * of this software and associated documentation files (the "Software"), to deal
- * in the Software without restriction, including without limitation the rights
- * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
- * copies of the Software, and to permit persons to whom the Software is
- * furnished to do so, subject to the following conditions:
- *
- * The above copyright notice and this permission notice shall be included in
- * all copies or substantial portions of the Software.
- *
- * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
- * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
- * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
- * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
- * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
- * THE SOFTWARE.
- */
-#ifndef BLOCKJOB_H
-#define BLOCKJOB_H 1
-
-#include "block/block.h"
-
-/**
- * BlockJobType:
- *
- * A class type for block job objects.
- */
-typedef struct BlockJobType {
-    /** Derived BlockJob struct size */
-    size_t instance_size;
-
-    /** String describing the operation, part of query-block-jobs QMP API */
-    const char *job_type;
-
-    /** Optional callback for job types that support setting a speed limit */
-    void (*set_speed)(BlockJob *job, int64_t speed, Error **errp);
-
-    /** Optional callback for job types that need to forward I/O status reset */
-    void (*iostatus_reset)(BlockJob *job);
-
-    /**
-     * Optional callback for job types whose completion must be triggered
-     * manually.
-     */
-    void (*complete)(BlockJob *job, Error **errp);
-} BlockJobType;
-
-/**
- * BlockJob:
- *
- * Long-running operation on a BlockDriverState.
- */
-struct BlockJob {
-    /** The job type, including the job vtable.  */
-    const BlockJobType *job_type;
-
-    /** The block device on which the job is operating.  */
-    BlockDriverState *bs;
-
-    /**
-     * The coroutine that executes the job.  If not NULL, it is
-     * reentered when busy is false and the job is cancelled.
-     */
-    Coroutine *co;
-
-    /**
-     * Set to true if the job should cancel itself.  The flag must
-     * always be tested just before toggling the busy flag from false
-     * to true.  After a job has been cancelled, it should only yield
-     * if #qemu_aio_wait will ("sooner or later") reenter the coroutine.
-     */
-    bool cancelled;
-
-    /**
-     * Set to true if the job is either paused, or will pause itself
-     * as soon as possible (if busy == true).
-     */
-    bool paused;
-
-    /**
-     * Set to false by the job while it is in a quiescent state, where
-     * no I/O is pending and the job has yielded on any condition
-     * that is not detected by #qemu_aio_wait, such as a timer.
-     */
-    bool busy;
-
-    /** Status that is published by the query-block-jobs QMP API */
-    BlockDeviceIoStatus iostatus;
-
-    /** Offset that is published by the query-block-jobs QMP API */
-    int64_t offset;
-
-    /** Length that is published by the query-block-jobs QMP API */
-    int64_t len;
-
-    /** Speed that was set with @block_job_set_speed.  */
-    int64_t speed;
-
-    /** The completion function that will be called when the job completes.  */
-    BlockDriverCompletionFunc *cb;
-
-    /** The opaque value that is passed to the completion function.  */
-    void *opaque;
-};
-
-/**
- * block_job_create:
- * @job_type: The class object for the newly-created job.
- * @bs: The block
- * @speed: The maximum speed, in bytes per second, or 0 for unlimited.
- * @cb: Completion function for the job.
- * @opaque: Opaque pointer value passed to @cb.
- * @errp: Error object.
- *
- * Create a new long-running block device job and return it.  The job
- * will call @cb asynchronously when the job completes.  Note that
- * @bs may have been closed at the time the @cb it is called.  If
- * this is the case, the job may be reported as either cancelled or
- * completed.
- *
- * This function is not part of the public job interface; it should be
- * called from a wrapper that is specific to the job type.
- */
-void *block_job_create(const BlockJobType *job_type, BlockDriverState *bs,
-                       int64_t speed, BlockDriverCompletionFunc *cb,
-                       void *opaque, Error **errp);
-
-/**
- * block_job_sleep_ns:
- * @job: The job that calls the function.
- * @clock: The clock to sleep on.
- * @ns: How many nanoseconds to stop for.
- *
- * Put the job to sleep (assuming that it wasn't canceled) for @ns
- * nanoseconds.  Canceling the job will interrupt the wait immediately.
- */
-void block_job_sleep_ns(BlockJob *job, QEMUClock *clock, int64_t ns);
-
-/**
- * block_job_completed:
- * @job: The job being completed.
- * @ret: The status code.
- *
- * Call the completion function that was registered at creation time, and
- * free @job.
- */
-void block_job_completed(BlockJob *job, int ret);
-
-/**
- * block_job_set_speed:
- * @job: The job to set the speed for.
- * @speed: The new value
- * @errp: Error object.
- *
- * Set a rate-limiting parameter for the job; the actual meaning may
- * vary depending on the job type.
- */
-void block_job_set_speed(BlockJob *job, int64_t speed, Error **errp);
-
-/**
- * block_job_cancel:
- * @job: The job to be canceled.
- *
- * Asynchronously cancel the specified job.
- */
-void block_job_cancel(BlockJob *job);
-
-/**
- * block_job_complete:
- * @job: The job to be completed.
- * @errp: Error object.
- *
- * Asynchronously complete the specified job.
- */
-void block_job_complete(BlockJob *job, Error **errp);
-
-/**
- * block_job_is_cancelled:
- * @job: The job being queried.
- *
- * Returns whether the job is scheduled for cancellation.
- */
-bool block_job_is_cancelled(BlockJob *job);
-
-/**
- * block_job_query:
- * @job: The job to get information about.
- *
- * Return information about a job.
- */
-BlockJobInfo *block_job_query(BlockJob *job);
-
-/**
- * block_job_pause:
- * @job: The job to be paused.
- *
- * Asynchronously pause the specified job.
- */
-void block_job_pause(BlockJob *job);
-
-/**
- * block_job_resume:
- * @job: The job to be resumed.
- *
- * Resume the specified job.
- */
-void block_job_resume(BlockJob *job);
-
-/**
- * qobject_from_block_job:
- * @job: The job whose information is requested.
- *
- * Return a QDict corresponding to @job's query-block-jobs entry.
- */
-QObject *qobject_from_block_job(BlockJob *job);
-
-/**
- * block_job_ready:
- * @job: The job which is now ready to complete.
- *
- * Send a BLOCK_JOB_READY event for the specified job.
- */
-void block_job_ready(BlockJob *job);
-
-/**
- * block_job_is_paused:
- * @job: The job being queried.
- *
- * Returns whether the job is currently paused, or will pause
- * as soon as it reaches a sleeping point.
- */
-bool block_job_is_paused(BlockJob *job);
-
-/**
- * block_job_cancel_sync:
- * @job: The job to be canceled.
- *
- * Synchronously cancel the job.  The completion callback is called
- * before the function returns.  The job may actually complete
- * instead of canceling itself; the circumstances under which this
- * happens depend on the kind of job that is active.
- *
- * Returns the return value from the job if the job actually completed
- * during the call, or -ECANCELED if it was canceled.
- */
-int block_job_cancel_sync(BlockJob *job);
-
-/**
- * block_job_iostatus_reset:
- * @job: The job whose I/O status should be reset.
- *
- * Reset I/O status on @job and on BlockDriverState objects it uses,
- * other than job->bs.
- */
-void block_job_iostatus_reset(BlockJob *job);
-
-/**
- * block_job_error_action:
- * @job: The job to signal an error for.
- * @bs: The block device on which to set an I/O error.
- * @on_err: The error action setting.
- * @is_read: Whether the operation was a read.
- * @error: The error that was reported.
- *
- * Report an I/O error for a block job and possibly stop the VM.  Return the
- * action that was selected based on @on_err and @error.
- */
-BlockErrorAction block_job_error_action(BlockJob *job, BlockDriverState *bs,
-                                        BlockdevOnError on_err,
-                                        int is_read, int error);
-#endif
diff --git a/contrib/qemu/include/block/coroutine.h b/contrib/qemu/include/block/coroutine.h
deleted file mode 100644
index 377805a3b08..00000000000
--- a/contrib/qemu/include/block/coroutine.h
+++ /dev/null
@@ -1,218 +0,0 @@
-/*
- * QEMU coroutine implementation
- *
- * Copyright IBM, Corp. 2011
- *
- * Authors:
- *  Stefan Hajnoczi    <stefanha@linux.vnet.ibm.com>
- *  Kevin Wolf         <kwolf@redhat.com>
- *
- * This work is licensed under the terms of the GNU LGPL, version 2 or later.
- * See the COPYING.LIB file in the top-level directory.
- *
- */
-
-#ifndef QEMU_COROUTINE_H
-#define QEMU_COROUTINE_H
-
-#include <stdbool.h>
-#include "qemu/queue.h"
-#include "qemu/timer.h"
-
-/**
- * Coroutines are a mechanism for stack switching and can be used for
- * cooperative userspace threading.  These functions provide a simple but
- * useful flavor of coroutines that is suitable for writing sequential code,
- * rather than callbacks, for operations that need to give up control while
- * waiting for events to complete.
- *
- * These functions are re-entrant and may be used outside the global mutex.
- */
-
-/**
- * Mark a function that executes in coroutine context
- *
- * Functions that execute in coroutine context cannot be called directly from
- * normal functions.  In the future it would be nice to enable compiler or
- * static checker support for catching such errors.  This annotation might make
- * it possible and in the meantime it serves as documentation.
- *
- * For example:
- *
- *   static void coroutine_fn foo(void) {
- *       ....
- *   }
- */
-#define coroutine_fn
-
-typedef struct Coroutine Coroutine;
-
-/**
- * Coroutine entry point
- *
- * When the coroutine is entered for the first time, opaque is passed in as an
- * argument.
- *
- * When this function returns, the coroutine is destroyed automatically and
- * execution continues in the caller who last entered the coroutine.
- */
-typedef void coroutine_fn CoroutineEntry(void *opaque);
-
-/**
- * Create a new coroutine
- *
- * Use qemu_coroutine_enter() to actually transfer control to the coroutine.
- */
-Coroutine *qemu_coroutine_create(CoroutineEntry *entry);
-
-/**
- * Transfer control to a coroutine
- *
- * The opaque argument is passed as the argument to the entry point when
- * entering the coroutine for the first time.  It is subsequently ignored.
- */
-void qemu_coroutine_enter(Coroutine *coroutine, void *opaque);
-
-/**
- * Transfer control back to a coroutine's caller
- *
- * This function does not return until the coroutine is re-entered using
- * qemu_coroutine_enter().
- */
-void coroutine_fn qemu_coroutine_yield(void);
-
-/**
- * Get the currently executing coroutine
- */
-Coroutine *coroutine_fn qemu_coroutine_self(void);
-
-/**
- * Return whether or not currently inside a coroutine
- *
- * This can be used to write functions that work both when in coroutine context
- * and when not in coroutine context.  Note that such functions cannot use the
- * coroutine_fn annotation since they work outside coroutine context.
- */
-bool qemu_in_coroutine(void);
-
-
-
-/**
- * CoQueues are a mechanism to queue coroutines in order to continue executing
- * them later. They provide the fundamental primitives on which coroutine locks
- * are built.
- */
-typedef struct CoQueue {
-    QTAILQ_HEAD(, Coroutine) entries;
-    AioContext *ctx;
-} CoQueue;
-
-/**
- * Initialise a CoQueue. This must be called before any other operation is used
- * on the CoQueue.
- */
-void qemu_co_queue_init(CoQueue *queue);
-
-/**
- * Adds the current coroutine to the CoQueue and transfers control to the
- * caller of the coroutine.
- */
-void coroutine_fn qemu_co_queue_wait(CoQueue *queue);
-
-/**
- * Adds the current coroutine to the head of the CoQueue and transfers control to the
- * caller of the coroutine.
- */
-void coroutine_fn qemu_co_queue_wait_insert_head(CoQueue *queue);
-
-/**
- * Restarts the next coroutine in the CoQueue and removes it from the queue.
- *
- * Returns true if a coroutine was restarted, false if the queue is empty.
- */
-bool qemu_co_queue_next(CoQueue *queue);
-
-/**
- * Restarts all coroutines in the CoQueue and leaves the queue empty.
- */
-void qemu_co_queue_restart_all(CoQueue *queue);
-
-/**
- * Checks if the CoQueue is empty.
- */
-bool qemu_co_queue_empty(CoQueue *queue);
-
-
-/**
- * Provides a mutex that can be used to synchronise coroutines
- */
-typedef struct CoMutex {
-    bool locked;
-    CoQueue queue;
-} CoMutex;
-
-/**
- * Initialises a CoMutex. This must be called before any other operation is used
- * on the CoMutex.
- */
-void qemu_co_mutex_init(CoMutex *mutex);
-
-/**
- * Locks the mutex. If the lock cannot be taken immediately, control is
- * transferred to the caller of the current coroutine.
- */
-void coroutine_fn qemu_co_mutex_lock(CoMutex *mutex);
-
-/**
- * Unlocks the mutex and schedules the next coroutine that was waiting for this
- * lock to be run.
- */
-void coroutine_fn qemu_co_mutex_unlock(CoMutex *mutex);
-